mirror of
https://github.com/rustfs/rustfs.git
synced 2026-11-18 14:05:27 +00:00
Compare commits
79 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 7676140898 | |||
| 66eb65da06 | |||
| 85898d94f9 | |||
| caefc17dd1 | |||
| c677030610 | |||
| 08deaeea38 | |||
| 17b01efc89 | |||
| 88d8cc86b1 | |||
| 640f872054 | |||
| ee333a2774 | |||
| 177572b0d9 | |||
| 2e286d3220 | |||
| cfa93b9b40 | |||
| 142cff57a3 | |||
| 447f18db43 | |||
| 975e232a63 | |||
| d921f3cb24 | |||
| 62117054d9 | |||
| 2f51a39c64 | |||
| 292c288896 | |||
| d8519549d1 | |||
| c3efe4be2a | |||
| f56b2185b5 | |||
| d0218bb870 | |||
| 10cb203c76 | |||
| 1d3137e52a | |||
| f3df5a6808 | |||
| 87a910e335 | |||
| a94c00fc83 | |||
| bef37ecd6a | |||
| 9b24e8cef1 | |||
| 71fb13ed31 | |||
| 0f60640e8d | |||
| 1cd2b12df1 | |||
| a4d4ce5522 | |||
| b108663fcc | |||
| 396c1d8972 | |||
| 52402c762b | |||
| 7cbe41ad6f | |||
| 5f4188d1da | |||
| 18a3416a7e | |||
| a0bf1a5ef6 | |||
| ac6bc21852 | |||
| b417d25165 | |||
| d8e8ced097 | |||
| 86d9da837a | |||
| 7efd4e59f4 | |||
| 3a23b87b31 | |||
| 200eb34834 | |||
| cc5c0f6945 | |||
| 07769f052d | |||
| 061e5464a5 | |||
| ce232d7a79 | |||
| 8067ad1076 | |||
| ad7c0748c5 | |||
| 9192c36712 | |||
| 7ace7b6d77 | |||
| 4ee8651837 | |||
| 6b0e445f0e | |||
| 6babd93693 | |||
| 748e422651 | |||
| c840cd34a0 | |||
| 889acaae1c | |||
| 257f14fdbf | |||
| 667f82bf1a | |||
| 8f12b81f09 | |||
| 92ad98b1e5 | |||
| 69f8b27728 | |||
| 50952e10cf | |||
| 3ddbb80b20 | |||
| 941fa254fc | |||
| d4af3cee3e | |||
| 850640f452 | |||
| 1baaf0b84d | |||
| 83032a3671 | |||
| e110baa3e0 | |||
| f90137b73f | |||
| a5b69359d2 | |||
| c6a52d87cf |
@@ -34,6 +34,10 @@ If you want the Kafka-backed HA Tempo path, use `docker-compose-example-for-rust
|
||||
- **High Performance**: Optimized configurations for batching, compression, and memory management.
|
||||
- **Standardized Protocols**: Built entirely on OpenTelemetry standards.
|
||||
|
||||
## HTTP Request Latency Percentiles
|
||||
|
||||
The main RustFS dashboard shows request latency percentiles in milliseconds. The HTTP server request, response-body chunk, and response-body stream histograms export bucket boundaries in seconds. During a rolling upgrade or rollback that changes the bucket layout, compute percentiles from nodes that share one layout and from a query window entirely on that layout. The bundled dashboard and recording rules use a five-minute window. The mean from `_sum` divided by `_count` remains usable across the change.
|
||||
|
||||
## GET Performance Optimization Dashboards
|
||||
|
||||
Three pre-built Grafana dashboards are included for monitoring RustFS GET performance optimization rollout:
|
||||
|
||||
@@ -34,6 +34,10 @@
|
||||
- **高性能**: 针对批处理、压缩和内存管理进行了优化配置。
|
||||
- **标准化协议**: 完全基于 OpenTelemetry 标准构建。
|
||||
|
||||
## HTTP 请求延迟分位数
|
||||
|
||||
RustFS 主仪表盘以毫秒显示请求延迟分位数。HTTP 服务的请求耗时、响应体分块延迟和响应体流传输耗时直方图以秒为单位导出桶边界。滚动升级或回滚导致桶边界变化时,只对采用同一套桶边界的节点和时间窗口计算分位数。自带仪表盘和记录规则使用五分钟窗口。变更期间仍可用 `_sum` 除以 `_count` 计算平均值。
|
||||
|
||||
## GET 性能优化仪表盘
|
||||
|
||||
包含三个预构建的 Grafana 仪表盘,用于监控 RustFS GET 性能优化发布:
|
||||
|
||||
Generated
+444
-390
File diff suppressed because it is too large
Load Diff
+33
-33
@@ -151,16 +151,16 @@ rustfs-zip = { path = "./crates/zip", version = "1.0.5" }
|
||||
# engine crates move onto from s3s (rustfs/backlog#2745). The one pin of the
|
||||
# gateway in this workspace; no crate may enable a `compat-s3s*` feature except
|
||||
# the s3s-facing edge of the rustfs binary.
|
||||
rustfs-gateway-types = { git = "https://github.com/rustfs/gateway.git", rev = "fb4d21fca840036faef784aa1b0e965b03451db3", version = "0.42.0", default-features = false }
|
||||
rustfs-gateway-stream = { git = "https://github.com/rustfs/gateway.git", rev = "fb4d21fca840036faef784aa1b0e965b03451db3", version = "0.2.0", default-features = false }
|
||||
rustfs-gateway-types = { git = "https://github.com/rustfs/gateway.git", rev = "d6840d54187382d31526317e61369734c759e2cd", version = "0.42.0", default-features = false }
|
||||
rustfs-gateway-stream = { git = "https://github.com/rustfs/gateway.git", rev = "d6840d54187382d31526317e61369734c759e2cd", version = "0.2.0", default-features = false }
|
||||
# The gateway framework itself, consumed for its extension traits by the ring-2
|
||||
# adapter. `default-features = false` keeps the `server` feature, and with it the
|
||||
# server crate, out of the build.
|
||||
rustfs-gateway = { git = "https://github.com/rustfs/gateway.git", rev = "fb4d21fca840036faef784aa1b0e965b03451db3", version = "0.66.0", default-features = false }
|
||||
rustfs-gateway = { git = "https://github.com/rustfs/gateway.git", rev = "d6840d54187382d31526317e61369734c759e2cd", version = "0.66.0", default-features = false }
|
||||
# Test-only: builds the `EffectiveHost` a host-resolution test queries with.
|
||||
rustfs-gateway-http = { git = "https://github.com/rustfs/gateway.git", rev = "fb4d21fca840036faef784aa1b0e965b03451db3", default-features = false }
|
||||
rustfs-gateway-core = { git = "https://github.com/rustfs/gateway.git", rev = "fb4d21fca840036faef784aa1b0e965b03451db3", version = "0.63.0", default-features = false }
|
||||
rustfs-gateway-dialect-rustfs-admin = { git = "https://github.com/rustfs/gateway.git", rev = "fb4d21fca840036faef784aa1b0e965b03451db3", version = "0.13.0", default-features = false }
|
||||
rustfs-gateway-http = { git = "https://github.com/rustfs/gateway.git", rev = "d6840d54187382d31526317e61369734c759e2cd", default-features = false }
|
||||
rustfs-gateway-core = { git = "https://github.com/rustfs/gateway.git", rev = "d6840d54187382d31526317e61369734c759e2cd", version = "0.63.0", default-features = false }
|
||||
rustfs-gateway-dialect-rustfs-admin = { git = "https://github.com/rustfs/gateway.git", rev = "d6840d54187382d31526317e61369734c759e2cd", version = "0.13.0", default-features = false }
|
||||
|
||||
# Async Runtime and Networking
|
||||
async-channel = "2.5.0"
|
||||
@@ -168,13 +168,13 @@ async_zip = { default-features = false, version = "0.0.19" }
|
||||
mysql_async = { default-features = false, version = "0.37.1" }
|
||||
async-compression = { version = "0.4.52" }
|
||||
async-trait = "0.1.92"
|
||||
async-nats = { version = "0.52.0", default-features = false }
|
||||
async-nats = { version = "0.53.0", default-features = false }
|
||||
axum = "0.8.9"
|
||||
futures = "0.3.34"
|
||||
futures-core = "0.3.34"
|
||||
futures-lite = "2.6.1"
|
||||
futures-util = "0.3.34"
|
||||
pollster = "1.0.1"
|
||||
pollster = "1.0.2"
|
||||
pulsar = { default-features = false, version = "=6.9.0" }
|
||||
lapin = { default-features = false, version = "4.12.3" }
|
||||
hyper = { version = "1.12.0" }
|
||||
@@ -209,8 +209,8 @@ quinn = { version = "0.11.12", default-features = false, features = [
|
||||
] }
|
||||
|
||||
# Serialization and Data Formats
|
||||
apache-avro = { version = "0.22.0", features = ["snappy", "zstandard"] }
|
||||
bytes = { version = "1.12.1" }
|
||||
apache-avro = { version = "0.23.0", features = ["snappy", "zstandard"] }
|
||||
bytes = { version = "1.12.2" }
|
||||
csv-core = "0.1.13"
|
||||
bytesize = "2.7.0"
|
||||
byteorder = "1.5.0"
|
||||
@@ -242,7 +242,7 @@ jsonwebtoken = { version = "12.0.0" }
|
||||
openidconnect = { default-features = false, version = "4.1.0" }
|
||||
pbkdf2 = "0.13.0"
|
||||
p256 = { version = "0.14.0", features = ["ecdsa", "pkcs8"] }
|
||||
rsa = { version = "=0.10.0-rc.18" }
|
||||
rsa = { version = "0.10.0", features = ["pkcs1v15"] }
|
||||
rustls = { default-features = false, version = "0.23.46" }
|
||||
rustls-native-certs = "0.8"
|
||||
rustls-pki-types = "1.16.0"
|
||||
@@ -250,13 +250,13 @@ x509-parser = "0.18.1"
|
||||
sha1 = "0.11.0"
|
||||
sha2 = "0.11.1"
|
||||
subtle = "2.6"
|
||||
zeroize = { version = "1.9.1" }
|
||||
zeroize = { version = "1.9.2" }
|
||||
proptest = "1"
|
||||
|
||||
# Time and Date
|
||||
chrono = { version = "0.4.45" }
|
||||
humantime = "2.4.0"
|
||||
jiff = { version = "0.2.43" }
|
||||
jiff = { version = "1.0.1" }
|
||||
time = { version = "0.3.55" }
|
||||
|
||||
# Database
|
||||
@@ -273,18 +273,18 @@ tar-codec = "0.0.16"
|
||||
tar-framing = "0.0.16"
|
||||
atoi = "3.1.1"
|
||||
atomic_enum = "0.3.0"
|
||||
aws-config = { version = "1.13.2" }
|
||||
aws-config = { version = "1.13.3" }
|
||||
aws-credential-types = { version = "1.3.1" }
|
||||
aws-sdk-kms = { default-features = false, version = "1.131.0" }
|
||||
aws-sdk-s3 = { default-features = false, version = "1.159.0" }
|
||||
aws-sdk-sts = { default-features = false, version = "1.126.0" }
|
||||
aws-sdk-kms = { default-features = false, version = "1.133.0" }
|
||||
aws-sdk-s3 = { default-features = false, version = "1.160.0" }
|
||||
aws-sdk-sts = { default-features = false, version = "1.127.0" }
|
||||
aws-smithy-async = { version = "1.3.1" }
|
||||
aws-smithy-http-client = { default-features = false, version = "1.5.2" }
|
||||
aws-smithy-runtime-api = { version = "1.21.0" }
|
||||
aws-smithy-http-client = { default-features = false, version = "1.5.3" }
|
||||
aws-smithy-runtime-api = { version = "1.22.0" }
|
||||
aws-smithy-types = { version = "1.9.4" }
|
||||
base64-simd = "0.8.0"
|
||||
brotli = "9.1.1"
|
||||
clap = { version = "4.6.7" }
|
||||
brotli = "9.1.3"
|
||||
clap = { version = "4.6.8" }
|
||||
const-str = { version = "1.1.0" }
|
||||
convert_case = "0.12.0"
|
||||
criterion = { version = "0.8" }
|
||||
@@ -292,7 +292,7 @@ crossbeam-queue = "0.3.14"
|
||||
crossbeam-channel = "0.5.17"
|
||||
crossbeam-deque = "0.8.8"
|
||||
crossbeam-utils = "0.8.23"
|
||||
datafusion = { default-features = false, version = "55.2.0" }
|
||||
datafusion = { default-features = false, version = "55.3.0" }
|
||||
derive_builder = "0.20.2"
|
||||
ed25519-dalek = "3.0.0"
|
||||
enumset = "1.1.15"
|
||||
@@ -318,7 +318,7 @@ log = "0.4"
|
||||
memmap2 = "0.9.12"
|
||||
nixl-sys = { git = "https://github.com/houseme/nixl", rev = "cc636c83f13d692e85b67e55aed092413d16fb41" }
|
||||
lz4 = "1.28.1"
|
||||
lz4_flex = { version = "=0.14.0", default-features = false, features = ["std", "frame"] }
|
||||
lz4_flex = { version = "0.14.2", default-features = false, features = ["std", "frame"] }
|
||||
matchit = "0.9.2"
|
||||
md-5 = "0.11.0"
|
||||
mime_guess = "2.0.5"
|
||||
@@ -337,14 +337,14 @@ percent-encoding = "2.3.2"
|
||||
qrcode-rs = { version = "2.2.0", default-features = false, features = ["std", "svg"] }
|
||||
pin-project-lite = "0.2.17"
|
||||
pretty_assertions = "1.4.1"
|
||||
rand = { version = "0.10.3" }
|
||||
rand = { version = "0.10.4" }
|
||||
ratelimit = "2.0.0"
|
||||
rayon = "1.12.0"
|
||||
rustfs-erasure-codec = { version = "9.1.0" }
|
||||
reed-solomon-simd = "3.1.0"
|
||||
regex = { version = "1.13.1" }
|
||||
regex = { version = "1.13.2" }
|
||||
rumqttc = { package = "rumqttc-next", version = "0.34.0" }
|
||||
redis = { version = "1.8.1" }
|
||||
redis = { version = "1.8.2" }
|
||||
rustify = { version = "0.7", default-features = false }
|
||||
rustix = { version = "1.1.6" }
|
||||
rust-embed = { version = "8.14.0" }
|
||||
@@ -364,7 +364,7 @@ snap = "1.1.2"
|
||||
starshard = { version = "2.4.1" }
|
||||
strum = { version = "0.28.0" }
|
||||
sync_wrapper = "1.0.2"
|
||||
sysinfo = "0.40.3"
|
||||
sysinfo = "0.40.4"
|
||||
temp-env = "0.3.6"
|
||||
tempfile = "3.27.0"
|
||||
test-case = "3.4.1"
|
||||
@@ -378,15 +378,15 @@ tracing-subscriber = { version = "0.3.23" }
|
||||
transform-stream = "0.3.1"
|
||||
url = "2.5.8"
|
||||
urlencoding = "2.1.3"
|
||||
uuid = { version = "1.29.2" }
|
||||
uuid = { version = "1.29.3" }
|
||||
vaultrs = { version = "0.8.0" }
|
||||
tar = "0.4.46"
|
||||
walkdir = "2.5.0"
|
||||
winapi-util = "0.1.11"
|
||||
windows = { version = "0.62.2" }
|
||||
windows-sys = "0.61.2"
|
||||
xxhash-rust = { version = "0.8.19" }
|
||||
zip = "9.5.1"
|
||||
xxhash-rust = { version = "0.8.21" }
|
||||
zip = "9.7.0"
|
||||
zstd = "0.14.4"
|
||||
|
||||
# Observability and Metrics
|
||||
@@ -405,9 +405,9 @@ pyroscope = { version = "2.1.5" }
|
||||
# FTP and SFTP
|
||||
libunftp = { version = "0.25.0" }
|
||||
unftp-core = "0.1.1"
|
||||
suppaftp = { version = "12.2.1" }
|
||||
suppaftp = { version = "13.0.0" }
|
||||
rcgen = { version = "0.14.10", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
|
||||
russh = { version = "0.65.1" }
|
||||
russh = { version = "0.65.2" }
|
||||
russh-sftp = "3.0.1"
|
||||
|
||||
# WebDAV
|
||||
@@ -416,7 +416,7 @@ dav-server = "0.12.0"
|
||||
# Performance Analysis and Memory Profiling
|
||||
rustfs-mimalloc = { version = "0.7.4" }
|
||||
# Preserve Unicode focus filters until rustfs/backlog#2302 is resolved.
|
||||
hotpath = { version = "0.31.2", default-features = false }
|
||||
hotpath = { version = "0.32.1", default-features = false }
|
||||
# Snapshot testing for output format regression detection
|
||||
insta = { version = "1.49" }
|
||||
|
||||
|
||||
@@ -36,7 +36,7 @@ jsonwebtoken = { workspace = true, features = ["aws_lc_rs"] }
|
||||
base64-simd = { workspace = true }
|
||||
pbkdf2 = { workspace = true, optional = true }
|
||||
rand = { workspace = true, features = ["serde"] }
|
||||
rsa = { workspace = true, features = ["sha2"] }
|
||||
rsa = { workspace = true }
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
sha2 = { workspace = true, optional = true }
|
||||
thiserror.workspace = true
|
||||
|
||||
@@ -13,9 +13,9 @@
|
||||
// limitations under the License.
|
||||
|
||||
use rsa::{
|
||||
Pkcs1v15Encrypt, RsaPrivateKey, RsaPublicKey,
|
||||
Pkcs1v15Encrypt, PrivateKey, PublicKey,
|
||||
pkcs8::{DecodePrivateKey, DecodePublicKey},
|
||||
pss::{BlindedSigningKey, Signature, VerifyingKey},
|
||||
pss::{Signature, SigningKey, VerifyingKey},
|
||||
sha2::Sha256,
|
||||
signature::{RandomizedSigner, Verifier},
|
||||
traits::PublicKeyParts,
|
||||
@@ -37,7 +37,7 @@ pub struct Token {
|
||||
pub fn gencode(token: &Token, key: &str) -> Result<String> {
|
||||
let data = serde_json::to_vec(token)?;
|
||||
let mut rng = rand::rng();
|
||||
let public_key = RsaPublicKey::from_public_key_pem(key).map_err(Error::other)?;
|
||||
let public_key = PublicKey::from_public_key_pem(key).map_err(Error::other)?;
|
||||
let encrypted_data = public_key.encrypt(&mut rng, Pkcs1v15Encrypt, &data).map_err(Error::other)?;
|
||||
Ok(base64_simd::URL_SAFE_NO_PAD.encode_to_string(&encrypted_data))
|
||||
}
|
||||
@@ -51,7 +51,7 @@ pub fn parse(token: &str, key: &str) -> Result<Token> {
|
||||
let encrypted_data = base64_simd::URL_SAFE_NO_PAD
|
||||
.decode_to_vec(token.as_bytes())
|
||||
.map_err(Error::other)?;
|
||||
let private_key = RsaPrivateKey::from_pkcs8_pem(key).map_err(Error::other)?;
|
||||
let private_key = PrivateKey::from_pkcs8_pem(key).map_err(Error::other)?;
|
||||
let decrypted_data = private_key.decrypt(Pkcs1v15Encrypt, &encrypted_data).map_err(Error::other)?;
|
||||
serde_json::from_slice(&decrypted_data).map_err(Error::other)
|
||||
}
|
||||
@@ -63,8 +63,8 @@ pub fn parse(token: &str, key: &str) -> Result<Token> {
|
||||
pub fn sign_license_token(token: &Token, private_key_pem: &str) -> Result<String> {
|
||||
let payload = serde_json::to_vec(token)?;
|
||||
let mut rng = rand::rng();
|
||||
let private_key = RsaPrivateKey::from_pkcs8_pem(private_key_pem).map_err(Error::other)?;
|
||||
let signing_key = BlindedSigningKey::<Sha256>::new(private_key);
|
||||
let private_key = PrivateKey::from_pkcs8_pem(private_key_pem).map_err(Error::other)?;
|
||||
let signing_key = SigningKey::<Sha256>::new(private_key);
|
||||
let signature: Signature = signing_key.try_sign_with_rng(&mut rng, &payload).map_err(Error::other)?;
|
||||
let signature: Box<[u8]> = signature.into();
|
||||
|
||||
@@ -80,7 +80,7 @@ pub fn parse_signed_license_token(token: &str, public_key_pem: &str) -> Result<T
|
||||
let signed_payload = base64_simd::URL_SAFE_NO_PAD
|
||||
.decode_to_vec(token.as_bytes())
|
||||
.map_err(Error::other)?;
|
||||
let public_key = RsaPublicKey::from_public_key_pem(public_key_pem).map_err(Error::other)?;
|
||||
let public_key = PublicKey::from_public_key_pem(public_key_pem).map_err(Error::other)?;
|
||||
let signature_len = public_key.size();
|
||||
|
||||
if signed_payload.len() <= signature_len {
|
||||
@@ -103,7 +103,7 @@ pub fn parse_license_with_public_key(license: &str, public_key: &str) -> Result<
|
||||
mod tests {
|
||||
use super::*;
|
||||
use rsa::{
|
||||
RsaPrivateKey,
|
||||
PrivateKey,
|
||||
pkcs8::{EncodePrivateKey, EncodePublicKey, LineEnding},
|
||||
};
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
@@ -112,8 +112,8 @@ mod tests {
|
||||
fn test_sign_license_token_and_parse_signed_license_token() {
|
||||
let mut rng = rand::rng();
|
||||
let bits = 2048;
|
||||
let private_key = RsaPrivateKey::new(&mut rng, bits).expect("Failed to generate private key");
|
||||
let public_key = RsaPublicKey::from(&private_key);
|
||||
let private_key = PrivateKey::new(&mut rng, bits).expect("Failed to generate private key");
|
||||
let public_key = PublicKey::from(&private_key);
|
||||
|
||||
let private_key_pem = private_key
|
||||
.to_pkcs8_pem(LineEnding::LF)
|
||||
@@ -144,8 +144,8 @@ mod tests {
|
||||
fn test_legacy_gencode_and_parse_roundtrip() {
|
||||
let mut rng = rand::rng();
|
||||
let bits = 2048;
|
||||
let private_key = RsaPrivateKey::new(&mut rng, bits).expect("Failed to generate private key");
|
||||
let public_key = RsaPublicKey::from(&private_key);
|
||||
let private_key = PrivateKey::new(&mut rng, bits).expect("Failed to generate private key");
|
||||
let public_key = PublicKey::from(&private_key);
|
||||
|
||||
let private_key_pem = private_key
|
||||
.to_pkcs8_pem(LineEnding::LF)
|
||||
@@ -173,8 +173,8 @@ mod tests {
|
||||
#[test]
|
||||
fn test_parse_signed_license_token_rejects_tampered_payload() {
|
||||
let mut rng = rand::rng();
|
||||
let private_key = RsaPrivateKey::new(&mut rng, 2048).expect("Failed to generate private key");
|
||||
let public_key = RsaPublicKey::from(&private_key);
|
||||
let private_key = PrivateKey::new(&mut rng, 2048).expect("Failed to generate private key");
|
||||
let public_key = PublicKey::from(&private_key);
|
||||
let private_key_pem = private_key
|
||||
.to_pkcs8_pem(LineEnding::LF)
|
||||
.expect("failed to encode private key pem");
|
||||
@@ -206,8 +206,8 @@ mod tests {
|
||||
#[test]
|
||||
fn test_parse_signed_license_token_rejects_invalid_token() {
|
||||
let mut rng = rand::rng();
|
||||
let private_key = RsaPrivateKey::new(&mut rng, 2048).expect("Failed to generate private key");
|
||||
let public_key = RsaPublicKey::from(&private_key);
|
||||
let private_key = PrivateKey::new(&mut rng, 2048).expect("Failed to generate private key");
|
||||
let public_key = PublicKey::from(&private_key);
|
||||
let public_key_pem = public_key
|
||||
.to_public_key_pem(LineEnding::LF)
|
||||
.expect("failed to encode public key pem");
|
||||
|
||||
@@ -66,6 +66,12 @@ pub const LEGACY_DATA_USAGE_OBJECT_NAME: &str = ".usage.json";
|
||||
/// stale tier names from growing persisted per-tier maps without bound.
|
||||
pub const UNKNOWN_TIER: &str = "UNKNOWN_TIER";
|
||||
|
||||
/// Whether a remote tier name can be represented in scanner accounting.
|
||||
/// Local storage classes are checked separately by the configuration owner.
|
||||
pub fn is_scannable_tier_name(name: &str) -> bool {
|
||||
!name.is_empty() && name.len() <= 256 && !name.bytes().any(|byte| byte.is_ascii_control()) && name != UNKNOWN_TIER
|
||||
}
|
||||
|
||||
/// Returns true when `existing_last_update` is ahead of `now` by more than
|
||||
/// [`USAGE_LAST_UPDATE_FUTURE_TOLERANCE`], i.e. the persisted timestamp cannot be
|
||||
/// trusted for staleness comparisons and a fresh snapshot save must be allowed.
|
||||
|
||||
@@ -98,7 +98,6 @@ rustfs-madmin.workspace = true
|
||||
rustfs-filemeta.workspace = true
|
||||
bytes = { workspace = true, features = ["serde"] }
|
||||
aws-sdk-s3 = { workspace = true, default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
|
||||
aws-sdk-sts = { workspace = true, default-features = false, features = ["default-https-client", "rt-tokio"] }
|
||||
aws-smithy-http-client = { workspace = true, default-features = false, features = ["rustls-aws-lc"] }
|
||||
aws-smithy-types.workspace = true
|
||||
async-trait = { workspace = true }
|
||||
@@ -124,7 +123,7 @@ urlencoding.workspace = true
|
||||
walkdir.workspace = true
|
||||
base64-simd = { workspace = true }
|
||||
rand = { workspace = true, features = ["serde"] }
|
||||
chrono = { workspace = true, features = ["serde"] }
|
||||
jiff = { workspace = true, features = ["serde"] }
|
||||
hex-simd = { workspace = true }
|
||||
md-5 = { workspace = true }
|
||||
prost.workspace = true
|
||||
@@ -142,6 +141,7 @@ clap = { workspace = true, features = ["derive", "env"] }
|
||||
[dev-dependencies]
|
||||
async-compression = { workspace = true, features = ["tokio", "bzip2", "lz4", "xz"] }
|
||||
aws-config = { workspace = true }
|
||||
aws-sdk-sts = { workspace = true, default-features = false, features = ["default-https-client", "rt-tokio"] }
|
||||
opentelemetry-proto = { workspace = true }
|
||||
russh = { workspace = true, features = ["serde"] }
|
||||
russh-sftp = { workspace = true }
|
||||
|
||||
@@ -170,6 +170,15 @@ impl DiskFaultHarness {
|
||||
/// Start (or restart after [`Self::kill_server`]) the RustFS server with
|
||||
/// all volume directories and the same address.
|
||||
pub async fn start_server(&mut self) -> ChaosResult<()> {
|
||||
self.start_server_with_readiness(false).await
|
||||
}
|
||||
|
||||
/// Start a process once its node read-readiness probe passes.
|
||||
pub async fn start_server_at_read_quorum(&mut self) -> ChaosResult<()> {
|
||||
self.start_server_with_readiness(true).await
|
||||
}
|
||||
|
||||
async fn start_server_with_readiness(&mut self, read_quorum_only: bool) -> ChaosResult<()> {
|
||||
// The common helper always appends `env.temp_dir` as the final storage
|
||||
// path: point it at the last disk for startup and pass the remaining
|
||||
// disks explicitly, then restore it to the root so Drop cleans up
|
||||
@@ -181,7 +190,13 @@ impl DiskFaultHarness {
|
||||
let arg_refs: Vec<&str> = disk_args.iter().map(String::as_str).collect();
|
||||
let env_refs: Vec<(&str, &str)> = self.extra_env.iter().map(|(k, v)| (k.as_str(), v.as_str())).collect();
|
||||
|
||||
let result = self.env.start_rustfs_server_with_env(arg_refs, &env_refs).await;
|
||||
let result = if read_quorum_only {
|
||||
self.env
|
||||
.start_rustfs_server_with_env_at_read_quorum(arg_refs, &env_refs)
|
||||
.await
|
||||
} else {
|
||||
self.env.start_rustfs_server_with_env(arg_refs, &env_refs).await
|
||||
};
|
||||
self.env.temp_dir = self.root.to_string_lossy().to_string();
|
||||
result
|
||||
}
|
||||
|
||||
@@ -722,7 +722,7 @@ impl RustFSTestEnvironment {
|
||||
cleanup_existing: bool,
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let binary_path = rustfs_binary_path();
|
||||
self.start_rustfs_server_inner_with_binary(&binary_path, extra_args, extra_env, cleanup_existing)
|
||||
self.start_rustfs_server_inner_with_binary(&binary_path, extra_args, extra_env, cleanup_existing, false)
|
||||
.await
|
||||
}
|
||||
|
||||
@@ -732,6 +732,7 @@ impl RustFSTestEnvironment {
|
||||
extra_args: Vec<&str>,
|
||||
extra_env: &[(&str, &str)],
|
||||
cleanup_existing: bool,
|
||||
read_quorum_only: bool,
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
if cleanup_existing {
|
||||
self.cleanup_existing_processes().await?;
|
||||
@@ -756,7 +757,7 @@ impl RustFSTestEnvironment {
|
||||
self.process = Some(process);
|
||||
|
||||
// Wait for server to be ready
|
||||
self.wait_for_server_ready().await?;
|
||||
self.wait_for_server_ready_with_probe(read_quorum_only).await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -770,7 +771,7 @@ impl RustFSTestEnvironment {
|
||||
extra_args: Vec<&str>,
|
||||
extra_env: &[(&str, &str)],
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
self.start_rustfs_server_inner_with_binary(binary_path, extra_args, extra_env, true)
|
||||
self.start_rustfs_server_inner_with_binary(binary_path, extra_args, extra_env, true, false)
|
||||
.await
|
||||
}
|
||||
|
||||
@@ -782,7 +783,7 @@ impl RustFSTestEnvironment {
|
||||
extra_args: Vec<&str>,
|
||||
extra_env: &[(&str, &str)],
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
self.start_rustfs_server_inner_with_binary(binary_path, extra_args, extra_env, false)
|
||||
self.start_rustfs_server_inner_with_binary(binary_path, extra_args, extra_env, false, false)
|
||||
.await
|
||||
}
|
||||
|
||||
@@ -800,6 +801,18 @@ impl RustFSTestEnvironment {
|
||||
self.start_rustfs_server_inner(extra_args, extra_env, true).await
|
||||
}
|
||||
|
||||
/// Start a degraded read-serving process using the node readiness probe.
|
||||
/// ListBuckets retains its namespace write-quorum requirement.
|
||||
pub async fn start_rustfs_server_with_env_at_read_quorum(
|
||||
&mut self,
|
||||
extra_args: Vec<&str>,
|
||||
extra_env: &[(&str, &str)],
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let binary_path = rustfs_binary_path();
|
||||
self.start_rustfs_server_inner_with_binary(&binary_path, extra_args, extra_env, true, true)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Start RustFS server without cleaning up other running RustFS processes.
|
||||
///
|
||||
/// This is useful for tests that need multiple independent RustFS instances
|
||||
@@ -828,8 +841,16 @@ impl RustFSTestEnvironment {
|
||||
/// Fails immediately if the spawned server process has already exited
|
||||
/// (e.g. a port bind failure) instead of polling out the full timeout.
|
||||
pub async fn wait_for_server_ready(&mut self) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
self.wait_for_server_ready_with_probe(false).await
|
||||
}
|
||||
|
||||
async fn wait_for_server_ready_with_probe(
|
||||
&mut self,
|
||||
read_quorum_only: bool,
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
info!("Waiting for RustFS server to be ready on {}", self.address);
|
||||
let client = self.create_s3_client();
|
||||
let health_client = HttpClient::builder().no_proxy().timeout(Duration::from_secs(2)).build()?;
|
||||
|
||||
for i in 0..60 {
|
||||
if let Some(process) = self.process.as_mut()
|
||||
@@ -838,7 +859,16 @@ impl RustFSTestEnvironment {
|
||||
return Err(format!("RustFS server process exited before becoming ready: {status}").into());
|
||||
}
|
||||
|
||||
if TcpStream::connect(&self.address).await.is_ok() && client.list_buckets().send().await.is_ok() {
|
||||
let ready = if read_quorum_only {
|
||||
health_client
|
||||
.get(format!("{}/minio/health/ready", self.url))
|
||||
.send()
|
||||
.await
|
||||
.is_ok_and(|response| response.status().is_success())
|
||||
} else {
|
||||
TcpStream::connect(&self.address).await.is_ok() && client.list_buckets().send().await.is_ok()
|
||||
};
|
||||
if ready {
|
||||
info!("✅ RustFS server is ready after {} attempts", i + 1);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
@@ -13,8 +13,8 @@
|
||||
// limitations under the License.
|
||||
|
||||
//! Over-the-wire smoke net for the embedded console listener (backlog#1154
|
||||
//! peri-4). The console is a second HTTP listener that serves the full S3/admin
|
||||
//! surface plus the unauthenticated console routes under `/rustfs/console`;
|
||||
//! peri-4). The console is a second HTTP listener for the control-plane and
|
||||
//! unauthenticated console routes under `/rustfs/console`;
|
||||
//! before this test its behaviour was only covered by in-process router tests
|
||||
//! (`rustfs/src/admin/console.rs`), so a misconfiguration that exposed
|
||||
//! credentials through the public console endpoints or fail-opened the
|
||||
@@ -26,13 +26,13 @@
|
||||
//! * `/rustfs/console/license` exposes only the coarse `licensed` flag.
|
||||
//! * the console SPA prefix dispatches to the static handler, never to the
|
||||
//! S3 API (no S3 error XML), whether or not console assets are embedded.
|
||||
//! * unauthenticated requests to the admin API and the S3 root on the console
|
||||
//! listener are denied (403 AccessDenied) — the extra listener does not
|
||||
//! fail-open the protected surface.
|
||||
//! * unauthenticated admin API requests are denied with 403 AccessDenied.
|
||||
//! * S3 root/object requests on the console listener return an empty 404,
|
||||
//! including authenticated requests, before S3 dispatch.
|
||||
//! * a listener with the console disabled (the main S3 port here) does not
|
||||
//! serve the unauthenticated console endpoints at all.
|
||||
|
||||
use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
|
||||
use crate::common::{RustFSTestEnvironment, init_logging, local_http_client, signed_request_with_client};
|
||||
use std::error::Error;
|
||||
use tokio::time::{Duration, sleep};
|
||||
|
||||
@@ -166,13 +166,39 @@ async fn test_console_over_the_wire_smoke() -> TestResult {
|
||||
let admin_body = admin_response.text().await?;
|
||||
assert!(admin_body.contains("AccessDenied"), "admin denial must be AccessDenied: {admin_body}");
|
||||
|
||||
let s3_root_response = client.get(format!("{console_base}/")).send().await?;
|
||||
let signed_data_root = signed_request_with_client(
|
||||
&client,
|
||||
http::Method::GET,
|
||||
&format!("{}/", env.url),
|
||||
&env.access_key,
|
||||
&env.secret_key,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
assert_eq!(
|
||||
s3_root_response.status(),
|
||||
reqwest::StatusCode::FORBIDDEN,
|
||||
"unauthenticated S3 root on the console listener must be denied"
|
||||
signed_data_root.status(),
|
||||
reqwest::StatusCode::OK,
|
||||
"the same credentials must reach S3 on the data listener"
|
||||
);
|
||||
|
||||
for path in ["/", "/console-object-bucket/key"] {
|
||||
let url = format!("{console_base}{path}");
|
||||
for signed in [false, true] {
|
||||
let response = if signed {
|
||||
signed_request_with_client(&client, http::Method::GET, &url, &env.access_key, &env.secret_key, None, None).await?
|
||||
} else {
|
||||
client.get(&url).send().await?
|
||||
};
|
||||
assert_eq!(
|
||||
response.status(),
|
||||
reqwest::StatusCode::NOT_FOUND,
|
||||
"S3 route {path}, signed={signed} must be isolated from the console listener"
|
||||
);
|
||||
assert!(response.bytes().await?.is_empty(), "isolated S3 routes must not reach a handler");
|
||||
}
|
||||
}
|
||||
|
||||
// --- console disabled on a listener means no console surface at all ------
|
||||
// The main S3 listener of this same process runs with the console disabled,
|
||||
// so its console paths must not answer with the unauthenticated console
|
||||
|
||||
@@ -26,7 +26,7 @@ use aws_sdk_s3::types::{
|
||||
DefaultRetention, ObjectLockConfiguration, ObjectLockEnabled, ObjectLockLegalHoldStatus, ObjectLockRetentionMode,
|
||||
ObjectLockRule,
|
||||
};
|
||||
use chrono::{Duration as ChronoDuration, Utc};
|
||||
use jiff::{SignedDuration, Timestamp};
|
||||
|
||||
fn delete_denied(error: &SdkError<DeleteObjectError>, context: &str) -> TestResult {
|
||||
let code = error.as_service_error().and_then(ProvideErrorMetadata::code);
|
||||
@@ -66,7 +66,7 @@ async fn four_node_four_drive_object_lock_worm_blocks_delete() -> TestResult {
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let retain_until = Utc::now() + ChronoDuration::days(1);
|
||||
let retain_until = Timestamp::now() + SignedDuration::from_hours(24);
|
||||
|
||||
let compliance_key = "compliance.bin";
|
||||
let compliance_version = put_object_with_retention(
|
||||
@@ -176,7 +176,10 @@ async fn four_node_default_retention_is_visible_and_non_lock_bucket_rejects_conf
|
||||
let retain_until = retention
|
||||
.retain_until_date()
|
||||
.ok_or("default retention omitted retain-until date")?;
|
||||
assert!(retain_until.secs() > Utc::now().timestamp(), "default retention is not in the future");
|
||||
assert!(
|
||||
retain_until.secs() > Timestamp::now().as_second(),
|
||||
"default retention is not in the future"
|
||||
);
|
||||
|
||||
let versioning = reader.get_bucket_versioning().bucket(&bucket).send().await?;
|
||||
assert_eq!(versioning.status().map(|status| status.as_str()), Some("Enabled"));
|
||||
|
||||
@@ -38,8 +38,8 @@ use aws_sdk_s3::types::{
|
||||
BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, ObjectLockLegalHoldStatus, ObjectLockRetentionMode,
|
||||
VersioningConfiguration,
|
||||
};
|
||||
use chrono::{DateTime, Utc};
|
||||
use http::{Method, StatusCode};
|
||||
use jiff::Timestamp;
|
||||
use serde_json::Value;
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::path::{Path, PathBuf};
|
||||
@@ -526,8 +526,8 @@ async fn put_multipart(client: &Client, bucket: &str, key: &str) -> TestResult<V
|
||||
Ok(body)
|
||||
}
|
||||
|
||||
fn retention_deadline() -> TestResult<DateTime<Utc>> {
|
||||
DateTime::<Utc>::from_timestamp(Utc::now().timestamp() + 24 * 3600, 0).ok_or_else(|| "retention deadline out of range".into())
|
||||
fn retention_deadline() -> TestResult<Timestamp> {
|
||||
Ok(Timestamp::now() + jiff::SignedDuration::from_hours(24))
|
||||
}
|
||||
|
||||
/// Seed the deterministic inventory into the seed pool and record it.
|
||||
@@ -654,7 +654,7 @@ pub(crate) async fn seed_manifest(run: &TopologyRun) -> TestResult<TopologyManif
|
||||
let mut entry = ManifestEntry::body(&locked, &key, "compliance", Some(version_id.clone()), Some(version_id), &body);
|
||||
entry.retention = Some(RetentionRecord {
|
||||
mode: "COMPLIANCE".to_string(),
|
||||
retain_until_unix: retain_until.timestamp(),
|
||||
retain_until_unix: retain_until.as_second(),
|
||||
});
|
||||
entries.push(entry);
|
||||
}
|
||||
|
||||
@@ -29,6 +29,7 @@ use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::ServerSideEncryption;
|
||||
use base64_simd::STANDARD as BASE64;
|
||||
use http::header::{CONTENT_TYPE, HOST};
|
||||
use jiff::Timestamp;
|
||||
use md5::{Digest as Md5Digest, Md5};
|
||||
use rustfs_s3_types::Body;
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
@@ -386,7 +387,7 @@ pub async fn create_key_with_specific_id(key_dir: &str, key_id: &str) -> Result<
|
||||
"usage": "EncryptDecrypt",
|
||||
"status": "Active",
|
||||
"metadata": HashMap::<String, String>::new(),
|
||||
"created_at": format!("{}[UTC]", chrono::Utc::now().to_rfc3339()),
|
||||
"created_at": format!("{}[UTC]", Timestamp::now().strftime("%Y-%m-%dT%H:%M:%S%.fZ")),
|
||||
"rotated_at": serde_json::Value::Null,
|
||||
"created_by": "e2e-test",
|
||||
"encrypted_key_material": BASE64.encode_to_string(key_data),
|
||||
|
||||
@@ -23,13 +23,13 @@ use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{
|
||||
ServerSideEncryption, ServerSideEncryptionByDefault, ServerSideEncryptionConfiguration, ServerSideEncryptionRule,
|
||||
};
|
||||
use chrono::{Duration as ChronoDuration, Utc};
|
||||
use flate2::{
|
||||
Compression,
|
||||
write::{GzEncoder, ZlibEncoder},
|
||||
};
|
||||
use http::HeaderValue;
|
||||
use http::header::{CONTENT_TYPE, HOST};
|
||||
use jiff::{SignedDuration, Timestamp};
|
||||
use md5::{Digest as Md5Digest, Md5};
|
||||
use rustfs_s3_types::Body;
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
@@ -42,8 +42,8 @@ use tokio::io::AsyncWriteExt;
|
||||
use uuid::Uuid;
|
||||
|
||||
fn encode_post_policy(conditions: Vec<serde_json::Value>) -> String {
|
||||
let expiration = (Utc::now() + ChronoDuration::hours(1))
|
||||
.format("%Y-%m-%dT%H:%M:%S.000Z")
|
||||
let expiration = (Timestamp::now() + SignedDuration::from_hours(1))
|
||||
.strftime("%Y-%m-%dT%H:%M:%S.000Z")
|
||||
.to_string();
|
||||
let policy = serde_json::json!({
|
||||
"expiration": expiration,
|
||||
|
||||
@@ -283,6 +283,7 @@ async fn test_cold_degraded_object_mutations_require_only_storage_quorum() -> Te
|
||||
#[tokio::test]
|
||||
async fn test_degraded_cluster_read_quorum_follows_erasure_layout() -> TestResult {
|
||||
crate::common::init_logging();
|
||||
assert_single_node_cold_start_with_missing_volume().await?;
|
||||
|
||||
for (node_count, parity) in [(3, 1), (4, 2), (6, 3), (6, 2), (4, 0)] {
|
||||
let read_quorum = node_count - parity;
|
||||
@@ -430,6 +431,138 @@ async fn test_degraded_cluster_read_quorum_follows_erasure_layout() -> TestResul
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn assert_single_node_cold_start_with_missing_volume() -> TestResult {
|
||||
for drive_count in [2, 3, 4] {
|
||||
let mut server = crate::chaos::DiskFaultHarness::new(drive_count).await?;
|
||||
server.set_env("RUSTFS_STORAGE_CLASS_STANDARD", format!("EC:{}", drive_count / 2));
|
||||
server.set_env("RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE", "false");
|
||||
server.set_env("RUSTFS_OBS_METRICS_EXPORT_ENABLED", "false");
|
||||
server.set_env("RUSTFS_HEALTH_ENDPOINT_ENABLE", "true");
|
||||
server.set_env("RUST_LOG", "warn");
|
||||
server.start_server_at_read_quorum().await?;
|
||||
let client = server.env.create_s3_client();
|
||||
let client = Client::from_conf(
|
||||
client
|
||||
.config()
|
||||
.to_builder()
|
||||
.retry_config(aws_sdk_s3::config::retry::RetryConfig::standard().with_max_attempts(1))
|
||||
.build(),
|
||||
);
|
||||
let bucket = format!("missing-volume-{drive_count}");
|
||||
client.create_bucket().bucket(&bucket).send().await?;
|
||||
let small = vec![17; 4096];
|
||||
let large = vec![73; 8 * 1024 * 1024];
|
||||
for (key, body) in [("small", &small), ("large", &large)] {
|
||||
client
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.body(Bytes::copy_from_slice(body).into())
|
||||
.send()
|
||||
.await?;
|
||||
for disk in 0..drive_count {
|
||||
let census = crate::chaos::census_object_version_on_disk(server.disk_path(disk), &bucket, key, None)?;
|
||||
assert!(census.is_complete(), "seed shard must exist before restart: {census:?}");
|
||||
}
|
||||
}
|
||||
server.kill_server();
|
||||
let missing = drive_count - 1;
|
||||
server.take_disk_offline(missing)?;
|
||||
server.start_server_at_read_quorum().await?;
|
||||
|
||||
for (key, body) in [("small", &small), ("large", &large)] {
|
||||
assert_quorum_object_body(&client, &bucket, key, body).await?;
|
||||
let head = client.head_object().bucket(&bucket).key(key).send().await?;
|
||||
assert_eq!(head.content_length(), Some(i64::try_from(body.len())?));
|
||||
}
|
||||
let listing = client.list_objects_v2().bucket(&bucket).send().await?;
|
||||
let keys: Vec<_> = listing.contents().iter().filter_map(|object| object.key()).collect();
|
||||
assert_eq!(keys, ["large", "small"]);
|
||||
|
||||
let put = client
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key("degraded-write")
|
||||
.body(Bytes::from_static(b"degraded write").into())
|
||||
.send()
|
||||
.await;
|
||||
if drive_count == 2 {
|
||||
let error = put.expect_err("one of two drives cannot commit a new object");
|
||||
assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(503));
|
||||
let error = client
|
||||
.delete_object()
|
||||
.bucket(&bucket)
|
||||
.key("large")
|
||||
.send()
|
||||
.await
|
||||
.expect_err("one of two drives cannot commit an object deletion");
|
||||
assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(503));
|
||||
assert_quorum_object_body(&client, &bucket, "large", &large).await?;
|
||||
} else {
|
||||
put?;
|
||||
assert_quorum_object_body(&client, &bucket, "degraded-write", b"degraded write").await?;
|
||||
client.delete_object().bucket(&bucket).key("degraded-write").send().await?;
|
||||
}
|
||||
assert!(
|
||||
!server.disk_path(missing).exists(),
|
||||
"degraded startup and S3 operations must leave the root absent"
|
||||
);
|
||||
|
||||
if drive_count == 2 {
|
||||
server.kill_server();
|
||||
std::fs::create_dir(server.disk_path(missing))?;
|
||||
server.start_server_at_read_quorum().await?;
|
||||
for (key, body) in [("small", &small), ("large", &large)] {
|
||||
assert_quorum_object_body(&client, &bucket, key, body).await?;
|
||||
client.head_object().bucket(&bucket).key(key).send().await?;
|
||||
}
|
||||
assert_eq!(client.list_objects_v2().bucket(&bucket).send().await?.contents().len(), 2);
|
||||
let error = client
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key("empty-root-write")
|
||||
.body(Bytes::from_static(b"must remain unavailable").into())
|
||||
.send()
|
||||
.await
|
||||
.expect_err("an empty replacement cannot restore write quorum");
|
||||
assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(503));
|
||||
assert!(
|
||||
!server
|
||||
.disk_path(missing)
|
||||
.join(crate::storage_api::RUSTFS_META_BUCKET)
|
||||
.join("format.json")
|
||||
.exists(),
|
||||
"read admission must not format the empty replacement"
|
||||
);
|
||||
}
|
||||
server.kill_server();
|
||||
server.bring_disk_online(missing)?;
|
||||
server.start_server_at_read_quorum().await?;
|
||||
for (key, body) in [("small", &small), ("large", &large)] {
|
||||
assert_quorum_object_body(&client, &bucket, key, body).await?;
|
||||
}
|
||||
for key in ["degraded-write", "empty-root-write"] {
|
||||
let error = client
|
||||
.head_object()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("rejected or deleted degraded write must stay absent after recovery");
|
||||
assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(404));
|
||||
}
|
||||
client
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key("recovery-write")
|
||||
.body(Bytes::from_static(b"recovered write").into())
|
||||
.send()
|
||||
.await?;
|
||||
assert_quorum_object_body(&client, &bucket, "recovery-write", b"recovered write").await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_cold_start_read_quorum_follows_erasure_layout() -> TestResult {
|
||||
crate::common::init_logging();
|
||||
|
||||
@@ -29,7 +29,7 @@ use aws_sdk_s3::types::{
|
||||
DefaultRetention, ObjectLockConfiguration, ObjectLockEnabled, ObjectLockLegalHold, ObjectLockLegalHoldStatus, ObjectLockMode,
|
||||
ObjectLockRetention, ObjectLockRetentionMode, ObjectLockRule,
|
||||
};
|
||||
use chrono::{DateTime, Duration, Utc};
|
||||
use jiff::{SignedDuration, Timestamp};
|
||||
use tracing::info;
|
||||
|
||||
use crate::common::RustFSTestEnvironment;
|
||||
@@ -126,10 +126,10 @@ pub async fn put_object_with_retention(
|
||||
key: &str,
|
||||
data: &[u8],
|
||||
mode: ObjectLockRetentionMode,
|
||||
retain_until: DateTime<Utc>,
|
||||
retain_until: Timestamp,
|
||||
) -> Result<String, Box<dyn std::error::Error + Send + Sync>> {
|
||||
// AWS SDK requires UTC time without timezone offset (e.g., "2026-01-24T11:20:14Z")
|
||||
let retain_until_str = retain_until.format("%Y-%m-%dT%H:%M:%SZ").to_string();
|
||||
let retain_until_str = retain_until.strftime("%Y-%m-%dT%H:%M:%SZ").to_string();
|
||||
let retain_until_datetime =
|
||||
aws_sdk_s3::primitives::DateTime::from_str(&retain_until_str, aws_sdk_s3::primitives::DateTimeFormat::DateTime)?;
|
||||
|
||||
@@ -181,10 +181,10 @@ pub async fn put_object_retention(
|
||||
key: &str,
|
||||
version_id: Option<&str>,
|
||||
mode: ObjectLockRetentionMode,
|
||||
retain_until: DateTime<Utc>,
|
||||
retain_until: Timestamp,
|
||||
bypass_governance: bool,
|
||||
) -> Result<(), Box<SdkError<PutObjectRetentionError>>> {
|
||||
let retain_until_datetime = aws_sdk_s3::primitives::DateTime::from_secs(retain_until.timestamp());
|
||||
let retain_until_datetime = aws_sdk_s3::primitives::DateTime::from_secs(retain_until.as_second());
|
||||
|
||||
let retention = ObjectLockRetention::builder()
|
||||
.mode(mode.clone())
|
||||
@@ -253,6 +253,6 @@ pub async fn delete_object_with_bypass(
|
||||
}
|
||||
|
||||
/// Calculate a future retain_until date
|
||||
pub fn future_retain_until(days: i64) -> DateTime<Utc> {
|
||||
Utc::now() + Duration::days(days)
|
||||
pub fn future_retain_until(days: i64) -> Timestamp {
|
||||
Timestamp::now() + SignedDuration::from_hours(days * 24)
|
||||
}
|
||||
|
||||
@@ -34,7 +34,7 @@ use aws_sdk_s3::types::{
|
||||
CompletedMultipartUpload, CompletedPart, Delete, MetadataDirective, ObjectIdentifier, ObjectLockLegalHoldStatus,
|
||||
ObjectLockMode, ObjectLockRetentionMode,
|
||||
};
|
||||
use chrono::{DateTime, Duration, Utc};
|
||||
use jiff::{SignedDuration, Timestamp};
|
||||
use tracing::info;
|
||||
|
||||
/// Initialize test logging
|
||||
@@ -67,7 +67,7 @@ async fn put_bucket_deny_policy(
|
||||
}
|
||||
|
||||
fn retention_timestamp(days: i64) -> aws_sdk_s3::primitives::DateTime {
|
||||
let retain_until = future_retain_until(days).format("%Y-%m-%dT%H:%M:%SZ").to_string();
|
||||
let retain_until = future_retain_until(days).strftime("%Y-%m-%dT%H:%M:%SZ").to_string();
|
||||
aws_sdk_s3::primitives::DateTime::from_str(&retain_until, aws_sdk_s3::primitives::DateTimeFormat::DateTime)
|
||||
.expect("retention timestamp should parse")
|
||||
}
|
||||
@@ -118,13 +118,14 @@ where
|
||||
);
|
||||
}
|
||||
|
||||
fn parse_s3_datetime(value: &aws_sdk_s3::primitives::DateTime) -> DateTime<Utc> {
|
||||
fn parse_s3_datetime(value: &aws_sdk_s3::primitives::DateTime) -> Timestamp {
|
||||
let formatted = value
|
||||
.fmt(DateTimeFormat::DateTime)
|
||||
.expect("S3 timestamp should format as RFC3339");
|
||||
DateTime::parse_from_rfc3339(&formatted)
|
||||
.expect("S3 timestamp should parse as RFC3339")
|
||||
.with_timezone(&Utc)
|
||||
let normalized = formatted
|
||||
.strip_suffix('Z')
|
||||
.map_or(formatted.clone(), |prefix| format!("{prefix}+00:00"));
|
||||
Timestamp::strptime("%Y-%m-%dT%H:%M:%S%.f%:z", normalized).expect("S3 timestamp should parse as RFC3339")
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
@@ -470,7 +471,7 @@ async fn test_get_object_retention_returns_configured_values() {
|
||||
let bucket = "test-get-retention";
|
||||
let key = "retained-object";
|
||||
let retain_until = future_retain_until(30);
|
||||
let retain_until_expected = retain_until.format("%Y-%m-%dT%H:%M:%SZ").to_string();
|
||||
let retain_until_expected = retain_until.strftime("%Y-%m-%dT%H:%M:%SZ").to_string();
|
||||
|
||||
env.create_object_lock_bucket(bucket).await.unwrap();
|
||||
|
||||
@@ -1924,7 +1925,7 @@ async fn test_copy_object_retention_uses_destination_policy() {
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let copy_started = Utc::now();
|
||||
let copy_started = Timestamp::now();
|
||||
client
|
||||
.copy_object()
|
||||
.copy_source(format!("{src_bucket}/{src_key}"))
|
||||
@@ -1947,7 +1948,7 @@ async fn test_copy_object_retention_uses_destination_policy() {
|
||||
assert_eq!(retention.mode().map(|value| value.as_str()), Some("GOVERNANCE"));
|
||||
let retain_until = parse_s3_datetime(retention.retain_until_date().expect("retain-until date should be present"));
|
||||
assert!(
|
||||
retain_until < copy_started + Duration::days(3),
|
||||
retain_until < copy_started + SignedDuration::from_hours(3 * 24),
|
||||
"destination default retention should not inherit the source's longer retention"
|
||||
);
|
||||
|
||||
@@ -2008,7 +2009,7 @@ async fn test_copy_object_retention_uses_destination_policy() {
|
||||
.expect("explicit retain-until date should be present"),
|
||||
);
|
||||
assert!(
|
||||
explicit_retain_until > Utc::now() + Duration::days(20),
|
||||
explicit_retain_until > Timestamp::now() + SignedDuration::from_hours(20 * 24),
|
||||
"explicit retention should override the shorter bucket default"
|
||||
);
|
||||
|
||||
@@ -2088,7 +2089,7 @@ async fn test_multipart_default_retention_fixed_at_create() {
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let create_started = Utc::now();
|
||||
let create_started = Timestamp::now();
|
||||
let create_output = client.create_multipart_upload().bucket(bucket).key(key).send().await.unwrap();
|
||||
let upload_id = create_output.upload_id().unwrap();
|
||||
|
||||
@@ -2131,7 +2132,7 @@ async fn test_multipart_default_retention_fixed_at_create() {
|
||||
assert_eq!(retention.mode().map(|value| value.as_str()), Some("GOVERNANCE"));
|
||||
let retain_until = parse_s3_datetime(retention.retain_until_date().expect("retain-until date should be present"));
|
||||
assert!(
|
||||
retain_until < create_started + Duration::days(3),
|
||||
retain_until < create_started + SignedDuration::from_hours(3 * 24),
|
||||
"CompleteMultipartUpload should keep the default retention calculated at create time"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -573,6 +573,15 @@ impl NodeService for MinimalLockNodeService {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
type RenameDataBatchStream = ResponseStream<rustfs_protos::proto_gen::node_service::RenameDataBatchResult>;
|
||||
|
||||
async fn rename_data_batch(
|
||||
&self,
|
||||
_request: Request<rustfs_protos::proto_gen::node_service::RenameDataBatchRequest>,
|
||||
) -> Result<Response<Self::RenameDataBatchStream>, Status> {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
async fn rename_multipart_data(
|
||||
&self,
|
||||
_request: Request<rustfs_protos::proto_gen::node_service::RenameDataRequest>,
|
||||
|
||||
@@ -154,7 +154,6 @@ arc-swap.workspace = true
|
||||
async-trait.workspace = true
|
||||
bytes = { workspace = true, features = ["serde"] }
|
||||
byteorder = { workspace = true }
|
||||
chrono = { workspace = true, features = ["serde"] }
|
||||
jiff = { workspace = true, features = ["serde"] }
|
||||
thiserror.workspace = true
|
||||
flatbuffers.workspace = true
|
||||
|
||||
@@ -90,7 +90,9 @@ fn remote_credentials(credentials: &Credentials, account_id: &str) -> RemoteCred
|
||||
access_key: credentials.access_key.clone(),
|
||||
secret_key: credentials.secret_key.clone(),
|
||||
session_token: credentials.effective_session_token().map(str::to_string),
|
||||
expiration: credentials.effective_expiration().map(SystemTime::from),
|
||||
expiration: credentials
|
||||
.effective_expiration()
|
||||
.and_then(|timestamp| SystemTime::try_from(timestamp).ok()),
|
||||
account_id: account_id.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5591,7 +5591,7 @@ impl RestoreRequestOps for RestoreRequest {
|
||||
}
|
||||
|
||||
// For non-SELECT requests, days must be at least 1
|
||||
if self.type_.is_none() && self.days.is_none_or(|d| d <= 0) {
|
||||
if !valid_restore_days(self) {
|
||||
return Err(std::io::Error::other("restoration days should be at least 1"));
|
||||
}
|
||||
|
||||
@@ -5599,6 +5599,10 @@ impl RestoreRequestOps for RestoreRequest {
|
||||
}
|
||||
}
|
||||
|
||||
fn valid_restore_days(request: &RestoreRequest) -> bool {
|
||||
request.type_.as_deref() == Some("SELECT") || request.days.is_some_and(|days| days > 0)
|
||||
}
|
||||
|
||||
const _MAX_RESTORE_OBJECT_REQUEST_SIZE: i64 = 2 << 20;
|
||||
|
||||
/// Builds an exact-version lifecycle delete target with the concrete
|
||||
@@ -9373,6 +9377,18 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn standard_restore_days_are_positive_with_or_without_type() {
|
||||
let mut request = select_restore_request(None);
|
||||
for kind in [None, Some("STANDARD".to_owned())] {
|
||||
request.type_ = kind;
|
||||
for days in [None, Some(0), Some(-1), Some(1)] {
|
||||
request.days = days;
|
||||
assert_eq!(super::valid_restore_days(&request), days == Some(1));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn select_restore_s3_location_rejects_missing_s3_output_location() {
|
||||
let request = select_restore_request(Some(OutputLocation { s3: None }));
|
||||
|
||||
@@ -1607,6 +1607,7 @@ fn shard_object(ledger_object: &str, shard_index: u16, bucket_incarnation: Uuid,
|
||||
}
|
||||
|
||||
fn shard_index(object: &str) -> u16 {
|
||||
let object = rustfs_utils::path::encode_dir_object_ref(object);
|
||||
let digest = Sha256::digest(object.as_bytes());
|
||||
u16::from_be_bytes([digest[0], digest[1]]) % SHARDED_LEDGER_COUNT
|
||||
}
|
||||
@@ -2314,6 +2315,12 @@ fn log_deferred_settlement(data: &ReservationLedgerData, state: &'static str, er
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
#[test]
|
||||
fn quota_shard_agrees_with_directory_listing_name() {
|
||||
for listed in ["photos/", "reports/june/"] {
|
||||
assert_eq!(shard_index(listed), shard_index(&rustfs_utils::path::encode_dir_object(listed)));
|
||||
}
|
||||
}
|
||||
use crate::storage_api_contracts::bucket::{BucketOperations as _, MakeBucketOptions};
|
||||
use serial_test::serial;
|
||||
|
||||
|
||||
@@ -256,6 +256,64 @@ pub(crate) fn remote_sdk_credentials(credentials: &RemoteCredentials, now: Syste
|
||||
Ok(builder.build())
|
||||
}
|
||||
|
||||
/// Bound the bytes Smithy can aggregate for a control document or service error.
|
||||
/// Successful GetObject responses are streamed directly to the caller.
|
||||
#[derive(Debug)]
|
||||
struct ControlResponseLimit(usize);
|
||||
|
||||
#[derive(Debug)]
|
||||
struct OversizedResponseRetryClassifier;
|
||||
|
||||
impl aws_smithy_runtime_api::client::retries::classifiers::ClassifyRetry for OversizedResponseRetryClassifier {
|
||||
fn classify_retry(
|
||||
&self,
|
||||
ctx: &aws_smithy_runtime_api::client::interceptors::context::InterceptorContext,
|
||||
) -> aws_smithy_runtime_api::client::retries::classifiers::RetryAction {
|
||||
use aws_smithy_runtime_api::client::retries::classifiers::RetryAction;
|
||||
if let Some(Err(error)) = ctx.output_or_error() {
|
||||
let mut cause = std::error::Error::source(error);
|
||||
while let Some(error) = cause {
|
||||
if error.is::<http_body_util::LengthLimitError>() {
|
||||
return RetryAction::RetryForbidden;
|
||||
}
|
||||
cause = error.source();
|
||||
}
|
||||
}
|
||||
RetryAction::NoActionIndicated
|
||||
}
|
||||
|
||||
fn name(&self) -> &'static str {
|
||||
"RustfsOversizedResponseRetryClassifier"
|
||||
}
|
||||
}
|
||||
|
||||
impl Intercept for ControlResponseLimit {
|
||||
fn name(&self) -> &'static str {
|
||||
"RustfsControlResponseLimit"
|
||||
}
|
||||
|
||||
fn modify_before_deserialization(
|
||||
&self,
|
||||
context: &mut BeforeDeserializationInterceptorContextMut<'_>,
|
||||
_runtime_components: &RuntimeComponents,
|
||||
cfg: &mut ConfigBag,
|
||||
) -> Result<(), BoxError> {
|
||||
let operation = cfg
|
||||
.load::<aws_smithy_runtime_api::client::orchestrator::Metadata>()
|
||||
.map(|m| m.name());
|
||||
bound_control_response(context.response_mut(), operation, self.0);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn bound_control_response(response: &mut HttpResponse, operation: Option<&str>, limit: usize) {
|
||||
if operation == Some("GetObject") && response.status().is_success() {
|
||||
return;
|
||||
}
|
||||
let body = std::mem::replace(response.body_mut(), SdkBody::taken());
|
||||
*response.body_mut() = SdkBody::from_body_1_x(http_body_util::Limited::new(body, limit));
|
||||
}
|
||||
|
||||
/// Only installed on non-streaming upload operations. The budget begins at
|
||||
/// response headers, independently of the potentially long request upload.
|
||||
#[derive(Debug)]
|
||||
@@ -375,6 +433,8 @@ pub async fn build_remote_s3_config(spec: &RemoteS3EndpointSpec) -> Result<aws_s
|
||||
.region(SdkRegion::new(spec.region.clone()))
|
||||
.behavior_version(aws_sdk_s3::config::BehaviorVersion::latest())
|
||||
.request_checksum_calculation(replication_request_checksum_calculation())
|
||||
.interceptor(ControlResponseLimit(16 * 1024 * 1024))
|
||||
.retry_classifier(OversizedResponseRetryClassifier)
|
||||
.retry_config(spec.retry.retry_config());
|
||||
|
||||
if spec.path_style.force_path_style() {
|
||||
@@ -785,6 +845,33 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn remote_control_response_limit_preserves_object_streams() {
|
||||
use http_body_util::BodyExt;
|
||||
for (operation, status, bytes, allowed) in [
|
||||
(Some("ListObjectsV2"), 200, 4, true),
|
||||
(Some("ListObjectsV2"), 200, 5, false),
|
||||
(Some("GetObject"), 206, 5, true),
|
||||
(Some("GetObject"), 404, 5, false),
|
||||
(None, 200, 5, false),
|
||||
] {
|
||||
let mut response = HttpResponse::try_from(
|
||||
http::Response::builder()
|
||||
.status(status)
|
||||
.body(SdkBody::from(vec![1; bytes]))
|
||||
.expect("fixture response"),
|
||||
)
|
||||
.expect("Smithy response");
|
||||
bound_control_response(&mut response, operation, 4);
|
||||
let result = response.into_body().collect().await;
|
||||
if allowed {
|
||||
assert_eq!(result.expect("body is permitted").to_bytes().len(), bytes);
|
||||
} else {
|
||||
assert!(result.is_err(), "oversized control response must fail");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn build_remote_s3_client_rejects_loopback_and_metadata_endpoints() {
|
||||
// Default (no loopback opt-in): loopback in IPv4, IPv6 and hostname
|
||||
@@ -886,6 +973,59 @@ mod tests {
|
||||
wire_requests.load(Ordering::SeqCst)
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn sdk_remote_control_limit_stops_chunked_errors_and_preserves_downloads() {
|
||||
#[derive(Debug)]
|
||||
struct ChunkConnector(u16, Arc<AtomicUsize>);
|
||||
impl SmithyHttpConnector for ChunkConnector {
|
||||
fn call(&self, _: HttpRequest) -> HttpConnectorFuture {
|
||||
self.1.fetch_add(1, Ordering::SeqCst);
|
||||
let frames =
|
||||
[b"abc", b"def"].map(|s| Ok::<_, std::io::Error>(http_body::Frame::data(bytes::Bytes::from_static(s))));
|
||||
let body = http_body_util::StreamBody::new(futures::stream::iter(frames));
|
||||
HttpConnectorFuture::ready(Ok(HttpResponse::new(
|
||||
SmithyStatusCode::try_from(self.0).expect("fixture status"),
|
||||
SdkBody::from_body_1_x(body),
|
||||
)))
|
||||
}
|
||||
}
|
||||
for status in [200, 503] {
|
||||
let calls = Arc::new(AtomicUsize::new(0));
|
||||
let connector = SharedHttpConnector::new(ChunkConnector(status, calls.clone()));
|
||||
let mut target = spec("s3.example.com", true);
|
||||
target.retry = REPLICATION_TARGET_RETRY_POLICY;
|
||||
let config = build_remote_s3_config(&target)
|
||||
.await
|
||||
.expect("build remote config")
|
||||
.http_client(http_client_fn(move |_, _| connector.clone()))
|
||||
.interceptor(ControlResponseLimit(4))
|
||||
.build();
|
||||
let result = S3Client::from_conf(config)
|
||||
.get_object()
|
||||
.bucket("bucket")
|
||||
.key("key")
|
||||
.send()
|
||||
.await;
|
||||
if status == 200 {
|
||||
let bytes = result
|
||||
.expect("successful object stream")
|
||||
.body
|
||||
.collect()
|
||||
.await
|
||||
.expect("collect fixture object")
|
||||
.into_bytes();
|
||||
assert_eq!(&bytes[..], b"abcdef");
|
||||
} else {
|
||||
assert!(result.is_err(), "oversized error response must fail");
|
||||
}
|
||||
assert_eq!(
|
||||
calls.load(Ordering::SeqCst),
|
||||
1,
|
||||
"oversized control responses must not trigger SDK retries"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn retry_policy_decides_how_many_wire_requests_one_call_costs() {
|
||||
assert_eq!(
|
||||
@@ -930,8 +1070,8 @@ mod tests {
|
||||
.unwrap_or_else(|| panic!("signed request must contain {name}"))
|
||||
}
|
||||
|
||||
fn signing_time(headers: &[(String, String)]) -> chrono::NaiveDateTime {
|
||||
chrono::NaiveDateTime::parse_from_str(recorded_header(headers, "x-amz-date"), "%Y%m%dT%H%M%SZ")
|
||||
fn signing_time(headers: &[(String, String)]) -> jiff::Timestamp {
|
||||
jiff::Timestamp::strptime("%Y%m%dT%H%M%SZ", recorded_header(headers, "x-amz-date"))
|
||||
.expect("SDK signing timestamp must use the SigV4 format")
|
||||
}
|
||||
|
||||
@@ -946,8 +1086,8 @@ mod tests {
|
||||
.map(|(key, value)| (key.to_string(), value.to_string()))
|
||||
.collect(),
|
||||
);
|
||||
let server_time = chrono::DateTime::<chrono::Utc>::from(self.clock.now()).naive_utc()
|
||||
+ chrono::Duration::seconds(self.skew_seconds);
|
||||
let server_time = jiff::Timestamp::try_from(self.clock.now()).expect("clock timestamp")
|
||||
+ jiff::SignedDuration::from_secs(self.skew_seconds);
|
||||
let (status, body) = if headers.len() == 1 {
|
||||
(
|
||||
403,
|
||||
@@ -958,7 +1098,7 @@ mod tests {
|
||||
};
|
||||
let response = http::Response::builder()
|
||||
.status(status)
|
||||
.header("date", server_time.format("%a, %d %b %Y %H:%M:%S GMT").to_string())
|
||||
.header("date", server_time.strftime("%a, %d %b %Y %H:%M:%S GMT").to_string())
|
||||
.header("content-type", "application/xml")
|
||||
.header("content-length", body.len())
|
||||
.body(SdkBody::from(body))
|
||||
@@ -996,7 +1136,7 @@ mod tests {
|
||||
for error_code in ["RequestTimeTooSkewed", "SignatureDoesNotMatch"] {
|
||||
for skew_seconds in [-600, 600] {
|
||||
let (client, headers, clock) = clock_skew_client(error_code, skew_seconds, REPLICATION_TARGET_RETRY_POLICY).await;
|
||||
let initial = chrono::DateTime::<chrono::Utc>::from(clock.now()).naive_utc();
|
||||
let initial = jiff::Timestamp::try_from(clock.now()).expect("clock timestamp");
|
||||
client
|
||||
.get_object()
|
||||
.bucket("bucket")
|
||||
@@ -1023,12 +1163,12 @@ mod tests {
|
||||
assert_eq!(signing_time(&headers[0]), initial, "the first attempt must use the injected clock");
|
||||
assert_eq!(
|
||||
signing_time(&headers[1]),
|
||||
initial + chrono::Duration::seconds(skew_seconds),
|
||||
initial + jiff::SignedDuration::from_secs(skew_seconds),
|
||||
"{error_code}: retry must apply the measured offset exactly"
|
||||
);
|
||||
assert_eq!(
|
||||
signing_time(&headers[2]),
|
||||
initial + chrono::Duration::seconds(skew_seconds + 17),
|
||||
initial + jiff::SignedDuration::from_secs(skew_seconds + 17),
|
||||
"{error_code}: the next operation must apply cached skew to the advanced signing clock"
|
||||
);
|
||||
let signature = |index: usize| {
|
||||
|
||||
@@ -73,10 +73,11 @@ use rustfs_protos::proto_gen::node_service::{
|
||||
PreparePartTransactionRequest, ReadAllRequest, ReadMetadataRequest, ReadMultipleRequest, ReadMultipleResponse,
|
||||
ReadPartsRequest, ReadVersionRequest, ReadXlRequest, RecoverDeleteTransactionRequest,
|
||||
RecoverMultipartSourceRequest as RecoverMultipartSourceRpcRequest,
|
||||
RecoverMultipartSourceResponse as RecoverMultipartSourceRpcResponse, RenameDataRequest, RenameFileRequest,
|
||||
SettlePartTransactionRequest, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest, SnapshotLeaseRequest,
|
||||
SnapshotLeaseResponse, StatVolumeRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest, WriteAllResponse,
|
||||
WriteFormatControlRequest, WriteMetadataRequest, node_service_client::NodeServiceClient,
|
||||
RecoverMultipartSourceResponse as RecoverMultipartSourceRpcResponse, RenameDataBatchItem, RenameDataBatchRequest,
|
||||
RenameDataBatchResult, RenameDataRequest, RenameFileRequest, SettlePartTransactionRequest, SnapshotLeaseReleaseRequest,
|
||||
SnapshotLeaseRenewRequest, SnapshotLeaseRequest, SnapshotLeaseResponse, StatVolumeRequest, UpdateMetadataRequest,
|
||||
VerifyFileRequest, WriteAllRequest, WriteAllResponse, WriteFormatControlRequest, WriteMetadataRequest,
|
||||
node_service_client::NodeServiceClient,
|
||||
};
|
||||
use rustfs_protos::{
|
||||
ChannelClass, ChannelRebuildReason, ConnectionEvictionLogLevel, evict_failed_connection_with_reason, prewarm_channel_class,
|
||||
@@ -104,7 +105,7 @@ use tokio::{
|
||||
time::timeout,
|
||||
};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tonic::{Code, Request, service::interceptor::InterceptedService};
|
||||
use tonic::{Code, Request, Streaming, service::interceptor::InterceptedService};
|
||||
use tracing::{debug, trace, warn};
|
||||
use uuid::Uuid;
|
||||
|
||||
@@ -1045,6 +1046,7 @@ pub struct RemoteDisk {
|
||||
/// Optional per-peer gate held through complete bulk-unary decoding and validation.
|
||||
bulk_body_admission: Option<Arc<Semaphore>>,
|
||||
bulk_body_wait_timeout: Duration,
|
||||
rename_admission: Option<Arc<Semaphore>>,
|
||||
}
|
||||
|
||||
/// Shortest gap between two failure warnings of one kind from one remote disk.
|
||||
@@ -1060,6 +1062,251 @@ struct BulkBodyAdmissionRegistry {
|
||||
static BULK_BODY_ADMISSION_REGISTRY: std::sync::LazyLock<StdMutex<BulkBodyAdmissionRegistry>> =
|
||||
std::sync::LazyLock::new(|| StdMutex::new(BulkBodyAdmissionRegistry::default()));
|
||||
|
||||
const ENV_RUSTFS_EXPERIMENTAL_RENAME_STREAM_BATCH: &str = "RUSTFS_EXPERIMENTAL_RENAME_STREAM_BATCH";
|
||||
const RENAME_ADMISSION_REGISTRY_CAPACITY: usize = 512;
|
||||
|
||||
#[derive(Default)]
|
||||
struct RenameAdmissionRegistry {
|
||||
peers: HashMap<String, Weak<Semaphore>>,
|
||||
overflow: Option<Arc<Semaphore>>,
|
||||
}
|
||||
|
||||
static RENAME_ADMISSION_REGISTRY: std::sync::LazyLock<StdMutex<RenameAdmissionRegistry>> =
|
||||
std::sync::LazyLock::new(|| StdMutex::new(RenameAdmissionRegistry::default()));
|
||||
|
||||
fn peer_rename_admission(addr: &str, enabled: bool) -> Option<Arc<Semaphore>> {
|
||||
if !enabled {
|
||||
return None;
|
||||
}
|
||||
let mut registry = RENAME_ADMISSION_REGISTRY
|
||||
.lock()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
if let Some(semaphore) = registry.peers.get(addr).and_then(Weak::upgrade) {
|
||||
return Some(semaphore);
|
||||
}
|
||||
if registry.peers.len() >= RENAME_ADMISSION_REGISTRY_CAPACITY {
|
||||
registry.peers.retain(|_, semaphore| semaphore.strong_count() > 0);
|
||||
}
|
||||
if registry.peers.len() >= RENAME_ADMISSION_REGISTRY_CAPACITY {
|
||||
return Some(Arc::clone(registry.overflow.get_or_insert_with(|| {
|
||||
Arc::new(Semaphore::new(rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT))
|
||||
})));
|
||||
}
|
||||
let semaphore = Arc::new(Semaphore::new(rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT));
|
||||
registry.peers.insert(addr.to_owned(), Arc::downgrade(&semaphore));
|
||||
Some(semaphore)
|
||||
}
|
||||
|
||||
struct RenameAdmissionGrant {
|
||||
permit: Option<OwnedSemaphorePermit>,
|
||||
batch: bool,
|
||||
}
|
||||
|
||||
impl RenameAdmissionGrant {
|
||||
fn release_one(&mut self) -> bool {
|
||||
self.permit.as_mut().is_some_and(|permit| permit.split(1).is_some())
|
||||
}
|
||||
|
||||
fn release_all(mut self) {
|
||||
drop(self.permit.take());
|
||||
}
|
||||
|
||||
fn quarantine_until(mut self, deadline: tokio::time::Instant) {
|
||||
let Some(permit) = self.permit.take() else {
|
||||
return;
|
||||
};
|
||||
let items = permit.num_permits();
|
||||
if items == 0 {
|
||||
return;
|
||||
}
|
||||
let hold = deadline.saturating_duration_since(tokio::time::Instant::now());
|
||||
rustfs_io_metrics::record_ecstore_rename_quarantine(self.batch, items, hold);
|
||||
let batch = self.batch;
|
||||
if let Ok(handle) = tokio::runtime::Handle::try_current() {
|
||||
handle.spawn(async move {
|
||||
tokio::time::sleep_until(deadline).await;
|
||||
drop(permit);
|
||||
rustfs_io_metrics::record_ecstore_rename_quarantine_released(batch, items);
|
||||
});
|
||||
} else {
|
||||
// Without a runtime there is no safe deadline task. Leaking until
|
||||
// process exit is safer than admitting work over a still-running mutation.
|
||||
rustfs_io_metrics::record_ecstore_rename_quarantine_no_runtime(batch, items);
|
||||
std::mem::forget(permit);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct RenameBatchCallGuard {
|
||||
admission: Option<RenameAdmissionGrant>,
|
||||
deadline: tokio::time::Instant,
|
||||
dispatch_state: RenameAdmissionDispatchState,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
enum RenameAdmissionDispatchState {
|
||||
NotDispatched,
|
||||
MayHavePublished,
|
||||
}
|
||||
|
||||
impl RenameBatchCallGuard {
|
||||
fn mark_dispatched(&mut self) {
|
||||
self.dispatch_state = RenameAdmissionDispatchState::MayHavePublished;
|
||||
}
|
||||
|
||||
fn complete(mut self) {
|
||||
debug_assert_eq!(self.dispatch_state, RenameAdmissionDispatchState::MayHavePublished);
|
||||
if let Some(admission) = self.admission.take() {
|
||||
admission.release_all();
|
||||
}
|
||||
}
|
||||
|
||||
fn release_without_quarantine(mut self) {
|
||||
if let Some(admission) = self.admission.take() {
|
||||
admission.release_all();
|
||||
}
|
||||
}
|
||||
|
||||
fn into_stream(mut self, inner: Streaming<RenameDataBatchResult>, expected_mask: u16) -> RemoteRenameBatchStream {
|
||||
debug_assert_eq!(self.dispatch_state, RenameAdmissionDispatchState::MayHavePublished);
|
||||
RemoteRenameBatchStream {
|
||||
inner,
|
||||
expected_mask,
|
||||
seen_mask: 0,
|
||||
admission: self.admission.take(),
|
||||
deadline: self.deadline,
|
||||
terminal: false,
|
||||
stream_started_at: rustfs_io_metrics::put_stage_metrics_enabled().then(Instant::now),
|
||||
first_item_at: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for RenameBatchCallGuard {
|
||||
fn drop(&mut self) {
|
||||
if self.dispatch_state == RenameAdmissionDispatchState::MayHavePublished
|
||||
&& let Some(admission) = self.admission.take()
|
||||
{
|
||||
admission.quarantine_until(self.deadline);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) struct RenameBatchClientItem<'a> {
|
||||
pub slot: usize,
|
||||
pub disk: String,
|
||||
pub file_info: &'a FileInfo,
|
||||
}
|
||||
|
||||
pub(crate) enum RenameBatchStartError {
|
||||
NotDispatched(Error),
|
||||
UnsupportedBeforeStream,
|
||||
MayHavePublished(Error),
|
||||
}
|
||||
|
||||
pub(crate) struct RemoteRenameBatchOutcome {
|
||||
pub slot: usize,
|
||||
pub result: Result<RenameDataResp>,
|
||||
}
|
||||
|
||||
pub(crate) struct RemoteRenameBatchStream {
|
||||
inner: Streaming<RenameDataBatchResult>,
|
||||
expected_mask: u16,
|
||||
seen_mask: u16,
|
||||
admission: Option<RenameAdmissionGrant>,
|
||||
deadline: tokio::time::Instant,
|
||||
terminal: bool,
|
||||
stream_started_at: Option<Instant>,
|
||||
first_item_at: Option<Instant>,
|
||||
}
|
||||
|
||||
impl RemoteRenameBatchStream {
|
||||
fn terminate_ambiguous(&mut self, error: Error) -> Error {
|
||||
if !self.terminal {
|
||||
let missing = (self.expected_mask & !self.seen_mask).count_ones() as usize;
|
||||
rustfs_io_metrics::record_ecstore_rename_batch_terminal_missing(missing);
|
||||
}
|
||||
self.terminal = true;
|
||||
if let Some(admission) = self.admission.take() {
|
||||
admission.quarantine_until(self.deadline);
|
||||
}
|
||||
error
|
||||
}
|
||||
|
||||
pub(crate) async fn next_outcome(&mut self) -> Result<Option<RemoteRenameBatchOutcome>> {
|
||||
if self.terminal {
|
||||
return Ok(None);
|
||||
}
|
||||
let response = match tokio::time::timeout_at(self.deadline, self.inner.message()).await {
|
||||
Err(_) => return Err(self.terminate_ambiguous(DiskError::Timeout)),
|
||||
Ok(Err(status)) => return Err(self.terminate_ambiguous(status.into())),
|
||||
Ok(Ok(None)) if self.seen_mask == self.expected_mask => {
|
||||
self.terminal = true;
|
||||
return Ok(None);
|
||||
}
|
||||
Ok(Ok(None)) => return Err(self.terminate_ambiguous(DiskError::Unexpected)),
|
||||
Ok(Ok(Some(response))) => response,
|
||||
};
|
||||
let Ok(slot) = usize::try_from(response.item_index) else {
|
||||
return Err(self.terminate_ambiguous(DiskError::Unexpected));
|
||||
};
|
||||
if slot >= 16 {
|
||||
return Err(self.terminate_ambiguous(DiskError::Unexpected));
|
||||
}
|
||||
let bit = 1_u16 << slot;
|
||||
if self.expected_mask & bit == 0 || self.seen_mask & bit != 0 {
|
||||
return Err(self.terminate_ambiguous(DiskError::Unexpected));
|
||||
}
|
||||
self.seen_mask |= bit;
|
||||
if self.admission.as_mut().is_some_and(|admission| !admission.release_one()) {
|
||||
return Err(self.terminate_ambiguous(DiskError::Unexpected));
|
||||
}
|
||||
if let Some(stream_started_at) = self.stream_started_at {
|
||||
let completed_at = Instant::now();
|
||||
match self.first_item_at {
|
||||
Some(first_item_at) if self.seen_mask == self.expected_mask => {
|
||||
rustfs_io_metrics::record_ecstore_rename_batch_client_first_to_last(
|
||||
completed_at.saturating_duration_since(first_item_at),
|
||||
);
|
||||
}
|
||||
Some(_) => {}
|
||||
None => {
|
||||
rustfs_io_metrics::record_ecstore_rename_batch_client_stream_to_first(
|
||||
completed_at.saturating_duration_since(stream_started_at),
|
||||
);
|
||||
self.first_item_at = Some(completed_at);
|
||||
if self.seen_mask == self.expected_mask {
|
||||
rustfs_io_metrics::record_ecstore_rename_batch_client_first_to_last(Duration::ZERO);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
rustfs_io_metrics::record_ecstore_rename_batch_item_completed();
|
||||
let result = if response.success {
|
||||
if response.error.is_some() {
|
||||
Err(DiskError::FileCorrupt)
|
||||
} else {
|
||||
decode_msgpack_strict(&response.rename_data_resp_msgpack)
|
||||
}
|
||||
} else {
|
||||
Err(response.error.unwrap_or_default().into())
|
||||
};
|
||||
Ok(Some(RemoteRenameBatchOutcome { slot, result }))
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for RemoteRenameBatchStream {
|
||||
fn drop(&mut self) {
|
||||
if !self.terminal {
|
||||
let missing = (self.expected_mask & !self.seen_mask).count_ones() as usize;
|
||||
rustfs_io_metrics::record_ecstore_rename_batch_terminal_missing(missing);
|
||||
}
|
||||
if let Some(admission) = self.admission.take() {
|
||||
admission.quarantine_until(self.deadline);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct BulkBodyPermit {
|
||||
_metrics: rustfs_io_metrics::internode_metrics::InternodeBulkBodyAdmissionGuard,
|
||||
_permit: OwnedSemaphorePermit,
|
||||
@@ -1226,6 +1473,10 @@ fn spawn_control_channel_prewarm(addr: String) {
|
||||
}
|
||||
|
||||
impl RemoteDisk {
|
||||
pub(crate) fn rename_stream_batch_enabled(&self) -> bool {
|
||||
self.rename_admission.is_some()
|
||||
}
|
||||
|
||||
async fn guarded_delete_versions_rpc(
|
||||
&self,
|
||||
volume: &str,
|
||||
@@ -1551,6 +1802,8 @@ impl RemoteDisk {
|
||||
rustfs_utils::get_env_bool(ENV_RUSTFS_DRIVE_ACTIVE_MONITORING, DEFAULT_RUSTFS_DRIVE_ACTIVE_MONITORING);
|
||||
let bulk_body_admission = peer_bulk_body_admission(&addr, bulk_body_concurrency_limit()?);
|
||||
let bulk_body_wait_timeout = bulk_body_wait_timeout();
|
||||
let rename_admission =
|
||||
peer_rename_admission(&addr, rustfs_utils::get_env_bool(ENV_RUSTFS_EXPERIMENTAL_RENAME_STREAM_BATCH, false));
|
||||
|
||||
let disk = Self {
|
||||
handle_id: Uuid::new_v4(),
|
||||
@@ -1573,6 +1826,7 @@ impl RemoteDisk {
|
||||
network_error_logs: rustfs_utils::LogThrottle::new(REMOTE_DISK_RPC_FAILURE_LOG_INTERVAL_MS),
|
||||
bulk_body_admission,
|
||||
bulk_body_wait_timeout,
|
||||
rename_admission,
|
||||
};
|
||||
record_drive_runtime_state(ep, RuntimeDriveHealthState::Online);
|
||||
|
||||
@@ -2308,6 +2562,54 @@ impl RemoteDisk {
|
||||
}
|
||||
}
|
||||
|
||||
async fn execute_peer_batch_with_timeout<T, F, Fut>(
|
||||
&self,
|
||||
op: &'static str,
|
||||
operation: F,
|
||||
timeout_duration: Duration,
|
||||
) -> Result<T>
|
||||
where
|
||||
F: FnOnce() -> Fut,
|
||||
Fut: std::future::Future<Output = Result<T>>,
|
||||
{
|
||||
if timeout_duration == Duration::ZERO {
|
||||
self.record_timeout(op, timeout_duration);
|
||||
return Err(DiskError::Timeout);
|
||||
}
|
||||
match time::timeout(timeout_duration, operation()).await {
|
||||
Ok(operation_result) => {
|
||||
if operation_result.is_ok() {
|
||||
rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&self.addr);
|
||||
}
|
||||
let evict = operation_result
|
||||
.as_ref()
|
||||
.err()
|
||||
.is_some_and(|error| !error.is_metacache_output_stream_closed() && is_network_like_disk_error(error));
|
||||
self.handle_network_like_error(op, timeout_duration, &operation_result, FailureHealthAction::IgnoreFailure)
|
||||
.await;
|
||||
if evict {
|
||||
evict_failed_connection_with_reason(
|
||||
&self.addr,
|
||||
ChannelRebuildReason::RpcFailure,
|
||||
ConnectionEvictionLogLevel::Debug,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
operation_result
|
||||
}
|
||||
Err(_) => {
|
||||
self.record_timeout(op, timeout_duration);
|
||||
evict_failed_connection_with_reason(
|
||||
&self.addr,
|
||||
ChannelRebuildReason::OperationTimeout,
|
||||
ConnectionEvictionLogLevel::Debug,
|
||||
)
|
||||
.await;
|
||||
Err(DiskError::Timeout)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn record_timeout(&self, op: &'static str, timeout_duration: Duration) {
|
||||
counter!(
|
||||
"rustfs_drive_op_timeout_total",
|
||||
@@ -2494,6 +2796,38 @@ impl RemoteDisk {
|
||||
}))
|
||||
}
|
||||
|
||||
async fn acquire_rename_admission_until(
|
||||
&self,
|
||||
item_count: usize,
|
||||
deadline: tokio::time::Instant,
|
||||
) -> Result<Option<RenameAdmissionGrant>> {
|
||||
let Some(semaphore) = &self.rename_admission else {
|
||||
return Ok(None);
|
||||
};
|
||||
if item_count == 0 || item_count > rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT {
|
||||
return Err(Error::other("invalid internode rename admission weight"));
|
||||
}
|
||||
if let Some(err) = self.offline_bypass_error() {
|
||||
return Err(err);
|
||||
}
|
||||
let weight = u32::try_from(item_count).map_err(Error::other)?;
|
||||
let started = rustfs_io_metrics::put_stage_metrics_enabled().then(Instant::now);
|
||||
let (result, timed_out) = match tokio::time::timeout_at(deadline, Arc::clone(semaphore).acquire_many_owned(weight)).await
|
||||
{
|
||||
Ok(Ok(permit)) => (Ok(permit), false),
|
||||
Ok(Err(_)) => (Err(Error::RemoteClientUnavailable("internode rename admission closed".to_owned())), false),
|
||||
Err(_) => (Err(DiskError::DiskOngoingReq), true),
|
||||
};
|
||||
if let Some(started) = started {
|
||||
rustfs_io_metrics::record_ecstore_rename_admission(item_count > 1, item_count, started.elapsed(), timed_out);
|
||||
}
|
||||
let permit = result?;
|
||||
Ok(Some(RenameAdmissionGrant {
|
||||
permit: Some(permit),
|
||||
batch: item_count > 1,
|
||||
}))
|
||||
}
|
||||
|
||||
async fn get_bulk_client(&self) -> Result<NodeRpcClient> {
|
||||
if let Some(err) = self.offline_bypass_error() {
|
||||
return Err(err);
|
||||
@@ -2634,6 +2968,18 @@ fn decode_msgpack_or_json<T: DeserializeOwned>(binary: &[u8], json: &str, value_
|
||||
}
|
||||
}
|
||||
|
||||
fn decode_msgpack_strict<T: DeserializeOwned>(binary: &[u8]) -> Result<T> {
|
||||
if binary.is_empty() {
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
let mut cursor = Cursor::new(binary);
|
||||
let value = rmp_serde::from_read(&mut cursor).map_err(|_| DiskError::FileCorrupt)?;
|
||||
if cursor.position() != u64::try_from(binary.len()).map_err(Error::other)? {
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
Ok(value)
|
||||
}
|
||||
|
||||
fn decode_multipart_publication_response(binary: &[u8]) -> Result<RenameDataResp> {
|
||||
let result = if binary.len() > MultipartSourceOperation::max_wire_payload_bytes() {
|
||||
Err(DiskError::FileCorrupt)
|
||||
@@ -2831,6 +3177,97 @@ impl RemoteDisk {
|
||||
.await
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
pub(crate) async fn start_rename_data_batch(
|
||||
&self,
|
||||
src_volume: &str,
|
||||
src_path: &str,
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
items: Vec<RenameBatchClientItem<'_>>,
|
||||
) -> std::result::Result<RemoteRenameBatchStream, RenameBatchStartError> {
|
||||
if self.rename_admission.is_none() || items.is_empty() || items.len() > 16 {
|
||||
return Err(RenameBatchStartError::NotDispatched(DiskError::UnsupportedDisk));
|
||||
}
|
||||
let deadline = tokio::time::Instant::now() + get_max_timeout_duration();
|
||||
let mut expected_mask = 0_u16;
|
||||
let mut request_items = Vec::with_capacity(items.len());
|
||||
for item in items {
|
||||
if item.slot >= 16 {
|
||||
return Err(RenameBatchStartError::NotDispatched(DiskError::Unexpected));
|
||||
}
|
||||
let bit = 1_u16 << item.slot;
|
||||
if expected_mask & bit != 0 {
|
||||
return Err(RenameBatchStartError::NotDispatched(DiskError::Unexpected));
|
||||
}
|
||||
expected_mask |= bit;
|
||||
let item_index = u32::try_from(item.slot)
|
||||
.map_err(Error::other)
|
||||
.map_err(RenameBatchStartError::NotDispatched)?;
|
||||
let file_info_msgpack = encode_file_info_msgpack(item.file_info).map_err(RenameBatchStartError::NotDispatched)?;
|
||||
request_items.push(RenameDataBatchItem {
|
||||
item_index,
|
||||
disk: item.disk,
|
||||
file_info_msgpack: file_info_msgpack.into(),
|
||||
});
|
||||
}
|
||||
let mut request = Request::new(RenameDataBatchRequest {
|
||||
src_volume: src_volume.to_owned(),
|
||||
src_path: src_path.to_owned(),
|
||||
dst_volume: dst_volume.to_owned(),
|
||||
dst_path: dst_path.to_owned(),
|
||||
items: request_items,
|
||||
});
|
||||
let canonical_body = rustfs_protos::canonical_rename_data_batch_request_body(request.get_ref())
|
||||
.map_err(Error::other)
|
||||
.map_err(RenameBatchStartError::NotDispatched)?;
|
||||
crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &canonical_body)
|
||||
.map_err(Error::other)
|
||||
.map_err(RenameBatchStartError::NotDispatched)?;
|
||||
let mut client = match tokio::time::timeout_at(deadline, self.get_client()).await {
|
||||
Ok(result) => result.map_err(RenameBatchStartError::NotDispatched)?,
|
||||
Err(_) => return Err(RenameBatchStartError::NotDispatched(DiskError::Timeout)),
|
||||
};
|
||||
let admission = self
|
||||
.acquire_rename_admission_until(request.get_ref().items.len(), deadline)
|
||||
.await
|
||||
.map_err(RenameBatchStartError::NotDispatched)?;
|
||||
let guard = RenameBatchCallGuard {
|
||||
admission,
|
||||
deadline,
|
||||
dispatch_state: RenameAdmissionDispatchState::NotDispatched,
|
||||
};
|
||||
let call_timeout = deadline.saturating_duration_since(tokio::time::Instant::now());
|
||||
if call_timeout == Duration::ZERO {
|
||||
guard.release_without_quarantine();
|
||||
return Err(RenameBatchStartError::NotDispatched(DiskError::Timeout));
|
||||
}
|
||||
let mut guard = guard;
|
||||
guard.mark_dispatched();
|
||||
let response = match self
|
||||
.execute_peer_batch_with_timeout(
|
||||
"rename_data_batch",
|
||||
|| async {
|
||||
match client.rename_data_batch(request).await {
|
||||
Ok(response) => Ok(response),
|
||||
Err(status) if status.code() == Code::Unimplemented => Err(DiskError::UnsupportedDisk),
|
||||
Err(status) => Err(status.into()),
|
||||
}
|
||||
},
|
||||
call_timeout,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Err(DiskError::UnsupportedDisk) => {
|
||||
guard.release_without_quarantine();
|
||||
return Err(RenameBatchStartError::UnsupportedBeforeStream);
|
||||
}
|
||||
Err(error) => return Err(RenameBatchStartError::MayHavePublished(error)),
|
||||
Ok(response) => response,
|
||||
};
|
||||
Ok(guard.into_stream(response.into_inner(), expected_mask))
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
pub(crate) async fn rename_data_borrowed_with_fence(
|
||||
&self,
|
||||
@@ -2902,6 +3339,27 @@ impl RemoteDisk {
|
||||
let scanner_publication_lease = scanner_publication_lease_token
|
||||
.map(|token| Bytes::copy_from_slice(token.as_bytes()))
|
||||
.unwrap_or_default();
|
||||
let rename_deadline = tokio::time::Instant::now() + get_max_timeout_duration();
|
||||
let rename_admission = self.acquire_rename_admission_until(1, rename_deadline).await?;
|
||||
let mut rename_admission_guard = RenameBatchCallGuard {
|
||||
admission: rename_admission,
|
||||
deadline: rename_deadline,
|
||||
dispatch_state: RenameAdmissionDispatchState::NotDispatched,
|
||||
};
|
||||
let rpc_kind = if multipart_rename {
|
||||
rustfs_io_metrics::ECSTORE_RENAME_RPC_KIND_MULTIPART
|
||||
} else if pool_meta_cas {
|
||||
rustfs_io_metrics::ECSTORE_RENAME_RPC_KIND_POOL_META_CAS
|
||||
} else if quota_fenced {
|
||||
rustfs_io_metrics::ECSTORE_RENAME_RPC_KIND_QUOTA_FENCE
|
||||
} else if incarnation_bound {
|
||||
rustfs_io_metrics::ECSTORE_RENAME_RPC_KIND_INCARNATION
|
||||
} else if scanner_publication_lease_token.is_some() {
|
||||
rustfs_io_metrics::ECSTORE_RENAME_RPC_KIND_SCANNER_LEASE
|
||||
} else {
|
||||
rustfs_io_metrics::ECSTORE_RENAME_RPC_KIND_ORDINARY
|
||||
};
|
||||
rustfs_io_metrics::record_ecstore_rename_remote_rpc(rpc_kind);
|
||||
let rpc_started = rustfs_io_metrics::put_stage_timer();
|
||||
let mut retried_dual_write = false;
|
||||
let response = loop {
|
||||
@@ -2958,6 +3416,7 @@ impl RemoteDisk {
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_PREPARE,
|
||||
attempt_prepare_started,
|
||||
);
|
||||
rename_admission_guard.mark_dispatched();
|
||||
let result = rustfs_io_metrics::observe_put_stage_future(
|
||||
async {
|
||||
if multipart_rename {
|
||||
@@ -2988,6 +3447,7 @@ impl RemoteDisk {
|
||||
Err(status) => return Err(status.into()),
|
||||
}
|
||||
};
|
||||
rename_admission_guard.complete();
|
||||
let response_process_started = rustfs_io_metrics::put_stage_timer();
|
||||
let response_headers_started = rustfs_io_metrics::put_stage_timer();
|
||||
let _ = response.metadata().len();
|
||||
@@ -12963,6 +13423,127 @@ mod tests {
|
||||
.expect("shared gate remains open");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rename_admission_is_weighted_shared_and_released_per_item() {
|
||||
use std::future::Future;
|
||||
use std::task::{Context, Poll, Waker};
|
||||
|
||||
let first = peer_rename_admission("http://rename-peer-a", true).expect("enabled rename gate");
|
||||
let same = peer_rename_admission("http://rename-peer-a", true).expect("same peer rename gate");
|
||||
let other = peer_rename_admission("http://rename-peer-b", true).expect("other peer rename gate");
|
||||
assert!(Arc::ptr_eq(&first, &same));
|
||||
assert!(!Arc::ptr_eq(&first, &other));
|
||||
assert!(peer_rename_admission("http://rename-disabled", false).is_none());
|
||||
assert_eq!(first.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT);
|
||||
|
||||
let permit = Arc::clone(&first)
|
||||
.acquire_many_owned(4)
|
||||
.await
|
||||
.expect("weighted batch admission");
|
||||
let mut grant = RenameAdmissionGrant {
|
||||
permit: Some(permit),
|
||||
batch: true,
|
||||
};
|
||||
assert_eq!(first.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT - 4);
|
||||
assert!(grant.release_one());
|
||||
assert_eq!(first.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT - 3);
|
||||
drop(grant);
|
||||
assert_eq!(first.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT);
|
||||
|
||||
let held = Arc::clone(&first)
|
||||
.acquire_many_owned(u32::try_from(rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT - 3).expect("small limit"))
|
||||
.await
|
||||
.expect("reserve all but three permits");
|
||||
let mut waiting = Box::pin(Arc::clone(&same).acquire_many_owned(4));
|
||||
assert!(matches!(waiting.as_mut().poll(&mut Context::from_waker(Waker::noop())), Poll::Pending));
|
||||
assert_eq!(first.available_permits(), 0, "the FIFO waiter reserves currently available permits");
|
||||
drop(waiting);
|
||||
assert_eq!(
|
||||
first.available_permits(),
|
||||
3,
|
||||
"cancelling a queued weighted acquire must return its reservation"
|
||||
);
|
||||
let waiting = Box::pin(Arc::clone(&same).acquire_many_owned(4));
|
||||
drop(held);
|
||||
let acquired = tokio::time::timeout(Duration::from_secs(1), waiting)
|
||||
.await
|
||||
.expect("weighted waiter wakes")
|
||||
.expect("rename gate remains open");
|
||||
drop(acquired);
|
||||
assert_eq!(first.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn rename_admission_quarantine_holds_until_absolute_deadline() {
|
||||
let gate = peer_rename_admission("http://rename-quarantine-peer", true).expect("enabled rename gate");
|
||||
let permit = Arc::clone(&gate)
|
||||
.acquire_many_owned(4)
|
||||
.await
|
||||
.expect("weighted batch admission");
|
||||
let deadline = tokio::time::Instant::now() + Duration::from_secs(1);
|
||||
RenameAdmissionGrant {
|
||||
permit: Some(permit),
|
||||
batch: true,
|
||||
}
|
||||
.quarantine_until(deadline);
|
||||
assert_eq!(gate.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT - 4);
|
||||
|
||||
tokio::time::advance(Duration::from_millis(999)).await;
|
||||
tokio::task::yield_now().await;
|
||||
assert_eq!(gate.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT - 4);
|
||||
tokio::time::advance(Duration::from_millis(1)).await;
|
||||
tokio::task::yield_now().await;
|
||||
assert_eq!(gate.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn rename_call_guard_quarantines_only_after_dispatch() {
|
||||
let gate = peer_rename_admission("http://rename-call-guard-peer", true).expect("enabled rename gate");
|
||||
let deadline = tokio::time::Instant::now() + Duration::from_secs(1);
|
||||
let permit = Arc::clone(&gate).acquire_many_owned(4).await.expect("pre-dispatch permits");
|
||||
drop(RenameBatchCallGuard {
|
||||
admission: Some(RenameAdmissionGrant {
|
||||
permit: Some(permit),
|
||||
batch: true,
|
||||
}),
|
||||
deadline,
|
||||
dispatch_state: RenameAdmissionDispatchState::NotDispatched,
|
||||
});
|
||||
assert_eq!(gate.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT);
|
||||
|
||||
let permit = Arc::clone(&gate).acquire_many_owned(4).await.expect("dispatched permits");
|
||||
let mut guard = RenameBatchCallGuard {
|
||||
admission: Some(RenameAdmissionGrant {
|
||||
permit: Some(permit),
|
||||
batch: true,
|
||||
}),
|
||||
deadline,
|
||||
dispatch_state: RenameAdmissionDispatchState::NotDispatched,
|
||||
};
|
||||
guard.mark_dispatched();
|
||||
drop(guard);
|
||||
assert_eq!(gate.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT - 4);
|
||||
tokio::time::advance(Duration::from_secs(1)).await;
|
||||
tokio::task::yield_now().await;
|
||||
assert_eq!(gate.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT);
|
||||
|
||||
let permit = Arc::clone(&gate)
|
||||
.acquire_many_owned(4)
|
||||
.await
|
||||
.expect("proven pre-mutation permits");
|
||||
let mut guard = RenameBatchCallGuard {
|
||||
admission: Some(RenameAdmissionGrant {
|
||||
permit: Some(permit),
|
||||
batch: true,
|
||||
}),
|
||||
deadline,
|
||||
dispatch_state: RenameAdmissionDispatchState::NotDispatched,
|
||||
};
|
||||
guard.mark_dispatched();
|
||||
guard.release_without_quarantine();
|
||||
assert_eq!(gate.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn remote_disk_builds_optional_complete_body_gate_from_configuration() {
|
||||
temp_env::async_with_vars([(rustfs_config::ENV_INTERNODE_BULK_BODY_CONCURRENCY_LIMIT, Some("3"))], async {
|
||||
|
||||
@@ -12378,6 +12378,7 @@ struct DecommissionCapacityLockOrderBarrierState {
|
||||
external_heal_target_lock_attempted: tokio::sync::Notify,
|
||||
target_gate_retry_entered: tokio::sync::Notify,
|
||||
target_gate_retry_entries: AtomicUsize,
|
||||
target_gate_retry_pause: std::sync::OnceLock<tokio::sync::Semaphore>,
|
||||
target_gate_inline_retry_entered: tokio::sync::Notify,
|
||||
target_gate_inline_retry_entries: AtomicUsize,
|
||||
target_gate_exact_reloads: AtomicUsize,
|
||||
@@ -12425,6 +12426,7 @@ impl DecommissionCapacityLockOrderBarrier {
|
||||
external_heal_target_lock_attempted: tokio::sync::Notify::new(),
|
||||
target_gate_retry_entered: tokio::sync::Notify::new(),
|
||||
target_gate_retry_entries: AtomicUsize::new(0),
|
||||
target_gate_retry_pause: std::sync::OnceLock::new(),
|
||||
target_gate_inline_retry_entered: tokio::sync::Notify::new(),
|
||||
target_gate_inline_retry_entries: AtomicUsize::new(0),
|
||||
target_gate_exact_reloads: AtomicUsize::new(0),
|
||||
@@ -12520,6 +12522,13 @@ impl DecommissionCapacityLockOrderBarrier {
|
||||
.expect("decommission entries should observe target gate contention");
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
pub(crate) fn pause_target_gate_retries(&self) -> &tokio::sync::Semaphore {
|
||||
self.state
|
||||
.target_gate_retry_pause
|
||||
.get_or_init(|| tokio::sync::Semaphore::new(0))
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
pub(crate) fn target_gate_inline_retries(&self) -> usize {
|
||||
self.state.target_gate_inline_retry_entries.load(Ordering::Acquire)
|
||||
@@ -12630,6 +12639,9 @@ impl DecommissionCapacityLockOrderBarrier {
|
||||
#[cfg(test)]
|
||||
impl Drop for DecommissionCapacityLockOrderBarrier {
|
||||
fn drop(&mut self) {
|
||||
if let Some(pause) = self.state.target_gate_retry_pause.get() {
|
||||
pause.close();
|
||||
}
|
||||
self.state.owner_release.notify_one();
|
||||
self.state.external_object_capacity_probe_release.notify_one();
|
||||
self.state.external_object_commit_phase_release.notify_one();
|
||||
@@ -12758,7 +12770,7 @@ async fn pause_decommission_cancel_target_timeout(store_id: uuid::Uuid) {
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn notify_decommission_target_gate_retry(store_id: uuid::Uuid) {
|
||||
async fn observe_decommission_target_gate_retry(store_id: uuid::Uuid) {
|
||||
let barrier = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
.lock()
|
||||
@@ -12769,6 +12781,11 @@ fn notify_decommission_target_gate_retry(store_id: uuid::Uuid) {
|
||||
if let Some(barrier) = barrier {
|
||||
barrier.target_gate_retry_entries.fetch_add(1, Ordering::AcqRel);
|
||||
barrier.target_gate_retry_entered.notify_one();
|
||||
if let Some(pause) = barrier.target_gate_retry_pause.get()
|
||||
&& let Ok(permit) = pause.acquire().await
|
||||
{
|
||||
permit.forget();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17638,7 +17655,7 @@ impl ECStore {
|
||||
target_busy_attempt: usize,
|
||||
) -> Result<Option<DecommissionCapacityTargetPermit>> {
|
||||
#[cfg(test)]
|
||||
notify_decommission_target_gate_retry(self.id);
|
||||
observe_decommission_target_gate_retry(self.id).await;
|
||||
let mut wait_attempt = target_busy_attempt;
|
||||
let target_guard = loop {
|
||||
let retry_delay =
|
||||
|
||||
@@ -8360,13 +8360,16 @@ mod decommission_lock_order_tests {
|
||||
.await
|
||||
.expect("hold the target gate before starting both waiters");
|
||||
let retry_observer = DecommissionCapacityLockOrderBarrier::install(other_store.id, other_store.id);
|
||||
// Both workers must observe the original gate as busy, but slow disk I/O
|
||||
// in the holder or first waiter must not consume the second waiter's retry budget.
|
||||
let retry_pause = retry_observer.pause_target_gate_retries();
|
||||
|
||||
let mut workers = Vec::new();
|
||||
let mut workers = tokio::task::JoinSet::new();
|
||||
for object in objects {
|
||||
let worker_store = Arc::clone(&other_store);
|
||||
let worker_bucket = bucket.clone();
|
||||
let source_set = other_store.pools[0].get_disks_by_key(object);
|
||||
workers.push(tokio::spawn(async move {
|
||||
workers.spawn(async move {
|
||||
worker_store
|
||||
.decommission_entry_for_test(
|
||||
0,
|
||||
@@ -8378,7 +8381,7 @@ mod decommission_lock_order_tests {
|
||||
source_set,
|
||||
)
|
||||
.await
|
||||
}));
|
||||
});
|
||||
}
|
||||
retry_observer.wait_until_target_gate_retries(2).await;
|
||||
assert_eq!(
|
||||
@@ -8425,6 +8428,7 @@ mod decommission_lock_order_tests {
|
||||
);
|
||||
|
||||
drop(target_guard);
|
||||
retry_pause.add_permits(1);
|
||||
retry_observer.wait_until_owner_paused().await;
|
||||
assert_eq!(
|
||||
retry_observer.target_gate_exact_reloads(),
|
||||
@@ -8443,6 +8447,13 @@ mod decommission_lock_order_tests {
|
||||
.expect("a pinned waiter must not hold the alternate target gate");
|
||||
drop(alternate_probe);
|
||||
retry_observer.release_owner();
|
||||
tokio::time::timeout(Duration::from_secs(30), workers.join_next())
|
||||
.await
|
||||
.expect("the pinned waiter should finish before the other retries")
|
||||
.expect("the pinned waiter must exist")
|
||||
.expect("the pinned waiter task should not panic")
|
||||
.expect("the pinned waiter migration should succeed");
|
||||
retry_pause.add_permits(1);
|
||||
retry_observer.wait_until_owner_paused().await;
|
||||
assert_eq!(
|
||||
retry_observer.target_gate_exact_reloads(),
|
||||
@@ -8462,13 +8473,12 @@ mod decommission_lock_order_tests {
|
||||
drop(original_probe);
|
||||
retry_observer.release_owner();
|
||||
|
||||
for worker in workers {
|
||||
tokio::time::timeout(Duration::from_secs(30), worker)
|
||||
.await
|
||||
.expect("same-target waiter should finish after permit transfer")
|
||||
.expect("same-target waiter task should not panic")
|
||||
.expect("same-target waiter migration should succeed");
|
||||
}
|
||||
tokio::time::timeout(Duration::from_secs(30), workers.join_next())
|
||||
.await
|
||||
.expect("the reselected waiter should finish after permit transfer")
|
||||
.expect("the reselected waiter must exist")
|
||||
.expect("the reselected waiter task should not panic")
|
||||
.expect("the reselected waiter migration should succeed");
|
||||
assert_eq!(
|
||||
retry_observer.target_gate_exact_reloads(),
|
||||
2,
|
||||
|
||||
@@ -842,6 +842,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for Sets {
|
||||
user_defined: dst_opts.user_defined.clone(),
|
||||
shard_integrity_write_mode: Some(src_info.shard_integrity_write_mode()),
|
||||
versioned: dst_opts.versioned,
|
||||
version_suspended: dst_opts.version_suspended,
|
||||
version_id: dst_opts.version_id.clone(),
|
||||
mod_time: dst_opts.mod_time,
|
||||
http_preconditions: dst_opts.http_preconditions.clone(),
|
||||
|
||||
@@ -11423,6 +11423,20 @@ impl LocalDisk {
|
||||
Ok(data)
|
||||
}
|
||||
|
||||
fn part_path_resolver(&self) -> impl Fn(&str, &str) -> Result<PathBuf> + Send + use<> {
|
||||
let root = self.io_root().to_owned();
|
||||
#[cfg(target_os = "linux")]
|
||||
let mount_lease = self.mount_lease.clone();
|
||||
move |bucket, key| {
|
||||
let (bucket_path, path) = build_local_disk_object_path(&root, bucket, key);
|
||||
#[cfg(target_os = "linux")]
|
||||
check_local_disk_valid_object_path_at(&root, &mount_lease, &bucket_path, &path)?;
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
check_local_disk_valid_object_path(&root, &bucket_path, &path)?;
|
||||
Ok(path)
|
||||
}
|
||||
}
|
||||
|
||||
async fn read_part_metadata(&self, bucket: &str, volume_dir: &Path, path_str: &str) -> Result<ObjectPartInfo> {
|
||||
let io_permit = acquire_context_disk_io_permit().await.map_err(DiskError::Io)?;
|
||||
let path = Path::new(path_str);
|
||||
@@ -11435,8 +11449,10 @@ impl LocalDisk {
|
||||
.and_then(|v| v.parse::<usize>().ok())
|
||||
.unwrap_or_default();
|
||||
let parent_str = path.parent().unwrap_or_else(|| Path::new("")).to_string_lossy();
|
||||
let data_path = self.io_get_object_path(bucket, &path_join_buf(&[parent_str.as_ref(), &format!("part.{num}")]))?;
|
||||
let metadata_path = self.io_get_object_path(bucket, path.to_string_lossy().as_ref());
|
||||
let data_key = path_join_buf(&[parent_str.as_ref(), &format!("part.{num}")]);
|
||||
let metadata_key = path_str.to_owned();
|
||||
let volume = bucket.to_owned();
|
||||
let resolve = self.part_path_resolver();
|
||||
// A part's existence check, metadata read and decode share one dispatch.
|
||||
// Keep open errors unmapped for the existing missing-volume fallback.
|
||||
let root = self.publication_root.clone();
|
||||
@@ -11446,12 +11462,13 @@ impl LocalDisk {
|
||||
error: Some(error),
|
||||
..Default::default()
|
||||
};
|
||||
let data_path = resolve(&volume, &data_key)?;
|
||||
if let Err(err) = std::fs::metadata(data_path) {
|
||||
return Ok(Ok(part_error(err.to_string())));
|
||||
}
|
||||
// Invalid metadata paths remain request errors, but missing data wins
|
||||
// first, as it does in the serial reader.
|
||||
let metadata_path = metadata_path?;
|
||||
let metadata_path = resolve(&volume, &metadata_key)?;
|
||||
Ok(read_all_data_std(root.open_readonly(&metadata_path))
|
||||
.map(|(data, _)| ObjectPartInfo::unmarshal(&data).unwrap_or_else(|err| part_error(err.to_string()))))
|
||||
})
|
||||
@@ -12944,7 +12961,10 @@ impl LocalDisk {
|
||||
let name = entry.trim_end_matches(SLASH_SEPARATOR);
|
||||
let name = decode_dir_object(format!("{}/{}", current, name).as_str());
|
||||
|
||||
if opts.limit <= 0 || metadata_counts_toward_limit(&metadata) {
|
||||
if opts.limit <= 0
|
||||
|| (opts.forward_to.as_ref().is_none_or(|marker| &name >= marker)
|
||||
&& metadata_counts_toward_limit(&metadata))
|
||||
{
|
||||
*objs_returned += 1;
|
||||
}
|
||||
|
||||
@@ -13214,7 +13234,9 @@ impl LocalDisk {
|
||||
|
||||
meta.metadata = read.bytes;
|
||||
|
||||
if opts.limit <= 0 || read.counts_toward_limit {
|
||||
if opts.limit <= 0
|
||||
|| (opts.forward_to.as_ref().is_none_or(|marker| &meta.name >= marker) && read.counts_toward_limit)
|
||||
{
|
||||
*objs_returned += 1;
|
||||
}
|
||||
|
||||
@@ -15487,42 +15509,36 @@ impl DiskAPI for LocalDisk {
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
async fn read_parts(&self, bucket: &str, paths: &[String]) -> Result<Vec<ObjectPartInfo>> {
|
||||
if bucket == RUSTFS_META_MULTIPART_BUCKET && !paths.is_empty() {
|
||||
let mut cached_parts = Vec::with_capacity(paths.len());
|
||||
let mut all_cached = true;
|
||||
let mut verified_source: Option<(&str, PathBuf)> = None;
|
||||
for path_str in paths {
|
||||
let Some(num) = Path::new(path_str)
|
||||
.file_name()
|
||||
.and_then(|v| v.to_str())
|
||||
.and_then(|v| v.strip_prefix("part."))
|
||||
.and_then(|v| v.strip_suffix(".meta"))
|
||||
.and_then(|v| v.parse::<usize>().ok())
|
||||
else {
|
||||
all_cached = false;
|
||||
break;
|
||||
};
|
||||
let scope_root = multipart_staging::upload_scope_root(path_str, true);
|
||||
let (source, head_already_verified) = match verified_source.as_ref() {
|
||||
Some((prev_root, prev_source)) if *prev_root == scope_root => (prev_source, true),
|
||||
_ => {
|
||||
let Ok(source) = self.io_get_object_open_path(bucket, scope_root) else {
|
||||
all_cached = false;
|
||||
break;
|
||||
};
|
||||
verified_source = Some((scope_root, source));
|
||||
(&verified_source.as_ref().expect("just initialized").1, false)
|
||||
let permit = acquire_context_disk_io_permit().await.map_err(DiskError::Io)?;
|
||||
let resolve = self.part_path_resolver();
|
||||
let keys = paths.to_owned();
|
||||
let cached = spawn_blocking_with_disk_io_permit(permit, move || {
|
||||
let mut parts = Vec::with_capacity(keys.len());
|
||||
let mut verified = None;
|
||||
for key in &keys {
|
||||
let num = Path::new(key)
|
||||
.file_name()?
|
||||
.to_str()?
|
||||
.strip_prefix("part.")?
|
||||
.strip_suffix(".meta")?
|
||||
.parse()
|
||||
.ok()?;
|
||||
let scope = multipart_staging::upload_scope_root(key, true);
|
||||
let same_head = verified.as_ref().is_some_and(|(prev, _)| *prev == scope);
|
||||
if !same_head {
|
||||
verified = Some((scope, resolve(RUSTFS_META_MULTIPART_BUCKET, scope).ok()?));
|
||||
}
|
||||
};
|
||||
let Some(part) =
|
||||
multipart_source_state::get_active_multipart_source_part_with_head(source, num, head_already_verified)
|
||||
else {
|
||||
all_cached = false;
|
||||
break;
|
||||
};
|
||||
cached_parts.push(part);
|
||||
}
|
||||
if all_cached {
|
||||
return Ok(cached_parts);
|
||||
let (_, source) = verified.as_ref()?;
|
||||
parts.push(multipart_source_state::get_active_multipart_source_part_with_head(
|
||||
source, num, same_head,
|
||||
)?);
|
||||
}
|
||||
Some(parts)
|
||||
})
|
||||
.await
|
||||
.map_err(DiskError::from)?;
|
||||
if let Some(parts) = cached {
|
||||
return Ok(parts);
|
||||
}
|
||||
}
|
||||
let volume_dir = self.io_get_bucket_path(bucket)?;
|
||||
@@ -17957,10 +17973,13 @@ fn should_read_legacy_inline_part(fi: &FileInfo, storage_class_config: &crate::c
|
||||
}
|
||||
|
||||
let part = &fi.parts[0];
|
||||
if !usize::try_from(part.actual_size).is_ok_and(|size| size <= fi.erasure.block_size) {
|
||||
if part.size > fi.erasure.block_size || !usize::try_from(part.actual_size).is_ok_and(|size| size <= fi.erasure.block_size) {
|
||||
return false;
|
||||
}
|
||||
let shard_size = fi.shard_file_size(part.actual_size);
|
||||
let Ok(stored_size) = i64::try_from(part.size) else {
|
||||
return false;
|
||||
};
|
||||
let shard_size = fi.shard_file_size(stored_size.max(part.actual_size));
|
||||
let versioned = fi.versioned || fi.version_id.is_some_and(|version_id| !version_id.is_nil());
|
||||
storage_class_config.should_inline(shard_size, fi.erasure.data_blocks, versioned)
|
||||
}
|
||||
@@ -18727,6 +18746,8 @@ mod test {
|
||||
"a multi-block external object can still fit the per-shard budget"
|
||||
);
|
||||
assert!(!should_read_legacy_inline_part(&ec12, &config));
|
||||
ec12.parts[0].actual_size = 1;
|
||||
assert!(!should_read_legacy_inline_part(&ec12, &config));
|
||||
}
|
||||
|
||||
fn guarded_delete_group(fi: FileInfo, expected_current: CurrentDeleteGeneration) -> GuardedFileInfoVersions {
|
||||
@@ -31781,13 +31802,14 @@ mod test {
|
||||
Endpoint::try_from(dir.path().to_str().expect("operation should succeed")).expect("operation should succeed");
|
||||
let disk = LocalDisk::new(&endpoint, false).await.expect("operation should succeed");
|
||||
|
||||
async fn scan_names(disk: &LocalDisk, bucket: &str, base_dir: &str, forward_to: &str) -> (Vec<String>, i32) {
|
||||
async fn scan_names(disk: &LocalDisk, bucket: &str, base_dir: &str, forward_to: &str, limit: i32) -> (Vec<String>, i32) {
|
||||
let (reader, mut writer) = tokio::io::duplex(4096);
|
||||
let mut out = MetacacheWriter::new(&mut writer);
|
||||
let opts = WalkDirOptions {
|
||||
bucket: bucket.to_string(),
|
||||
base_dir: base_dir.to_string(),
|
||||
recursive: true,
|
||||
limit,
|
||||
forward_to: Some(forward_to.to_string()),
|
||||
..Default::default()
|
||||
};
|
||||
@@ -31812,7 +31834,7 @@ mod test {
|
||||
}
|
||||
|
||||
let (engineering_names, engineering_count) =
|
||||
scan_names(&disk, bucket, "engineering/", "engineering/engineering/engineering/repo-0001").await;
|
||||
scan_names(&disk, bucket, "engineering/", "engineering/engineering/engineering/repo-0001", 0).await;
|
||||
|
||||
assert_eq!(
|
||||
engineering_names,
|
||||
@@ -31828,7 +31850,7 @@ mod test {
|
||||
assert_eq!(engineering_count as usize, engineering_names.len());
|
||||
|
||||
let (different_names, different_count) =
|
||||
scan_names(&disk, bucket, "different/", "different/prefix/prefix/repo-0001").await;
|
||||
scan_names(&disk, bucket, "different/", "different/prefix/prefix/repo-0001", 0).await;
|
||||
|
||||
assert_eq!(
|
||||
different_names,
|
||||
@@ -31840,7 +31862,8 @@ mod test {
|
||||
);
|
||||
assert_eq!(different_count as usize, different_names.len());
|
||||
|
||||
let (double_names, double_count) = scan_names(&disk, bucket, "engineering/", "engineering/engineering/repo-0001").await;
|
||||
let (double_names, double_count) =
|
||||
scan_names(&disk, bucket, "engineering/", "engineering/engineering/repo-0001", 0).await;
|
||||
|
||||
assert_eq!(
|
||||
double_names,
|
||||
@@ -31852,6 +31875,13 @@ mod test {
|
||||
"forward_to must not skip a child directory whose name repeats the base prefix"
|
||||
);
|
||||
assert_eq!(double_count as usize, double_names.len());
|
||||
|
||||
fs::write(bucket_dir.join("engineering/engineering").join(STORAGE_FORMAT_FILE), b"meta")
|
||||
.await
|
||||
.expect("write ancestor metadata");
|
||||
let (names, count) = scan_names(&disk, bucket, "engineering/", "engineering/engineering/repo-0001", 2).await;
|
||||
assert_eq!(count, 2);
|
||||
assert!(names.iter().any(|name| name == "engineering/engineering/repo-0002"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -336,31 +336,21 @@ pub(in crate::disk) fn generation(mut canonical: FileInfo) -> Result<crate::disk
|
||||
if !compat::contains_key_str(&canonical.metadata, suffix) {
|
||||
continue;
|
||||
}
|
||||
// The inline marker is a presence flag whose persisted value is empty.
|
||||
// Compare both declared aliases explicitly before normalizing it; the
|
||||
// general compatibility reader rejects empty strings by design.
|
||||
let value = if suffix == compat::SUFFIX_INLINE_DATA {
|
||||
let rustfs_key = compat::internal_key_rustfs(suffix);
|
||||
let minio_key = format!("{}{suffix}", compat::MINIO_INTERNAL_PREFIX);
|
||||
let mut value = None;
|
||||
for (key, candidate) in &canonical.metadata {
|
||||
if !key.eq_ignore_ascii_case(&rustfs_key) && !key.eq_ignore_ascii_case(&minio_key) {
|
||||
continue;
|
||||
}
|
||||
if value.is_some_and(|current: &String| current != candidate) {
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
value = Some(candidate);
|
||||
// Identity compares persisted bytes, including a consistently empty field.
|
||||
// Conflicting aliases still cannot identify a unique generation.
|
||||
let rustfs_key = compat::internal_key_rustfs(suffix);
|
||||
let minio_key = format!("{}{suffix}", compat::MINIO_INTERNAL_PREFIX);
|
||||
let mut value = None;
|
||||
for (key, candidate) in &canonical.metadata {
|
||||
if !key.eq_ignore_ascii_case(&rustfs_key) && !key.eq_ignore_ascii_case(&minio_key) {
|
||||
continue;
|
||||
}
|
||||
value.ok_or(DiskError::FileCorrupt)?.clone()
|
||||
} else {
|
||||
compat::get_consistent_str(&canonical.metadata, suffix)
|
||||
.ok_or(DiskError::FileCorrupt)?
|
||||
.to_owned()
|
||||
};
|
||||
if value.is_empty() && suffix != compat::SUFFIX_INLINE_DATA {
|
||||
continue;
|
||||
if value.is_some_and(|current: &String| current != candidate) {
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
value = Some(candidate);
|
||||
}
|
||||
let value = value.ok_or(DiskError::FileCorrupt)?.clone();
|
||||
compat::insert_str(&mut canonical.metadata, suffix, value);
|
||||
}
|
||||
fn sort_json_in_place(value: &mut serde_json::Value) {
|
||||
@@ -4759,7 +4749,12 @@ mod tests {
|
||||
legacy
|
||||
.metadata
|
||||
.insert(format!("{}{}", compat::MINIO_INTERNAL_PREFIX, compat::SUFFIX_COMPRESSION), "gzip".into());
|
||||
legacy.metadata.insert(
|
||||
format!("{}{}", compat::MINIO_INTERNAL_PREFIX, compat::SUFFIX_REPLICA_STATUS),
|
||||
String::new(),
|
||||
);
|
||||
let mut modern = legacy.clone();
|
||||
compat::insert_str(&mut modern.metadata, compat::SUFFIX_REPLICA_STATUS, String::new());
|
||||
compat::insert_str(&mut modern.metadata, compat::SUFFIX_INLINE_DATA, String::new());
|
||||
compat::insert_str(&mut modern.metadata, compat::SUFFIX_COMPRESSION, "gzip".into());
|
||||
assert_eq!(
|
||||
|
||||
@@ -17,6 +17,27 @@ use std::{
|
||||
};
|
||||
use uuid::Uuid;
|
||||
|
||||
fn verify_synced_target(
|
||||
dir: &Path,
|
||||
base: &Path,
|
||||
root: &os::PublicationRoot,
|
||||
length: u64,
|
||||
digest: [u8; 32],
|
||||
) -> std::io::Result<()> {
|
||||
// Target version histories are bounded by the prepared receipt, not the source-control record limit.
|
||||
let observed = os::observe_multipart_source_metadata(&dir.join(STORAGE_FORMAT_FILE), base, root, length).map_err(|err| {
|
||||
if err.kind() == std::io::ErrorKind::NotFound {
|
||||
IoError::other(DiskError::ConditionalDeleteConflict)
|
||||
} else {
|
||||
err
|
||||
}
|
||||
})?;
|
||||
if observed != (length, digest) {
|
||||
return Err(IoError::other(DiskError::ConditionalDeleteConflict));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
const STATE: &str = ".mpu-source-state";
|
||||
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
@@ -2470,30 +2491,7 @@ impl NativeMultipartPublication {
|
||||
return Err(IoError::other(DiskError::ConditionalDeleteConflict));
|
||||
}
|
||||
if target_already_synced {
|
||||
let pinned = os::pin_multipart_source_directory(&target_dir, base, root).map_err(|error| {
|
||||
if error.kind() == std::io::ErrorKind::NotFound {
|
||||
IoError::other(DiskError::ConditionalDeleteConflict)
|
||||
} else {
|
||||
error
|
||||
}
|
||||
})?;
|
||||
let target = os::read_multipart_source_metadata_in_pinned(
|
||||
&pinned,
|
||||
&target_dir,
|
||||
std::ffi::OsStr::new(STORAGE_FORMAT_FILE),
|
||||
base,
|
||||
root,
|
||||
)
|
||||
.map_err(|error| {
|
||||
if error.kind() == std::io::ErrorKind::NotFound {
|
||||
IoError::other(DiskError::ConditionalDeleteConflict)
|
||||
} else {
|
||||
error
|
||||
}
|
||||
})?;
|
||||
if target.len() as u64 != expected_length || raw_sha(&target) != expected_sha256 {
|
||||
return Err(IoError::other(DiskError::ConditionalDeleteConflict));
|
||||
}
|
||||
verify_synced_target(&target_dir, base, root, expected_length, expected_sha256)?;
|
||||
} else {
|
||||
os::read_and_sync_multipart_target_metadata(
|
||||
&target_dir,
|
||||
@@ -2779,3 +2777,29 @@ impl LocalDisk {
|
||||
.map_err(DiskError::from)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod target_receipt_tests {
|
||||
use super::*;
|
||||
#[test]
|
||||
fn synced_target_uses_prepared_receipt_for_large_history() {
|
||||
let tmp = tempfile::tempdir().expect("temporary publication root");
|
||||
let base = tmp.path().canonicalize().expect("absolute root");
|
||||
let dir = base.join("target");
|
||||
std::fs::create_dir(&dir).expect("target directory");
|
||||
let root = os::PublicationRoot::new(&base).expect("publication root");
|
||||
let block = [0u8; 64 * 1024];
|
||||
let length = 16 * 1024 * 1024 + 1;
|
||||
std::fs::File::create(dir.join(STORAGE_FORMAT_FILE))
|
||||
.expect("target head")
|
||||
.set_len(length)
|
||||
.expect("sparse head");
|
||||
let mut hash = Sha256::new();
|
||||
for _ in 0..256 {
|
||||
hash.update(block);
|
||||
}
|
||||
hash.update([0]);
|
||||
verify_synced_target(&dir, &base, &root, length, hash.finalize().into()).expect("matching receipt");
|
||||
assert!(verify_synced_target(&dir, &base, &root, length - 1, [0; 32]).is_err());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1218,6 +1218,8 @@ impl std::fmt::Debug for GetObjectSink {
|
||||
|
||||
#[derive(Default, Clone)]
|
||||
pub struct ObjectOptions {
|
||||
/// Part copies retain source locks and cannot queue for capped upload staging.
|
||||
pub multipart_copy_source_locked: bool,
|
||||
#[doc(hidden)]
|
||||
pub delete_recovery_work_budget: Option<Arc<DeleteRecoveryWorkBudget>>,
|
||||
/// Caller-provided destination for a GET body: when set, the legacy
|
||||
|
||||
@@ -604,10 +604,10 @@ pub(crate) async fn initialize_local_disk_maps(
|
||||
|
||||
let disk = match new_disk(ep, opt).await {
|
||||
Ok(disk) => disk,
|
||||
Err(crate::disk::error::DiskError::FileAccessDenied) => {
|
||||
// LocalDisk already logged the filesystem cause and repair
|
||||
// hint. Keep this slot unavailable so ECStore initialization
|
||||
// can apply its existing fatal-error and quorum rules.
|
||||
Err(crate::disk::error::DiskError::FileAccessDenied | crate::disk::error::DiskError::VolumeNotFound) => {
|
||||
// Preserve the configured slot when a local volume is unavailable.
|
||||
// LocalDisk logs the cause; format loading retains authority
|
||||
// over fatal errors and quorum. Never create a missing root.
|
||||
global_local_disk_map.insert(ep.to_string(), None);
|
||||
continue;
|
||||
}
|
||||
@@ -888,6 +888,95 @@ mod tests {
|
||||
disk_map.keys().collect::<Vec<_>>()
|
||||
);
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn missing_local_volume_preserves_offline_slot_without_creating_root() {
|
||||
use crate::disk::{DiskOption, error::DiskError};
|
||||
use crate::layout::endpoints::{EndpointServerPools, PoolEndpoints};
|
||||
|
||||
for drive_count in [2, 3, 4] {
|
||||
let directory = tempfile::tempdir().expect("create volume parent");
|
||||
let mut endpoints = Vec::new();
|
||||
for index in 0..drive_count {
|
||||
let path = directory.path().join(format!("disk{index}"));
|
||||
if index != 0 {
|
||||
std::fs::create_dir(&path).expect("create surviving volume");
|
||||
}
|
||||
let mut endpoint = Endpoint::try_from(path.to_str().expect("UTF-8 path")).expect("local endpoint");
|
||||
endpoint.set_pool_index(0);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(index);
|
||||
endpoints.push(endpoint);
|
||||
}
|
||||
let missing = endpoints[0].to_string();
|
||||
let pools = EndpointServerPools(vec![PoolEndpoints {
|
||||
legacy: false,
|
||||
set_count: 1,
|
||||
drives_per_set: drive_count,
|
||||
endpoints: endpoints.into(),
|
||||
cmd_line: "missing-volume".into(),
|
||||
platform: "test".into(),
|
||||
}]);
|
||||
let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
|
||||
let options = DiskOption {
|
||||
cleanup: true,
|
||||
health_check: false,
|
||||
};
|
||||
super::initialize_local_disk_maps(&ctx, pools.clone(), &options)
|
||||
.await
|
||||
.expect("missing volume should remain offline for quorum evaluation");
|
||||
let map = ctx.local_disk_map();
|
||||
let map = map.read().await;
|
||||
assert_eq!(map.len(), drive_count);
|
||||
assert!(matches!(map.get(&missing), Some(None)));
|
||||
assert_eq!(map.values().flatten().count(), drive_count - 1);
|
||||
drop(map);
|
||||
let drives = ctx.local_disk_set_drives();
|
||||
let drives = drives.read().await;
|
||||
assert_eq!(drives[0][0].len(), drive_count);
|
||||
assert!(drives[0][0][0].is_none());
|
||||
assert!(drives[0][0][1..].iter().all(Option::is_some));
|
||||
drop(drives);
|
||||
|
||||
let (disks, errors) = crate::store::init_format::init_disks(&ctx, &pools.0[0].endpoints, &options).await;
|
||||
assert!(disks[0].is_none());
|
||||
assert_eq!(errors[0], Some(DiskError::VolumeNotFound));
|
||||
assert!(disks[1..].iter().all(Option::is_some));
|
||||
assert!(errors[1..].iter().all(Option::is_none));
|
||||
assert!(!directory.path().join("disk0").exists(), "startup must not create a missing mount root");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn invalid_local_volume_still_fails_registry_initialization() {
|
||||
use crate::disk::DiskOption;
|
||||
use crate::layout::endpoints::{EndpointServerPools, PoolEndpoints};
|
||||
|
||||
let directory = tempfile::tempdir().expect("create volume parent");
|
||||
let path = directory.path().join("regular-file");
|
||||
std::fs::write(&path, b"preserve existing file").expect("create invalid volume");
|
||||
let mut endpoint = Endpoint::try_from(path.to_str().expect("UTF-8 path")).expect("local endpoint");
|
||||
endpoint.set_pool_index(0);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(0);
|
||||
let pools = EndpointServerPools(vec![PoolEndpoints {
|
||||
legacy: false,
|
||||
set_count: 1,
|
||||
drives_per_set: 1,
|
||||
endpoints: vec![endpoint].into(),
|
||||
cmd_line: "invalid-volume".into(),
|
||||
platform: "test".into(),
|
||||
}]);
|
||||
let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
|
||||
let options = DiskOption {
|
||||
cleanup: true,
|
||||
health_check: false,
|
||||
};
|
||||
super::initialize_local_disk_maps(&ctx, pools, &options)
|
||||
.await
|
||||
.expect_err("a non-directory volume must retain its initialization error");
|
||||
assert_eq!(std::fs::read(&path).expect("read existing file"), b"preserve existing file");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_invalid_registry_pool_index_has_no_authority_or_registry_effect() {
|
||||
use crate::disk::DiskOption;
|
||||
|
||||
@@ -15,8 +15,6 @@
|
||||
use crate::diagnostics::admin_server_info::get_local_server_property;
|
||||
use crate::runtime::sources as runtime_sources;
|
||||
use crate::storage_api_contracts::admin::StorageAdminApi;
|
||||
#[cfg(test)]
|
||||
use chrono::Utc;
|
||||
use jiff::Timestamp;
|
||||
use rustfs_heal_contracts::heal_channel::DriveState;
|
||||
use rustfs_io_metrics::internode_metrics::global_internode_metrics;
|
||||
@@ -600,10 +598,6 @@ mod test {
|
||||
use serial_test::serial;
|
||||
use std::time::Duration;
|
||||
|
||||
fn chrono_to_jiff_timestamp(timestamp: chrono::DateTime<Utc>) -> jiff::Timestamp {
|
||||
jiff::Timestamp::try_from(std::time::SystemTime::from(timestamp)).expect("test timestamp should fit in jiff")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tes_types() {
|
||||
let t = MetricType::ALL;
|
||||
@@ -685,10 +679,10 @@ mod test {
|
||||
|
||||
#[test]
|
||||
fn scanner_metrics_mapping_preserves_partial_source_status() {
|
||||
let current_started = Utc::now() - chrono::Duration::seconds(5);
|
||||
let current_started = Timestamp::now() - jiff::SignedDuration::from_secs(5);
|
||||
let scanner = to_madmin_scanner_metrics(rustfs_scanner_metrics::metrics::ScannerMetricsReport {
|
||||
current_cycle_active: true,
|
||||
current_started: chrono_to_jiff_timestamp(current_started),
|
||||
current_started,
|
||||
last_cycle_partial_source: "usage".to_string(),
|
||||
last_cycle_partial_source_code: 1,
|
||||
cycle_timeout_total: 3,
|
||||
@@ -703,7 +697,7 @@ mod test {
|
||||
});
|
||||
|
||||
assert_eq!(scanner.current_cycle_active, Some(true));
|
||||
assert_eq!(scanner.current_started, chrono_to_jiff_timestamp(current_started));
|
||||
assert_eq!(scanner.current_started, current_started);
|
||||
assert_eq!(scanner.last_cycle_partial_source, "usage");
|
||||
assert_eq!(scanner.last_cycle_partial_source_code, 1);
|
||||
assert_eq!(scanner.cycle_timeout_total, 3);
|
||||
@@ -723,8 +717,8 @@ mod test {
|
||||
async fn collect_local_metrics_preserves_scanner_cycle_started_time() {
|
||||
let previous_init_time = *rustfs_scanner_metrics::GLOBAL_INIT_TIME.read().await;
|
||||
let previous_cycle = global_metrics().get_cycle().await;
|
||||
let init_time = Utc::now() - chrono::Duration::hours(1);
|
||||
let cycle_started = Utc::now() - chrono::Duration::seconds(5);
|
||||
let init_time = Timestamp::now() - jiff::SignedDuration::from_hours(1);
|
||||
let cycle_started = Timestamp::now() - jiff::SignedDuration::from_secs(5);
|
||||
*rustfs_scanner_metrics::GLOBAL_INIT_TIME.write().await = Some(init_time);
|
||||
let cycle = CurrentCycle {
|
||||
current: 0,
|
||||
@@ -748,7 +742,7 @@ mod test {
|
||||
aggregated.merge(decoded);
|
||||
let scanner = aggregated.aggregated.scanner.expect("scanner metrics");
|
||||
assert_eq!(scanner.current_cycle_active, Some(true));
|
||||
assert_eq!(scanner.current_started, chrono_to_jiff_timestamp(cycle_started));
|
||||
assert_eq!(scanner.current_started, cycle_started);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -3662,6 +3662,9 @@ impl TierConfigMgr {
|
||||
}
|
||||
}
|
||||
}
|
||||
if !rustfs_data_usage::is_scannable_tier_name(&tier_config.name) {
|
||||
return Err(tier_invalid_config("Remote tier name cannot be represented in lifecycle accounting"));
|
||||
}
|
||||
self.ensure_generation_is_idle(&tier_config.name)?;
|
||||
let tier_name = tier_config.name.clone();
|
||||
if tier_name != tier_name.to_uppercase() {
|
||||
|
||||
@@ -335,6 +335,21 @@ pub(crate) struct S3CompatibleWarmBackendParams<'a> {
|
||||
pub validate_endpoint: fn(&url::Url) -> Result<(), rustfs_utils::egress::OutboundUrlError>,
|
||||
}
|
||||
|
||||
/// A bucket can become a host prefix or the first path segment. Keep legacy
|
||||
/// uppercase and underscore names, but never permit URL syntax in that field.
|
||||
pub(crate) fn validate_tier_bucket(bucket: &str) -> std::io::Result<()> {
|
||||
if bucket.is_empty() {
|
||||
return Err(std::io::Error::other("no bucket name was provided"));
|
||||
}
|
||||
if bucket
|
||||
.split('.')
|
||||
.any(|label| label.is_empty() || !label.bytes().all(|b| b.is_ascii_alphanumeric() || matches!(b, b'-' | b'_')))
|
||||
{
|
||||
return Err(std::io::Error::other("invalid tier bucket name"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Return the authority format accepted by `TransitionClient::new` while
|
||||
/// retaining an explicitly configured port. `url::Url::host_str()` omits the
|
||||
/// brackets needed when an IPv6 literal is combined with a port.
|
||||
@@ -342,6 +357,9 @@ pub(crate) fn endpoint_authority(url: &url::Url) -> Result<String, std::io::Erro
|
||||
let host = url
|
||||
.host_str()
|
||||
.ok_or_else(|| std::io::Error::other("Invalid endpoint URL: missing host"))?;
|
||||
if !matches!(url.scheme(), "https" | "http") {
|
||||
return Err(std::io::Error::other("tier endpoint must use HTTP or HTTPS"));
|
||||
}
|
||||
let port = url.port().unwrap_or(if url.scheme() == "https" { 443 } else { 80 });
|
||||
if host.starts_with('[') && host.ends_with(']') {
|
||||
Ok(format!("{host}:{port}"))
|
||||
@@ -385,9 +403,7 @@ pub(crate) async fn new_s3_compatible_warm_backend(
|
||||
return Err(std::io::Error::other("both access and secret keys are required"));
|
||||
}
|
||||
|
||||
if params.bucket.is_empty() {
|
||||
return Err(std::io::Error::other("no bucket name was provided"));
|
||||
}
|
||||
validate_tier_bucket(params.bucket)?;
|
||||
|
||||
let u = match url::Url::parse(params.endpoint) {
|
||||
Ok(u) => u,
|
||||
@@ -2092,6 +2108,20 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier_bucket_is_a_name_not_a_url_component() {
|
||||
assert!(validate_tier_bucket("archive.EU_1").is_ok());
|
||||
for bucket in ["a:b", "a/b", "a?b", "a#b", "a@b", "a%2Fb", "a\\b", "a..b"] {
|
||||
assert!(validate_tier_bucket(bucket).is_err());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn endpoint_authority_requires_http_transport() {
|
||||
let url = url::Url::parse("ftp://archive.example").expect("valid URL");
|
||||
assert!(endpoint_authority(&url).is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn s3_compatible_backend_strips_only_a_trailing_prefix_separator() {
|
||||
let mut params = s3_compatible_params("http://tier.example.com:9000");
|
||||
|
||||
@@ -24,7 +24,7 @@ use crate::services::tier::{
|
||||
tier_config::TierS3,
|
||||
warm_backend::{
|
||||
TransitionCandidateIdentity, TransitionCandidateProbe, TransitionCandidateReconciler, WarmBackend, WarmBackendGetOpts,
|
||||
build_transition_put_options, endpoint_authority, transition_client_timeouts_from_env,
|
||||
build_transition_put_options, endpoint_authority, transition_client_timeouts_from_env, validate_tier_bucket,
|
||||
},
|
||||
};
|
||||
use http::HeaderMap;
|
||||
@@ -119,9 +119,8 @@ impl WarmBackendS3 {
|
||||
return Err(std::io::Error::other(
|
||||
"AWS Role cannot be activated with static credentials or the web identity token file",
|
||||
));
|
||||
} else if conf.bucket.is_empty() {
|
||||
return Err(std::io::Error::other("no bucket name was provided"));
|
||||
}
|
||||
validate_tier_bucket(&conf.bucket)?;
|
||||
|
||||
let creds = if has_access_key && has_secret_key {
|
||||
//creds = Credentials::new_static_v4(conf.access_key, conf.secret_key, "");
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -17333,8 +17333,12 @@ mod tests {
|
||||
source.metadata_only = true;
|
||||
source.etag = Some("metadata-copy-etag".to_string());
|
||||
|
||||
let dst_opts = ObjectOptions {
|
||||
object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(ObjectLockConfigState::ConfirmedAbsent))),
|
||||
..opts.clone()
|
||||
};
|
||||
let copied = set_disks
|
||||
.copy_object(bucket, object, bucket, object, &mut source, &opts, &opts)
|
||||
.copy_object(bucket, object, bucket, object, &mut source, &opts, &dst_opts)
|
||||
.await
|
||||
.expect("metadata self-copy should succeed");
|
||||
assert_eq!(copied.user_defined.get("x-amz-meta-copy-check").map(String::as_str), Some("present"));
|
||||
|
||||
@@ -2786,14 +2786,18 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
// it is reacquired for the short final rename, so Complete/Abort are
|
||||
// not blocked behind a slow body upload.
|
||||
let _capped_staging_guard = if multipart_size_limit.is_some() {
|
||||
let semaphore = capped_multipart_staging_semaphore(&upload_id_path);
|
||||
let permit = if opts.multipart_copy_source_locked {
|
||||
semaphore.try_acquire_owned().map_err(|_| Error::SlowDown)?
|
||||
} else {
|
||||
semaphore
|
||||
.acquire_owned()
|
||||
.await
|
||||
.map_err(|_| Error::other("capped multipart staging semaphore closed"))?
|
||||
};
|
||||
Some(CappedMultipartStagingGuard {
|
||||
upload_id_path: upload_id_path.clone(),
|
||||
permit: Some(
|
||||
capped_multipart_staging_semaphore(&upload_id_path)
|
||||
.acquire_owned()
|
||||
.await
|
||||
.map_err(|_| Error::other("capped multipart staging semaphore closed"))?,
|
||||
),
|
||||
permit: Some(permit),
|
||||
})
|
||||
} else {
|
||||
None
|
||||
@@ -8926,6 +8930,18 @@ mod tests {
|
||||
});
|
||||
let input = tokio_util::io::StreamReader::new(stream);
|
||||
let mut reader = PutObjReader::new(HashReader::from_stream(input, 512, 512, None, None, false).expect("part reader"));
|
||||
let copy_options = ObjectOptions {
|
||||
multipart_copy_source_locked: true,
|
||||
..Default::default()
|
||||
};
|
||||
let refused = tokio::time::timeout(
|
||||
Duration::from_secs(10),
|
||||
set_disks.put_object_part(bucket, object, &upload.upload_id, 1, &mut reader, ©_options),
|
||||
)
|
||||
.await
|
||||
.expect("source-locked copy must not queue");
|
||||
assert!(matches!(refused, Err(Error::SlowDown)));
|
||||
assert_eq!(polls.load(Ordering::Relaxed), 0);
|
||||
let task = tokio::spawn(async move {
|
||||
set_disks
|
||||
.put_object_part(bucket, object, &upload.upload_id, 1, &mut reader, &ObjectOptions::default())
|
||||
|
||||
@@ -2450,6 +2450,14 @@ fn is_restore_control_metadata(key: &str) -> bool {
|
||||
.is_some_and(|remainder| remainder.is_empty())
|
||||
}
|
||||
|
||||
fn copy_overwrite_version_id(opts: &ObjectOptions, source_version: Option<Uuid>) -> Option<String> {
|
||||
opts.overwrites_existing_version().then(|| {
|
||||
opts.version_id
|
||||
.clone()
|
||||
.unwrap_or_else(|| source_version.unwrap_or_default().to_string())
|
||||
})
|
||||
}
|
||||
|
||||
fn restore_metadata_update_preserves_protected_metadata(
|
||||
existing: &HashMap<String, String>,
|
||||
replacement: &HashMap<String, String>,
|
||||
@@ -2471,6 +2479,16 @@ fn restore_metadata_update_preserves_protected_metadata(
|
||||
mod restore_metadata_update_tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn copy_overwrite_selects_protected_null_version() {
|
||||
assert_eq!(copy_overwrite_version_id(&ObjectOptions::default(), None), Some(Uuid::nil().to_string()));
|
||||
let opts = ObjectOptions {
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
};
|
||||
assert_eq!(copy_overwrite_version_id(&opts, None), None);
|
||||
}
|
||||
|
||||
/// The restore keys in pre-`metadata_keys` xl.meta are exactly the ones
|
||||
/// a restore metadata update may change; every other persisted key is
|
||||
/// protected (backlog#1735 A3b).
|
||||
@@ -9988,20 +10006,20 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
.keys()
|
||||
.any(|key| key.eq_ignore_ascii_case(metadata_keys::RESTORE))
|
||||
&& restore_metadata_update_preserves_protected_metadata(&fi.metadata, src_info.user_defined.as_ref());
|
||||
if let Some(dst_version_id) = dst_opts.version_id.as_deref()
|
||||
if let Some(dst_version_id) = copy_overwrite_version_id(dst_opts, src_info.version_id)
|
||||
&& !is_meta_bucketname(dst_bucket)
|
||||
&& !restore_metadata_update
|
||||
{
|
||||
let object_lock_config = dst_opts
|
||||
.object_lock_config_snapshot
|
||||
.as_deref()
|
||||
.ok_or_else(|| Error::other("explicit-version copy is missing its Object Lock configuration snapshot"))?;
|
||||
.ok_or_else(|| Error::other("destructive copy is missing its Object Lock configuration snapshot"))?;
|
||||
let current = self
|
||||
.get_object_info(
|
||||
dst_bucket,
|
||||
dst_object,
|
||||
&ObjectOptions {
|
||||
version_id: Some(dst_version_id.to_string()),
|
||||
version_id: Some(dst_version_id),
|
||||
no_lock: true,
|
||||
metadata_cache_safe: false,
|
||||
versioned: dst_opts.versioned,
|
||||
@@ -17164,7 +17182,12 @@ mod metadata_mutation_generation_tests {
|
||||
copy_object,
|
||||
&mut copy_info,
|
||||
&ObjectOptions::default(),
|
||||
&ObjectOptions::default(),
|
||||
&ObjectOptions {
|
||||
object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(
|
||||
ObjectLockConfigState::ConfirmedAbsent,
|
||||
))),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("metadata COPY should succeed");
|
||||
@@ -17342,7 +17365,11 @@ mod metadata_mutation_generation_tests {
|
||||
let metadata = Arc::make_mut(&mut source.user_defined);
|
||||
metadata.insert(AMZ_OBJECT_TAGGING.to_string(), "new=value".to_string());
|
||||
metadata.insert("expires".to_string(), expires.format(&Rfc3339).expect("expiry"));
|
||||
set.copy_object(bucket, "object", bucket, "object", &mut source, &opts, &opts)
|
||||
let dst_opts = ObjectOptions {
|
||||
object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(ObjectLockConfigState::ConfirmedAbsent))),
|
||||
..Default::default()
|
||||
};
|
||||
set.copy_object(bucket, "object", bucket, "object", &mut source, &opts, &dst_opts)
|
||||
.await
|
||||
.expect("explicit metadata wins over externalized fields");
|
||||
let mut copied = set.get_object_info(bucket, "object", &opts).await.expect("copied metadata");
|
||||
@@ -17353,7 +17380,7 @@ mod metadata_mutation_generation_tests {
|
||||
copied.metadata_only = true;
|
||||
copied.user_tags = Arc::new(String::new());
|
||||
copied.expires = None;
|
||||
set.copy_object(bucket, "object", bucket, "object", &mut copied, &opts, &opts)
|
||||
set.copy_object(bucket, "object", bucket, "object", &mut copied, &opts, &dst_opts)
|
||||
.await
|
||||
.expect("explicit empty replacement");
|
||||
let cleared = set.get_object_info(bucket, "object", &opts).await.expect("cleared metadata");
|
||||
@@ -17370,6 +17397,10 @@ mod metadata_mutation_generation_tests {
|
||||
}
|
||||
|
||||
let valid_object = "valid-sidecar";
|
||||
let dst_opts = ObjectOptions {
|
||||
object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(ObjectLockConfigState::ConfirmedAbsent))),
|
||||
..Default::default()
|
||||
};
|
||||
let (mut valid_source, _) = put_and_prime(&set_disks, bucket, valid_object, b"valid sidecar body").await;
|
||||
persist_part_checksum_sidecar(&set_disks, bucket, valid_object, r#"[[1,[["CRC32C","AAAAAA=="]]]]"#).await;
|
||||
valid_source.metadata_only = true;
|
||||
@@ -17382,7 +17413,7 @@ mod metadata_mutation_generation_tests {
|
||||
valid_object,
|
||||
&mut valid_source,
|
||||
&ObjectOptions::default(),
|
||||
&ObjectOptions::default(),
|
||||
&dst_opts,
|
||||
)
|
||||
.await
|
||||
.expect("metadata-only copy should preserve a valid checksum sidecar");
|
||||
@@ -17455,7 +17486,7 @@ mod metadata_mutation_generation_tests {
|
||||
conflicting_object,
|
||||
&mut conflicting_source,
|
||||
&ObjectOptions::default(),
|
||||
&ObjectOptions::default(),
|
||||
&dst_opts,
|
||||
)
|
||||
.await
|
||||
.expect_err("metadata-only copy must reject conflicting checksum aliases");
|
||||
@@ -25501,6 +25532,9 @@ mod put_object_tmp_cleanup_tests {
|
||||
&ObjectOptions {
|
||||
no_lock: true,
|
||||
namespace_lock_fence: Some(NamespaceLockFence::lost_for_test()),
|
||||
object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(
|
||||
ObjectLockConfigState::ConfirmedAbsent,
|
||||
))),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
|
||||
@@ -11043,7 +11043,12 @@ mod dangling_read_tests {
|
||||
object,
|
||||
&mut source,
|
||||
&ObjectOptions::default(),
|
||||
&ObjectOptions::default(),
|
||||
&ObjectOptions {
|
||||
object_lock_config_snapshot: Some(Arc::new(crate::object_api::ObjectLockConfigSnapshot::new(
|
||||
crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent,
|
||||
))),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("a CopyObject source without a version id must resolve like a latest GET");
|
||||
|
||||
@@ -205,6 +205,12 @@ fn heal_format_fence_lost_error() -> Error {
|
||||
}
|
||||
|
||||
impl ECStore {
|
||||
/// Whether this exact storage instance requires an incarnation-bound write
|
||||
/// for the object currently owned by a bucket-heal scope.
|
||||
pub fn object_requires_incarnation(&self, bucket: &str, object: &str) -> bool {
|
||||
bucket_heal_scope_for_object(bucket, object).is_some_and(|scope| std::ptr::eq(Arc::as_ptr(&scope.store), self))
|
||||
}
|
||||
|
||||
pub async fn purge_delete_marker_with_proof(
|
||||
self: &Arc<Self>,
|
||||
bucket: &str,
|
||||
|
||||
@@ -752,18 +752,60 @@ impl ECStore {
|
||||
Ok(ec)
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self, rx))]
|
||||
pub async fn init(self: &Arc<Self>, rx: CancellationToken) -> Result<()> {
|
||||
runtime_sources::ensure_boot_time().await;
|
||||
|
||||
async fn init_pool_metadata_for_startup(self: &Arc<Self>) -> Result<(PoolMeta, PoolMetaReplicaState)> {
|
||||
let should_persist_pool_meta = self
|
||||
.pools
|
||||
.first()
|
||||
.is_some_and(|pool| pool_first_endpoint_is_local(&pool.endpoints));
|
||||
// Lock order is save gate -> distributed metadata writer, matching
|
||||
// Lock order is save gate -> distributed metadata fence, matching
|
||||
// runtime mutations. RPC locks are available before ECStore publication
|
||||
// and do not depend on the later fleet proof.
|
||||
let mut startup_write_state = self.pool_meta_save_gate.write().await;
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
let startup_attempt = uuid::Uuid::new_v4();
|
||||
// A committed snapshot needs only a shared namespace fence. A writer
|
||||
// transition releases this reader and re-reads every observation below
|
||||
// under the existing exclusive fence before applying side effects.
|
||||
if !startup_write_state.bootstrap_identity_proven() && startup_write_state.attested_pool_indices().is_empty() {
|
||||
let pool = self
|
||||
.pools
|
||||
.first()
|
||||
.ok_or_else(|| Error::other("startup has no storage pools"))?;
|
||||
let lock = pool.new_ns_lock(crate::disk::RUSTFS_META_BUCKET, "pool.bin").await?;
|
||||
let reader = lock.get_read_lock(crate::set_disk::get_lock_acquire_timeout()).await?;
|
||||
let load = load_pool_meta_for_startup(self.pools.clone(), &mut startup_write_state);
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
let load = crate::core::pools::startup_cas_test_scope(startup_attempt, "load", &self.pools, load);
|
||||
match load.await {
|
||||
Ok((meta, replicas)) => {
|
||||
let topology_update = meta.validate(self.pools.clone())?;
|
||||
let requires_writer = should_persist_pool_meta
|
||||
&& (topology_update
|
||||
|| (replicas.needs_repair && replicas.repair_write_safe)
|
||||
|| startup_write_state.identity_requires_repair());
|
||||
if !topology_update && !requires_writer {
|
||||
startup_write_state.ensure_startup_identity_initialized()?;
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
crate::core::pools::startup_cas_test_observe(serde_json::json!({
|
||||
"kind": "startup-classifier", "attempt": startup_attempt,
|
||||
"elected_writer": should_persist_pool_meta,
|
||||
"needs_repair": replicas.needs_repair,
|
||||
"repair_write_safe": replicas.repair_write_safe,
|
||||
"topology_update": topology_update,
|
||||
}));
|
||||
self.install_startup_pool_metadata(&meta, Some(&reader)).await?;
|
||||
return Ok((meta, replicas));
|
||||
}
|
||||
}
|
||||
Err(err)
|
||||
if should_persist_pool_meta
|
||||
&& err
|
||||
.pool_metadata_failure()
|
||||
.is_some_and(|failure| failure.kind == crate::error::PoolMetadataFailure::ReadUnavailable) => {}
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
drop(reader);
|
||||
}
|
||||
let startup_pool_meta_guard = if should_persist_pool_meta {
|
||||
let pool = self
|
||||
.pools
|
||||
@@ -824,7 +866,6 @@ impl ECStore {
|
||||
"repair_write_safe": pool_meta_replica_state.repair_write_safe,
|
||||
"topology_update": update,
|
||||
}));
|
||||
let endpoints = runtime_sources::endpoint_pools_or_default();
|
||||
|
||||
let mut installed_pool_meta = if update {
|
||||
PoolMeta::new(&self.pools, &meta)?
|
||||
@@ -881,19 +922,34 @@ impl ECStore {
|
||||
installed_pool_meta = persist.await?;
|
||||
}
|
||||
|
||||
if startup_pool_meta_guard
|
||||
.as_ref()
|
||||
.is_some_and(|guard| guard.is_lock_lost() || guard.is_released())
|
||||
{
|
||||
return Err(Error::other("startup pool metadata writer fence was lost before publication"));
|
||||
}
|
||||
{
|
||||
let mut pool_meta = self.pool_meta.write().await;
|
||||
*pool_meta = installed_pool_meta.clone();
|
||||
}
|
||||
self.install_startup_pool_metadata(&installed_pool_meta, startup_pool_meta_guard.as_deref())
|
||||
.await?;
|
||||
drop(startup_pool_meta_guard);
|
||||
drop(startup_write_state);
|
||||
|
||||
Ok((installed_pool_meta, pool_meta_replica_state))
|
||||
}
|
||||
|
||||
async fn install_startup_pool_metadata(
|
||||
&self,
|
||||
metadata: &PoolMeta,
|
||||
fence: Option<&rustfs_lock::NamespaceLockGuard>,
|
||||
) -> Result<()> {
|
||||
let mut pool_meta = self.pool_meta.write().await;
|
||||
if fence.is_some_and(|guard| guard.is_lock_lost() || guard.is_released()) {
|
||||
return Err(Error::other("startup pool metadata fence was lost before publication"));
|
||||
}
|
||||
*pool_meta = metadata.clone();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self, rx))]
|
||||
pub async fn init(self: &Arc<Self>, rx: CancellationToken) -> Result<()> {
|
||||
runtime_sources::ensure_boot_time().await;
|
||||
|
||||
let (installed_pool_meta, pool_meta_replica_state) = self.init_pool_metadata_for_startup().await?;
|
||||
let endpoints = runtime_sources::endpoint_pools_or_default();
|
||||
|
||||
resolve_store_init_stage_result(self.load_rebalance_meta().await, "load_rebalance_meta")?;
|
||||
let rebalance_resume_required = {
|
||||
let rebalance_meta = self.rebalance_meta.read().await;
|
||||
@@ -3731,6 +3787,89 @@ mod tests {
|
||||
(instance_ctx, store, shutdown)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn committed_startup_pool_metadata_uses_shared_fence_without_disk_writes() {
|
||||
use crate::disk::{FORMAT_CONFIG_FILE, RUSTFS_META_BUCKET};
|
||||
use crate::storage_api_contracts::namespace::NamespaceLocking;
|
||||
|
||||
for reopen in [false, true] {
|
||||
let directory = tempfile::tempdir().expect("create committed metadata deployment");
|
||||
let (_, seeded, seed_shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(directory.path(), "startup-read-fence", &[4])).await;
|
||||
let (store, shutdown) = if reopen {
|
||||
seed_shutdown.cancel();
|
||||
drop(seeded);
|
||||
let (_, reopened, shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(directory.path(), "startup-read-fence", &[4])).await;
|
||||
(reopened, shutdown)
|
||||
} else {
|
||||
(seeded, seed_shutdown)
|
||||
};
|
||||
let disks = store.pools[0].disk_set[0].disk_inventory().await;
|
||||
let snapshot = || {
|
||||
let mut files = std::collections::BTreeMap::new();
|
||||
for (index, slot) in disks.iter().enumerate() {
|
||||
let disk = slot.as_ref().expect("committed drive handle");
|
||||
for object in [POOL_META_NAME, POOL_META_IDENTITY_NAME, FORMAT_CONFIG_FILE] {
|
||||
let root = disk.path().join(RUSTFS_META_BUCKET).join(object);
|
||||
assert!(root.exists(), "committed authority must be present: {root:?}");
|
||||
let mut pending = vec![root];
|
||||
while let Some(path) = pending.pop() {
|
||||
let metadata = std::fs::symlink_metadata(&path).expect("inspect authority file type");
|
||||
assert!(!metadata.file_type().is_symlink(), "authority fixture must not follow symlinks");
|
||||
if metadata.is_dir() {
|
||||
for entry in std::fs::read_dir(&path).expect("inspect authority object directory") {
|
||||
pending.push(entry.expect("authority directory entry").path());
|
||||
}
|
||||
} else {
|
||||
assert!(metadata.is_file(), "authority fixture must contain regular files");
|
||||
let contents = std::fs::read(&path).expect("read physical authority bytes");
|
||||
let relative = path
|
||||
.strip_prefix(disk.path())
|
||||
.expect("authority stays under fixture drive")
|
||||
.to_path_buf();
|
||||
files.insert((index, relative), contents);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
files
|
||||
};
|
||||
let before = snapshot();
|
||||
let lock = store.pools[0]
|
||||
.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME)
|
||||
.await
|
||||
.expect("metadata namespace lock");
|
||||
let mut reader = lock
|
||||
.get_read_lock(Duration::from_secs(1))
|
||||
.await
|
||||
.expect("hold a competing metadata reader");
|
||||
let (metadata, replicas) = tokio::time::timeout(Duration::from_secs(2), store.init_pool_metadata_for_startup())
|
||||
.await
|
||||
.expect("committed startup must not wait for an exclusive lock")
|
||||
.expect("committed startup metadata is readable under a shared fence");
|
||||
assert_eq!(metadata.pools[0].cmd_line, "startup-read-fence-pool-0");
|
||||
assert!(!replicas.needs_repair);
|
||||
assert!(!reader.is_lock_lost());
|
||||
let after = snapshot();
|
||||
assert_eq!(
|
||||
after, before,
|
||||
"shared startup observation cannot mutate metadata, identity, or format authority"
|
||||
);
|
||||
assert!(reader.release(), "release the snapshot's shared fence");
|
||||
let mut invalid = metadata.clone();
|
||||
invalid.pools[0].cmd_line = "must-not-be-published".to_string();
|
||||
store
|
||||
.install_startup_pool_metadata(&invalid, Some(&reader))
|
||||
.await
|
||||
.expect_err("a released fence cannot publish startup metadata");
|
||||
assert_eq!(store.pool_meta.read().await.pools[0].cmd_line, metadata.pools[0].cmd_line);
|
||||
drop(reader);
|
||||
shutdown.cancel();
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn interrupted_format_bootstrap_recovers_two_pools_through_store_init() {
|
||||
|
||||
@@ -169,36 +169,34 @@ pub(crate) async fn connect_load_init_formats_with_instance_ctx(
|
||||
{
|
||||
return Err(Error::CorruptedFormat);
|
||||
}
|
||||
if matches!(format_quorum, Some(Err(Error::ErasureReadQuorum)))
|
||||
&& let Some((format, members)) = select_existing_format_read_quorum(&formats, &errs, set_drive_count)
|
||||
{
|
||||
retain_format_quorum_members(
|
||||
instance_ctx,
|
||||
disks,
|
||||
&format,
|
||||
&members,
|
||||
Some(&unformatted_bootstrap_routes),
|
||||
set_drive_count,
|
||||
)
|
||||
.await?;
|
||||
// Loading an existing format grants no authority to initialize pool
|
||||
// metadata or migrate formats. The persisted storage classes and each
|
||||
// object's EC metadata still determine service and operation quorums.
|
||||
return Ok(LoadedFormat {
|
||||
format,
|
||||
pool_meta_bootstrap_authority: PoolMetaBootstrapAuthority::None,
|
||||
});
|
||||
}
|
||||
let resumable_partial_migration = formats_present > 0
|
||||
let native_read_candidate = if matches!(format_quorum, Some(Err(Error::ErasureReadQuorum))) {
|
||||
select_existing_format_read_quorum(&formats, &errs, set_drive_count)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let partial_native_with_blanks = formats_present > 0
|
||||
&& errs.iter().any(|error| matches!(error, Some(DiskError::UnformattedDisk)))
|
||||
&& format_quorum.as_ref().is_some_and(Result::is_err)
|
||||
&& format_quorum.as_ref().is_some_and(Result::is_err);
|
||||
let resumable_partial_migration = partial_native_with_blanks
|
||||
&& formats.iter().zip(&errs).all(|(format, error)| {
|
||||
format.is_some() || matches!(error, Some(DiskError::UnformattedDisk | DiskError::DiskNotFound))
|
||||
});
|
||||
|
||||
if first_disk && (all_unformatted || resumable_partial_migration) {
|
||||
let allow_migration = first_disk && (all_unformatted || resumable_partial_migration);
|
||||
if partial_native_with_blanks && !allow_migration && native_read_candidate.is_none() {
|
||||
return Err(Error::ErasureReadQuorum);
|
||||
}
|
||||
if allow_migration || (partial_native_with_blanks && native_read_candidate.is_some()) {
|
||||
info!(all_unformatted, resumable_partial_migration, "checking for a legacy storage format");
|
||||
match try_migrate_format(disks, &formats, set_count, set_drive_count).await {
|
||||
let mode = match native_read_candidate.as_ref() {
|
||||
Some((reference, _)) => LegacyFormatMode::NativeRead {
|
||||
reference,
|
||||
native_errors: &errs,
|
||||
allow_migration,
|
||||
},
|
||||
None => LegacyFormatMode::Migration,
|
||||
};
|
||||
match try_migrate_format(disks, &formats, set_count, set_drive_count, mode).await {
|
||||
Ok(LegacyFormatOutcome::Migrated { format, quorum_members }) => {
|
||||
info!("Migrated format from MinIO config");
|
||||
retain_format_quorum_members(instance_ctx, disks, &format, &quorum_members, None, set_drive_count).await?;
|
||||
@@ -217,10 +215,10 @@ pub(crate) async fn connect_load_init_formats_with_instance_ctx(
|
||||
);
|
||||
return Err(Error::CorruptedFormat);
|
||||
}
|
||||
Ok(LegacyFormatOutcome::None) => {}
|
||||
Ok(LegacyFormatOutcome::None | LegacyFormatOutcome::ReadCompatible) => {}
|
||||
Err(e) => return Err(e),
|
||||
}
|
||||
if fresh_bootstrap_proven {
|
||||
if first_disk && fresh_bootstrap_proven {
|
||||
let fm = init_format_erasure(instance_ctx, disks, set_count, set_drive_count, deployment_id).await?;
|
||||
return Ok(LoadedFormat {
|
||||
format: fm,
|
||||
@@ -229,6 +227,26 @@ pub(crate) async fn connect_load_init_formats_with_instance_ctx(
|
||||
}
|
||||
}
|
||||
|
||||
// Resolve legacy migration before admitting native reads. Blank replacement
|
||||
// slots contribute no votes and receive no formatting/bootstrap authority.
|
||||
if let Some((format, members)) = native_read_candidate {
|
||||
retain_format_quorum_members(
|
||||
instance_ctx,
|
||||
disks,
|
||||
&format,
|
||||
&members,
|
||||
Some(&unformatted_bootstrap_routes),
|
||||
set_drive_count,
|
||||
)
|
||||
.await?;
|
||||
// Loading an existing format grants no authority to initialize pool
|
||||
// metadata or migrate formats. The persisted storage classes and each
|
||||
// object's EC metadata still determine service and operation quorums.
|
||||
return Ok(LoadedFormat {
|
||||
format,
|
||||
pool_meta_bootstrap_authority: PoolMetaBootstrapAuthority::None,
|
||||
});
|
||||
}
|
||||
info!(
|
||||
"first_disk: {}, should_init_erasure_disks: {}",
|
||||
first_disk,
|
||||
@@ -271,9 +289,11 @@ async fn verified_legacy_adoption_source(
|
||||
set_count: usize,
|
||||
set_drive_count: usize,
|
||||
) -> Result<PoolMetaBootstrapAuthority> {
|
||||
match try_migrate_format(disks, rustfs_formats, set_count, set_drive_count).await {
|
||||
match try_migrate_format(disks, rustfs_formats, set_count, set_drive_count, LegacyFormatMode::Migration).await {
|
||||
Ok(LegacyFormatOutcome::Migrated { .. }) => Ok(PoolMetaBootstrapAuthority::LegacyAdoption),
|
||||
Ok(LegacyFormatOutcome::None | LegacyFormatOutcome::Incompatible) => Ok(PoolMetaBootstrapAuthority::None),
|
||||
Ok(LegacyFormatOutcome::None | LegacyFormatOutcome::ReadCompatible | LegacyFormatOutcome::Incompatible) => {
|
||||
Ok(PoolMetaBootstrapAuthority::None)
|
||||
}
|
||||
Err(err) => {
|
||||
debug!(
|
||||
error = %err,
|
||||
@@ -443,25 +463,49 @@ enum LegacyFormatOutcome {
|
||||
/// version mismatch, or a parse failure). The caller must decide how to
|
||||
/// proceed; creating a fresh format would leave the legacy objects unreadable.
|
||||
Incompatible,
|
||||
/// Observed legacy formats agree with the native read quorum, without granting writer authority.
|
||||
ReadCompatible,
|
||||
/// No MinIO `format.json` was present on any disk (a normal fresh install).
|
||||
None,
|
||||
}
|
||||
|
||||
/// Native reads validate observed legacy identities without electing a migration writer.
|
||||
enum LegacyFormatMode<'a> {
|
||||
Migration,
|
||||
NativeRead {
|
||||
reference: &'a FormatV3,
|
||||
native_errors: &'a [Option<DiskError>],
|
||||
allow_migration: bool,
|
||||
},
|
||||
}
|
||||
|
||||
/// Tries to migrate an on-disk MinIO `format.json` into RustFS format files.
|
||||
///
|
||||
/// Returns [`LegacyFormatOutcome`] describing whether a legacy format was present
|
||||
/// and, if so, whether it was compatible. `Err` is returned for read or persist
|
||||
/// failures that prevent a conclusive migration decision.
|
||||
/// failures that prevent a conclusive migration decision. Native read admission
|
||||
/// checks every observed legacy format against its existing layout and tolerates
|
||||
/// unavailable nonvoters. Migration requires complete observations and a strict majority.
|
||||
async fn try_migrate_format(
|
||||
disks: &[Option<DiskStore>],
|
||||
rustfs_formats: &[Option<FormatV3>],
|
||||
set_count: usize,
|
||||
set_drive_count: usize,
|
||||
mode: LegacyFormatMode<'_>,
|
||||
) -> Result<LegacyFormatOutcome> {
|
||||
let (native_reference, native_errors, allow_migration) = match mode {
|
||||
LegacyFormatMode::Migration => (None, None, true),
|
||||
LegacyFormatMode::NativeRead {
|
||||
reference,
|
||||
native_errors,
|
||||
allow_migration,
|
||||
} => (Some(reference), Some(native_errors), allow_migration),
|
||||
};
|
||||
let legacy_format_max_bytes = legacy_format_max_bytes(disks.len())?;
|
||||
let mut legacy_seen = false;
|
||||
let mut invalid_legacy_format = false;
|
||||
let mut read_error: Option<Error> = None;
|
||||
let mut unavailable_error = None;
|
||||
let mut legacy_formats = vec![None; disks.len()];
|
||||
|
||||
let reads = stream::iter(
|
||||
@@ -473,6 +517,13 @@ async fn try_migrate_format(
|
||||
let Some(disk) = disk else {
|
||||
return (index, None);
|
||||
};
|
||||
// Native format loading already proved this peer unavailable.
|
||||
// Reuse that observation and keep it ineligible for migration.
|
||||
if let Some(error) = native_errors.and_then(|errors| errors.get(index)).and_then(Option::as_ref)
|
||||
&& is_network_like_disk_error(error)
|
||||
{
|
||||
return (index, Some(Err(error.clone())));
|
||||
}
|
||||
let reader = match disk.read_file(MIGRATING_META_BUCKET, FORMAT_CONFIG_FILE).await {
|
||||
Ok(reader) => reader,
|
||||
Err(DiskError::FileNotFound | DiskError::VolumeNotFound) => return (index, None),
|
||||
@@ -512,6 +563,9 @@ async fn try_migrate_format(
|
||||
match read {
|
||||
Ok(Some(format)) => legacy_formats[index] = Some(format),
|
||||
Ok(None) => invalid_legacy_format = true,
|
||||
Err(err) if native_reference.is_some() && is_network_like_disk_error(&err) => {
|
||||
unavailable_error.get_or_insert(err);
|
||||
}
|
||||
Err(err) => {
|
||||
read_error.get_or_insert_with(|| err.into());
|
||||
}
|
||||
@@ -528,6 +582,15 @@ async fn try_migrate_format(
|
||||
return Ok(LegacyFormatOutcome::Incompatible);
|
||||
}
|
||||
|
||||
if let Some(reference) = native_reference {
|
||||
if !formats_match_reference_slots(&legacy_formats, reference, 0) {
|
||||
return Ok(LegacyFormatOutcome::Incompatible);
|
||||
}
|
||||
if !allow_migration || unavailable_error.is_some() || legacy_formats.iter().flatten().count() <= disks.len() / 2 {
|
||||
return Ok(LegacyFormatOutcome::ReadCompatible);
|
||||
}
|
||||
}
|
||||
|
||||
let format = match get_format_erasure_in_quorum(&legacy_formats, 0) {
|
||||
Ok(format) => format,
|
||||
Err(_) => return Ok(LegacyFormatOutcome::Incompatible),
|
||||
@@ -677,7 +740,7 @@ fn select_existing_format_read_quorum(
|
||||
|| check_format_erasure_value_for_topology(reference, formats.len(), set_drive_count).is_err()
|
||||
|| formats.iter().zip(errors).any(|(format, error)| match (format, error) {
|
||||
(Some(_), None) => false,
|
||||
(None, Some(DiskError::DiskNotFound)) => false,
|
||||
(None, Some(DiskError::DiskNotFound | DiskError::UnformattedDisk)) => false,
|
||||
(None, Some(error)) => !is_network_like_disk_error(error),
|
||||
_ => true,
|
||||
})
|
||||
@@ -1445,13 +1508,23 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn existing_format_read_quorum_rejects_conflicts_invalid_slots_and_blank_disks() {
|
||||
fn existing_format_read_quorum_excludes_blank_and_network_unavailable_slots() {
|
||||
let (formats, mut errors) = half_pool_formats();
|
||||
errors[2] = Some(DiskError::UnformattedDisk);
|
||||
errors[3] = Some(DiskError::Timeout);
|
||||
let (_, members) = select_existing_format_read_quorum(&formats, &errors, 4)
|
||||
.expect("blank and unreachable slots cannot block an existing native read quorum");
|
||||
assert!(!members[2] && !members[3]);
|
||||
assert_eq!(members.iter().filter(|member| **member).count(), 12);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn existing_format_read_quorum_rejects_conflicts_and_invalid_slots() {
|
||||
for failure in [
|
||||
"conflict",
|
||||
"wrong_slot",
|
||||
"nil",
|
||||
"duplicate",
|
||||
"blank",
|
||||
"corrupt",
|
||||
"access_denied",
|
||||
"tie",
|
||||
@@ -1473,7 +1546,6 @@ mod tests {
|
||||
format.erasure.sets[5][3] = format.erasure.sets[0][0];
|
||||
}
|
||||
}
|
||||
"blank" => errors[2] = Some(DiskError::UnformattedDisk),
|
||||
"corrupt" => errors[2] = Some(DiskError::CorruptedFormat),
|
||||
"access_denied" => errors[2] = Some(DiskError::FileAccessDenied),
|
||||
"tie" => {
|
||||
@@ -1532,6 +1604,186 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn existing_format_read_quorum_quarantines_blank_replacement_without_writes() {
|
||||
for first_disk in [false, true] {
|
||||
for surviving_slot in 0..2 {
|
||||
let (_directory, mut disks) = local_disks_with_set_width(2, 2).await;
|
||||
let blank = 1 - surviving_slot;
|
||||
let mut format = FormatV3::new(1, 2);
|
||||
format.erasure.this = format.erasure.sets[0][surviving_slot];
|
||||
save_format_file(&disks[surviving_slot], &Some(format.clone()))
|
||||
.await
|
||||
.expect("persist surviving format");
|
||||
let original = disks.clone();
|
||||
let before = format_bytes(&original).await;
|
||||
let loaded = connect_load_init_formats_with_instance_ctx(
|
||||
&Arc::new(InstanceContext::new()),
|
||||
first_disk,
|
||||
&mut disks,
|
||||
1,
|
||||
2,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.expect("existing native format must load at read quorum with a blank replacement");
|
||||
assert_eq!(loaded.format.shared_identity(), format.shared_identity());
|
||||
assert_eq!(loaded.pool_meta_bootstrap_authority, PoolMetaBootstrapAuthority::None);
|
||||
assert!(disks[surviving_slot].is_some());
|
||||
assert!(disks[blank].is_none(), "blank replacement cannot vote or serve data");
|
||||
assert_eq!(format_bytes(&original).await, before, "read admission cannot write either format");
|
||||
assert!(matches!(
|
||||
load_format_erasure(original[blank].as_ref().expect("blank handle"), false).await,
|
||||
Err(DiskError::UnformattedDisk)
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn native_read_quorum_with_blank_and_unavailable_slots_validates_legacy_without_writes() {
|
||||
for first_disk in [false, true] {
|
||||
for scenario in [
|
||||
"native-remote",
|
||||
"legacy-missing",
|
||||
"foreign-remote",
|
||||
"corrupt-remote",
|
||||
"wrong-slot-missing",
|
||||
] {
|
||||
let (_directory, mut disks) = local_disks_with_set_width(4, 4).await;
|
||||
let reference = FormatV3::new(1, 4);
|
||||
for slot in 0..2 {
|
||||
let mut format = reference.clone();
|
||||
format.erasure.this = format.erasure.sets[0][slot];
|
||||
save_format_file(&disks[slot], &Some(format.clone()))
|
||||
.await
|
||||
.expect("persist native read quorum");
|
||||
if scenario != "native-remote" {
|
||||
write_legacy_format(&disks[slot], &format).await;
|
||||
}
|
||||
}
|
||||
match scenario {
|
||||
"foreign-remote" => {
|
||||
let mut foreign = FormatV3::new(1, 4);
|
||||
foreign.erasure.this = foreign.erasure.sets[0][2];
|
||||
write_legacy_format(&disks[2], &foreign).await;
|
||||
}
|
||||
"corrupt-remote" => write_legacy_bytes(&disks[2], bytes::Bytes::from_static(b"corrupt legacy")).await,
|
||||
"wrong-slot-missing" => {
|
||||
let mut wrong_slot = reference.clone();
|
||||
wrong_slot.erasure.this = reference.erasure.sets[0][0];
|
||||
write_legacy_format(&disks[2], &wrong_slot).await;
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
let original = disks.clone();
|
||||
let before = format_bytes(&original).await;
|
||||
if scenario.ends_with("remote") {
|
||||
let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("reserve unavailable peer address");
|
||||
let address = listener.local_addr().expect("peer address");
|
||||
drop(listener);
|
||||
let mut endpoint = Endpoint::try_from(format!("http://{address}/disk").as_str()).expect("remote endpoint");
|
||||
endpoint.is_local = false;
|
||||
endpoint.set_pool_index(0);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(3);
|
||||
let remote = crate::cluster::rpc::RemoteDisk::new(
|
||||
&endpoint,
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
Arc::new(crate::cluster::rpc::TcpHttpInternodeDataTransport),
|
||||
)
|
||||
.await
|
||||
.expect("construct unavailable remote handle");
|
||||
let remote = Arc::new(crate::disk::Disk::Remote(Box::new(remote)));
|
||||
let error = remote
|
||||
.read_all(RUSTFS_META_BUCKET, FORMAT_CONFIG_FILE)
|
||||
.await
|
||||
.expect_err("remote peer must be unavailable");
|
||||
assert!(is_network_like_disk_error(&error), "expected unavailable peer, got {error:?}");
|
||||
disks[3] = Some(remote);
|
||||
} else {
|
||||
disks[3] = None;
|
||||
}
|
||||
let result = connect_load_init_formats_with_instance_ctx(
|
||||
&Arc::new(InstanceContext::new()),
|
||||
first_disk,
|
||||
&mut disks,
|
||||
1,
|
||||
4,
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
if matches!(scenario, "native-remote" | "legacy-missing") {
|
||||
let loaded = result.unwrap_or_else(|error| panic!("{scenario} must admit native reads: {error:?}"));
|
||||
assert_eq!(loaded.format.shared_identity(), reference.shared_identity());
|
||||
assert_eq!(loaded.pool_meta_bootstrap_authority, PoolMetaBootstrapAuthority::None);
|
||||
assert!(disks[..2].iter().all(Option::is_some));
|
||||
assert!(disks[2..].iter().all(Option::is_none), "unavailable and blank slots cannot vote");
|
||||
} else {
|
||||
assert!(
|
||||
matches!(result, Err(Error::CorruptedFormat)),
|
||||
"{scenario} observed legacy evidence must reject reads"
|
||||
);
|
||||
}
|
||||
assert_eq!(format_bytes(&original).await, before, "{scenario} cannot write native formats");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn partial_native_format_with_legacy_quorum_reads_without_migration_writes() {
|
||||
let (_directory, mut disks) = local_disks_with_set_width(2, 2).await;
|
||||
let legacy = FormatV3::new(1, 2);
|
||||
write_legacy_majority(&disks, &legacy).await;
|
||||
let mut native = legacy.clone();
|
||||
native.erasure.this = native.erasure.sets[0][0];
|
||||
save_format_file(&disks[0], &Some(native))
|
||||
.await
|
||||
.expect("persist partial native format");
|
||||
let original = disks.clone();
|
||||
let before = format_bytes(&disks).await;
|
||||
let loaded =
|
||||
connect_load_init_formats_with_instance_ctx(&Arc::new(InstanceContext::new()), false, &mut disks, 1, 2, None)
|
||||
.await
|
||||
.expect("follower can read the existing native layout without migrating legacy formats");
|
||||
assert_eq!(loaded.pool_meta_bootstrap_authority, PoolMetaBootstrapAuthority::None);
|
||||
assert!(disks[0].is_some() && disks[1].is_none(), "legacy-only slots cannot vote for native reads");
|
||||
assert_eq!(format_bytes(&original).await, before, "follower legacy inspection cannot persist formats");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn follower_below_native_read_quorum_skips_legacy_reads_without_writes() {
|
||||
let (_directory, mut disks) = local_disks_with_set_width(3, 3).await;
|
||||
let mut format = FormatV3::new(1, 3);
|
||||
format.erasure.this = format.erasure.sets[0][0];
|
||||
save_format_file(&disks[0], &Some(format))
|
||||
.await
|
||||
.expect("persist one native format below the two-drive read quorum");
|
||||
for disk in &disks {
|
||||
write_legacy_bytes(disk, bytes::Bytes::from_static(b"invalid legacy format")).await;
|
||||
}
|
||||
let before = format_bytes(&disks).await;
|
||||
|
||||
let result = connect_load_init_formats(false, &mut disks, 1, 3, None).await;
|
||||
|
||||
assert!(
|
||||
matches!(result, Err(Error::ErasureReadQuorum)),
|
||||
"a non-reader must reject quorum before probing corrupt legacy files"
|
||||
);
|
||||
assert_eq!(format_bytes(&disks).await, before);
|
||||
for disk in disks.iter().flatten() {
|
||||
assert_eq!(
|
||||
disk.read_all(MIGRATING_META_BUCKET, FORMAT_CONFIG_FILE)
|
||||
.await
|
||||
.expect("legacy bytes must remain untouched"),
|
||||
bytes::Bytes::from_static(b"invalid legacy format")
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn existing_format_load_succeeds_with_a_strict_majority() {
|
||||
let (_temp_dir, mut disks) = two_local_disks_with_missing_third().await;
|
||||
@@ -2375,7 +2627,7 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn legacy_format_migration_resumes_partial_rustfs_writes() {
|
||||
for drive_count in [3, 4] {
|
||||
for drive_count in [2, 3, 4] {
|
||||
let (_temp_dir, mut disks) = local_disks(drive_count).await;
|
||||
let legacy = FormatV3::new(1, drive_count);
|
||||
write_legacy_majority(&disks, &legacy).await;
|
||||
@@ -2434,23 +2686,32 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn legacy_format_migration_rejects_a_foreign_partial_rustfs_format() {
|
||||
let (_temp_dir, mut disks) = local_disks(3).await;
|
||||
let legacy = FormatV3::new(1, 3);
|
||||
write_legacy_majority(&disks, &legacy).await;
|
||||
let mut foreign = FormatV3::new(1, 3);
|
||||
foreign.erasure.this = foreign.erasure.sets[0][0];
|
||||
let foreign_id = foreign.id;
|
||||
save_format_file(&disks[0], &Some(foreign))
|
||||
.await
|
||||
.expect("foreign partial RustFS format should be written");
|
||||
for first_disk in [false, true] {
|
||||
for drive_count in [2, 3] {
|
||||
let (_temp_dir, mut disks) = local_disks(drive_count).await;
|
||||
let legacy = FormatV3::new(1, drive_count);
|
||||
write_legacy_majority(&disks, &legacy).await;
|
||||
let mut foreign = FormatV3::new(1, drive_count);
|
||||
foreign.erasure.this = foreign.erasure.sets[0][0];
|
||||
let foreign_id = foreign.id;
|
||||
save_format_file(&disks[0], &Some(foreign))
|
||||
.await
|
||||
.expect("foreign partial RustFS format should be written");
|
||||
|
||||
assert!(matches!(
|
||||
connect_load_init_formats(true, &mut disks, 1, 3, None).await,
|
||||
Err(Error::CorruptedFormat)
|
||||
));
|
||||
let (formats, _) = load_format_erasure_all(&disks, false).await;
|
||||
assert_eq!(formats[0].as_ref().map(|format| format.id), Some(foreign_id));
|
||||
assert!(formats[1..].iter().all(Option::is_none), "foreign partial state must not be extended");
|
||||
let result = connect_load_init_formats(first_disk, &mut disks, 1, drive_count, None).await;
|
||||
if !first_disk && drive_count == 3 {
|
||||
assert!(matches!(result, Err(Error::ErasureReadQuorum)), "a follower below read quorum must wait");
|
||||
} else {
|
||||
assert!(
|
||||
matches!(result, Err(Error::CorruptedFormat)),
|
||||
"an eligible reader or migration writer must reject the foreign legacy layout"
|
||||
);
|
||||
}
|
||||
let (formats, _) = load_format_erasure_all(&disks, false).await;
|
||||
assert_eq!(formats[0].as_ref().map(|format| format.id), Some(foreign_id));
|
||||
assert!(formats[1..].iter().all(Option::is_none), "foreign partial state must not be extended");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -2856,6 +2856,28 @@ fn check_batch_delete_condition(object: &ObjectToDelete, snapshot: &ConditionalD
|
||||
/// `handle_copy_object` shares the same fail-safe boundary. Metadata-only and
|
||||
/// version-only same-key copies do not use this helper and continue to preserve
|
||||
/// the existing version's protected state.
|
||||
fn materialized_copy_options(source: &HashMap<String, String>, dst_opts: &ObjectOptions) -> ObjectOptions {
|
||||
ObjectOptions {
|
||||
user_defined: materialized_copy_user_defined(source),
|
||||
versioned: dst_opts.versioned,
|
||||
version_suspended: dst_opts.version_suspended,
|
||||
shard_integrity_write_mode: dst_opts.shard_integrity_write_mode,
|
||||
version_id: dst_opts.version_id.clone(),
|
||||
no_lock: dst_opts.no_lock,
|
||||
write_completion: dst_opts.write_completion,
|
||||
mod_time: dst_opts.mod_time,
|
||||
http_preconditions: dst_opts.http_preconditions.clone(),
|
||||
expected_current_version_id: dst_opts.expected_current_version_id.clone(),
|
||||
expected_bucket_incarnation_id: dst_opts.expected_bucket_incarnation_id,
|
||||
namespace_lock_fence: dst_opts.namespace_lock_fence.clone(),
|
||||
write_lock_context: dst_opts.write_lock_context.clone(),
|
||||
bucket_lifecycle_lock_fence: dst_opts.bucket_lifecycle_lock_fence.clone(),
|
||||
object_lock_config_snapshot: dst_opts.object_lock_config_snapshot.clone(),
|
||||
quota_admission: dst_opts.quota_admission,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn materialized_copy_user_defined(source: &HashMap<String, String>) -> HashMap<String, String> {
|
||||
let mut metadata = source.clone();
|
||||
for suffix in [
|
||||
@@ -5337,24 +5359,7 @@ impl ECStore {
|
||||
.map(|oi| (oi, None));
|
||||
}
|
||||
// Transitioned object self-copy: restore from tier into the same pool.
|
||||
let mut put_opts = ObjectOptions {
|
||||
user_defined: materialized_copy_user_defined(src_info.user_defined.as_ref()),
|
||||
versioned: dst_opts.versioned,
|
||||
shard_integrity_write_mode: dst_opts.shard_integrity_write_mode,
|
||||
version_id: dst_opts.version_id.clone(),
|
||||
no_lock: dst_opts.no_lock,
|
||||
write_completion: dst_opts.write_completion,
|
||||
mod_time: dst_opts.mod_time,
|
||||
http_preconditions: dst_opts.http_preconditions.clone(),
|
||||
expected_current_version_id: dst_opts.expected_current_version_id.clone(),
|
||||
expected_bucket_incarnation_id: dst_opts.expected_bucket_incarnation_id,
|
||||
namespace_lock_fence: dst_opts.namespace_lock_fence.clone(),
|
||||
write_lock_context: dst_opts.write_lock_context.clone(),
|
||||
bucket_lifecycle_lock_fence: dst_opts.bucket_lifecycle_lock_fence.clone(),
|
||||
object_lock_config_snapshot: dst_opts.object_lock_config_snapshot.clone(),
|
||||
quota_admission: dst_opts.quota_admission,
|
||||
..Default::default()
|
||||
};
|
||||
let mut put_opts = materialized_copy_options(src_info.user_defined.as_ref(), &dst_opts);
|
||||
if !self.single_pool() {
|
||||
put_opts.decommission_capacity_admission =
|
||||
crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx).await;
|
||||
@@ -5382,24 +5387,7 @@ impl ECStore {
|
||||
// stays consistent with the new version's metadata. Sharing the source data_dir via
|
||||
// a metadata-only version copy would corrupt SSE/compressed objects (issue #4238).
|
||||
if let Some(reader) = src_info.put_object_reader.as_mut() {
|
||||
let mut put_opts = ObjectOptions {
|
||||
user_defined: materialized_copy_user_defined(src_info.user_defined.as_ref()),
|
||||
versioned: dst_opts.versioned,
|
||||
shard_integrity_write_mode: dst_opts.shard_integrity_write_mode,
|
||||
version_id: dst_opts.version_id.clone(),
|
||||
no_lock: dst_opts.no_lock,
|
||||
write_completion: dst_opts.write_completion,
|
||||
mod_time: dst_opts.mod_time,
|
||||
http_preconditions: dst_opts.http_preconditions.clone(),
|
||||
expected_current_version_id: dst_opts.expected_current_version_id.clone(),
|
||||
expected_bucket_incarnation_id: dst_opts.expected_bucket_incarnation_id,
|
||||
namespace_lock_fence: dst_opts.namespace_lock_fence.clone(),
|
||||
write_lock_context: dst_opts.write_lock_context.clone(),
|
||||
bucket_lifecycle_lock_fence: dst_opts.bucket_lifecycle_lock_fence.clone(),
|
||||
object_lock_config_snapshot: dst_opts.object_lock_config_snapshot.clone(),
|
||||
quota_admission: dst_opts.quota_admission,
|
||||
..Default::default()
|
||||
};
|
||||
let mut put_opts = materialized_copy_options(src_info.user_defined.as_ref(), &dst_opts);
|
||||
if !self.single_pool() {
|
||||
put_opts.decommission_capacity_admission =
|
||||
crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx).await;
|
||||
@@ -5440,24 +5428,7 @@ impl ECStore {
|
||||
};
|
||||
let dst_object_name = dst_object.as_str();
|
||||
|
||||
let mut put_opts = ObjectOptions {
|
||||
user_defined: materialized_copy_user_defined(src_info.user_defined.as_ref()),
|
||||
versioned: dst_opts.versioned,
|
||||
shard_integrity_write_mode: dst_opts.shard_integrity_write_mode,
|
||||
version_id: dst_opts.version_id.clone(),
|
||||
no_lock: dst_opts.no_lock,
|
||||
write_completion: dst_opts.write_completion,
|
||||
mod_time: dst_opts.mod_time,
|
||||
http_preconditions: dst_opts.http_preconditions.clone(),
|
||||
expected_current_version_id: dst_opts.expected_current_version_id.clone(),
|
||||
expected_bucket_incarnation_id: dst_opts.expected_bucket_incarnation_id,
|
||||
namespace_lock_fence: dst_opts.namespace_lock_fence.clone(),
|
||||
write_lock_context: dst_opts.write_lock_context.clone(),
|
||||
bucket_lifecycle_lock_fence: dst_opts.bucket_lifecycle_lock_fence.clone(),
|
||||
object_lock_config_snapshot: dst_opts.object_lock_config_snapshot.clone(),
|
||||
quota_admission: dst_opts.quota_admission,
|
||||
..Default::default()
|
||||
};
|
||||
let mut put_opts = materialized_copy_options(src_info.user_defined.as_ref(), &dst_opts);
|
||||
if !self.single_pool() {
|
||||
put_opts.decommission_capacity_admission =
|
||||
crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx).await;
|
||||
@@ -7737,6 +7708,19 @@ mod tests {
|
||||
.expect("target metadata should be visible after the commit boundary");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn materialized_copy_preserves_suspended_destination() {
|
||||
let opts = materialized_copy_options(
|
||||
&HashMap::new(),
|
||||
&ObjectOptions {
|
||||
version_suspended: true,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
assert!(opts.version_suspended);
|
||||
assert!(!opts.versioned);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn materialized_copy_drops_remote_tuple_and_free_version_compatibility_keys() {
|
||||
let protected_suffixes = [
|
||||
|
||||
@@ -83,6 +83,7 @@ rustfs-common = { workspace = true }
|
||||
rustfs-heal-contracts = { workspace = true }
|
||||
rustfs-madmin = { workspace = true }
|
||||
rustfs-utils = { workspace = true }
|
||||
jiff = { workspace = true }
|
||||
tokio = { workspace = true, features = ["sync", "io-util", "time", "macros", "fs", "rt-multi-thread"] }
|
||||
tokio-util = { workspace = true, features = ["io", "compat", "rt"] }
|
||||
tonic = { workspace = true }
|
||||
@@ -110,7 +111,7 @@ walkdir = { workspace = true }
|
||||
http = { workspace = true }
|
||||
temp-env = { workspace = true, features = ["async_closure"] }
|
||||
tokio = { workspace = true, features = ["test-util", "fs", "rt-multi-thread"] }
|
||||
chrono = { workspace = true }
|
||||
jiff = { workspace = true }
|
||||
|
||||
[lib]
|
||||
doctest = false
|
||||
|
||||
@@ -2871,7 +2871,7 @@ mod tests {
|
||||
}
|
||||
|
||||
fn w13_timestamp() -> String {
|
||||
chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Secs, true)
|
||||
jiff::Timestamp::now().strftime("%Y-%m-%dT%H:%M:%SZ").to_string()
|
||||
}
|
||||
|
||||
fn w13_selection_contains(selection: &str, lane: &str) -> bool {
|
||||
|
||||
+20
-3
@@ -311,9 +311,8 @@ impl LockedCache {
|
||||
return false;
|
||||
}
|
||||
|
||||
let mut new = CacheEntity::clone(target);
|
||||
op(&mut new);
|
||||
*target = Arc::new(new);
|
||||
// Detach from published snapshots once per partition and transaction.
|
||||
op(Arc::make_mut(target));
|
||||
if let Some(changes) = changes {
|
||||
changes
|
||||
.lock()
|
||||
@@ -756,6 +755,24 @@ mod tests {
|
||||
use crate::cache::Cache;
|
||||
use crate::store::MappedPolicy;
|
||||
|
||||
#[test]
|
||||
fn batched_deletes_reuse_private_partition() {
|
||||
let cache = Cache::default();
|
||||
let now = OffsetDateTime::now_utc();
|
||||
for key in ["one", "two"] {
|
||||
cache.add_or_update_sts_account(key, &UserIdentity::default(), now);
|
||||
}
|
||||
let before = cache.snapshot();
|
||||
cache.with_write_lock(|locked| {
|
||||
locked.delete_sts_account("one", now);
|
||||
let private = Arc::as_ptr(&locked.state().sts_accounts);
|
||||
locked.delete_sts_account("two", now);
|
||||
assert_eq!(private, Arc::as_ptr(&locked.state().sts_accounts));
|
||||
});
|
||||
assert_eq!(before.sts_accounts.len(), 2);
|
||||
assert!(cache.snapshot().sts_accounts.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reload_journal_preserves_same_key_mutations_and_expires_on_cancellation() {
|
||||
use super::{CacheEntity, CachePartition};
|
||||
|
||||
+106
-126
@@ -18,7 +18,7 @@ use crate::sys::{get_claims_from_token_with_secret, get_claims_from_token_with_s
|
||||
use crate::{
|
||||
cache::{Cache, CacheEntity, LockedCache},
|
||||
error::{Error as IamError, is_err_no_such_group, is_err_no_such_policy, is_err_no_such_user},
|
||||
store::{GroupInfo, MappedPolicy, Store, UserType, object::IAM_CONFIG_PREFIX},
|
||||
store::{GroupEdit, GroupInfo, MappedPolicy, Store, UserType, object::IAM_CONFIG_PREFIX},
|
||||
sys::{
|
||||
MAX_SVCSESSION_POLICY_SIZE, SESSION_POLICY_NAME, SESSION_POLICY_NAME_EXTRACTED, SITE_REPLICATOR_CLAIM,
|
||||
SITE_REPLICATOR_SERVICE_ACCOUNT, STATUS_DISABLED, STATUS_ENABLED, UpdateServiceAccountOpts,
|
||||
@@ -1122,7 +1122,9 @@ where
|
||||
|
||||
let direct_policies = mapped_policy.to_slice();
|
||||
let has_identity_mapping = !direct_policies.is_empty();
|
||||
let policies = self.policy_db_get_with_groups(direct_policies, groups).await?;
|
||||
// An uncached group mapping can contain a Deny even when a direct policy exists.
|
||||
let mut policies = direct_policies;
|
||||
policies.extend(self.policy_db_get_with_groups(Vec::new(), groups).await?);
|
||||
Ok((policies, has_identity_mapping))
|
||||
}
|
||||
|
||||
@@ -1578,7 +1580,7 @@ where
|
||||
args: &AddOrUpdateUserReq,
|
||||
updated_at: OffsetDateTime,
|
||||
) -> Result<OffsetDateTime> {
|
||||
self.add_user_at_with_secret_change(access_key, args, updated_at)
|
||||
self.add_user_at_with_secret_change(access_key, args, updated_at, true)
|
||||
.await
|
||||
.map(|(updated_at, _, _guard)| updated_at)
|
||||
}
|
||||
@@ -1589,7 +1591,22 @@ where
|
||||
access_key: &str,
|
||||
args: &AddOrUpdateUserReq,
|
||||
updated_at: OffsetDateTime,
|
||||
allow_status_change: bool,
|
||||
) -> Result<(OffsetDateTime, bool, tokio::sync::OwnedMutexGuard<crate::cache::MutationLockState>)> {
|
||||
// A virtual OIDC parent is a canonical base64url-encoded SHA-256 digest.
|
||||
// It has no stored user record to collide with in the checks below.
|
||||
if access_key.len() == 43
|
||||
&& base64_simd::URL_SAFE_NO_PAD
|
||||
.decode_to_vec(access_key.as_bytes())
|
||||
.is_ok_and(|bytes| bytes.len() == 32)
|
||||
{
|
||||
return Err(Error::AccessKeyAlreadyExists);
|
||||
}
|
||||
// The access key is a single durable IAM path component. Reject
|
||||
// unrepresentable keys before allocating or waiting on mutation locks.
|
||||
if access_key.contains(['\0', '/', '\\']) || matches!(access_key, "" | "." | "..") {
|
||||
return Err(Error::InvalidArgument);
|
||||
}
|
||||
let parent_lock = self.cache.parent_mutation_lock(access_key);
|
||||
let parent_guard = parent_lock.lock_owned().await;
|
||||
let _service_guard = self.cache.service_account_mutation_lock().lock().await;
|
||||
@@ -1632,9 +1649,14 @@ where
|
||||
});
|
||||
user_entry.update_at = Some(updated_at);
|
||||
|
||||
self.api
|
||||
.save_user_identity(access_key, UserType::Reg, user_entry.clone(), None)
|
||||
.await?;
|
||||
let secret_changed = if allow_status_change {
|
||||
self.api
|
||||
.save_user_identity(access_key, UserType::Reg, user_entry.clone(), None)
|
||||
.await?;
|
||||
secret_changed
|
||||
} else {
|
||||
self.api.save_regular_user_preserving_status(access_key, &user_entry).await?
|
||||
};
|
||||
|
||||
self.update_user_with_claims(access_key, user_entry)?;
|
||||
|
||||
@@ -1979,38 +2001,25 @@ where
|
||||
// whose time predates the entity's load time — a replicated edit
|
||||
// whose source time is older than this node's startup would
|
||||
// otherwise never reach the cache.
|
||||
let gi = match cache.groups.get(group) {
|
||||
Some(res) => {
|
||||
let mut gi = res.clone();
|
||||
drop(cache);
|
||||
let (edit, current) = self.api.begin_group_edit(group).await?;
|
||||
let gi = match current {
|
||||
Some(mut gi) => {
|
||||
let mut uniq_set: HashSet<String, std::collections::hash_map::RandomState> =
|
||||
HashSet::from_iter(gi.members.iter().cloned());
|
||||
uniq_set.extend(members.iter().cloned());
|
||||
uniq_set.extend(members);
|
||||
|
||||
gi.members = uniq_set.into_iter().collect();
|
||||
gi.update_at = Some(updated_at);
|
||||
gi
|
||||
}
|
||||
None => {
|
||||
let mut gi = GroupInfo::new(members.clone());
|
||||
let mut gi = GroupInfo::new(members);
|
||||
gi.update_at = Some(updated_at);
|
||||
gi
|
||||
}
|
||||
};
|
||||
drop(cache);
|
||||
|
||||
self.api.save_group_info(group, gi.clone()).await?;
|
||||
|
||||
self.cache.with_write_lock(|cache| {
|
||||
let now = OffsetDateTime::now_utc();
|
||||
cache.add_or_update_group(group, &gi, now);
|
||||
|
||||
let user_group_memberships = Arc::clone(&cache.state().user_group_memberships);
|
||||
members.iter().for_each(|member| {
|
||||
let mut m = user_group_memberships.get(member).cloned().unwrap_or_default();
|
||||
m.insert(group.to_string());
|
||||
cache.add_or_update_user_group_membership(member, &m, now);
|
||||
});
|
||||
});
|
||||
self.save_group_and_publish(group, &*edit, gi).await?;
|
||||
|
||||
Ok(updated_at)
|
||||
}
|
||||
@@ -2026,13 +2035,8 @@ where
|
||||
return Err(Error::InvalidArgument);
|
||||
}
|
||||
|
||||
let cache = self.cache.snapshot();
|
||||
let groups = Arc::clone(&cache.groups);
|
||||
let mut gi = match groups.get(name) {
|
||||
Some(gi) => gi.clone(),
|
||||
None => return Err(Error::NoSuchGroup(name.to_string())),
|
||||
};
|
||||
drop(cache);
|
||||
let (edit, current) = self.api.begin_group_edit(name).await?;
|
||||
let mut gi = current.ok_or_else(|| Error::NoSuchGroup(name.to_string()))?;
|
||||
|
||||
if enable {
|
||||
gi.status = STATUS_ENABLED.to_owned();
|
||||
@@ -2041,11 +2045,7 @@ where
|
||||
}
|
||||
gi.update_at = Some(updated_at);
|
||||
|
||||
self.api.save_group_info(name, gi.clone()).await?;
|
||||
|
||||
// Cache publication time is the local clock, not the record stamp
|
||||
// (see `add_users_to_group_at`).
|
||||
self.cache.add_or_update_group(name, &gi, OffsetDateTime::now_utc());
|
||||
self.save_group_and_publish(name, &*edit, gi).await?;
|
||||
|
||||
Ok(updated_at)
|
||||
}
|
||||
@@ -2132,37 +2132,27 @@ where
|
||||
update_cache_only: bool,
|
||||
updated_at: OffsetDateTime,
|
||||
) -> Result<OffsetDateTime> {
|
||||
let cache = self.cache.snapshot();
|
||||
let mut gi = cache
|
||||
.groups
|
||||
.get(name)
|
||||
.cloned()
|
||||
.ok_or_else(|| Error::NoSuchGroup(name.to_string()))?;
|
||||
drop(cache);
|
||||
|
||||
let s: HashSet<&String> = HashSet::from_iter(gi.members.iter());
|
||||
let d: HashSet<&String> = HashSet::from_iter(members.iter());
|
||||
gi.members = s.difference(&d).map(|v| v.to_string()).collect::<Vec<String>>();
|
||||
gi.update_at = Some(updated_at);
|
||||
if !update_cache_only {
|
||||
self.api.save_group_info(name, gi.clone()).await?;
|
||||
}
|
||||
|
||||
self.cache.with_write_lock(|cache| {
|
||||
// Sample after storage completes so a concurrent reload cannot
|
||||
// make this publication older than the cache it must update.
|
||||
let now = OffsetDateTime::now_utc();
|
||||
cache.add_or_update_group(name, &gi, now);
|
||||
|
||||
let user_group_memberships = Arc::clone(&cache.state().user_group_memberships);
|
||||
members.iter().for_each(|member| {
|
||||
if let Some(m) = user_group_memberships.get(member) {
|
||||
let mut m = m.clone();
|
||||
m.remove(name);
|
||||
cache.add_or_update_user_group_membership(member, &m, now);
|
||||
}
|
||||
let removed: HashSet<_> = members.into_iter().collect();
|
||||
let remove = |mut gi: GroupInfo| {
|
||||
gi.members.retain(|m| !removed.contains(m));
|
||||
gi.update_at = Some(updated_at);
|
||||
gi
|
||||
};
|
||||
if update_cache_only {
|
||||
return self.cache.with_write_lock(|cache| {
|
||||
let gi = cache
|
||||
.state()
|
||||
.groups
|
||||
.get(name)
|
||||
.cloned()
|
||||
.ok_or_else(|| Error::NoSuchGroup(name.to_string()))?;
|
||||
self.publish_group_unlocked(cache, name, &remove(gi));
|
||||
Ok(updated_at)
|
||||
});
|
||||
});
|
||||
}
|
||||
let (edit, current) = self.api.begin_group_edit(name).await?;
|
||||
let gi = current.ok_or_else(|| Error::NoSuchGroup(name.to_string()))?;
|
||||
self.save_group_and_publish(name, &*edit, remove(gi)).await?;
|
||||
|
||||
Ok(updated_at)
|
||||
}
|
||||
@@ -2187,7 +2177,6 @@ where
|
||||
|
||||
let cache = self.cache.snapshot();
|
||||
let users_cache = Arc::clone(&cache.users);
|
||||
let groups_cache = Arc::clone(&cache.groups);
|
||||
drop(cache);
|
||||
|
||||
for member in members.iter() {
|
||||
@@ -2200,65 +2189,60 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
let gi = if members.is_empty() {
|
||||
// Reload from backend so we see latest members (e.g. after user was deleted elsewhere)
|
||||
let mut m = HashMap::new();
|
||||
self.api.load_group(group, &mut m).await?;
|
||||
m.get(group).cloned().ok_or_else(|| Error::NoSuchGroup(group.to_string()))?
|
||||
} else {
|
||||
groups_cache
|
||||
.get(group)
|
||||
.cloned()
|
||||
.ok_or_else(|| Error::NoSuchGroup(group.to_string()))?
|
||||
};
|
||||
|
||||
if members.is_empty() && !gi.members.is_empty() {
|
||||
if !members.is_empty() {
|
||||
return self.remove_members_from_group_at(group, members, false, updated_at).await;
|
||||
}
|
||||
let (edit, current) = self.api.begin_group_edit(group).await?;
|
||||
let gi = current.ok_or_else(|| Error::NoSuchGroup(group.to_string()))?;
|
||||
if !gi.members.is_empty() {
|
||||
return Err(Error::GroupNotEmpty);
|
||||
}
|
||||
|
||||
if members.is_empty() {
|
||||
if let Err(err) = self.api.delete_mapped_policy(group, UserType::Reg, true).await
|
||||
&& !is_err_no_such_policy(&err)
|
||||
{
|
||||
return Err(err);
|
||||
}
|
||||
edit.delete_empty().await?;
|
||||
|
||||
if let Err(err) = self.api.delete_group_info(group).await
|
||||
&& !is_err_no_such_group(&err)
|
||||
{
|
||||
return Err(err);
|
||||
}
|
||||
self.cache.with_write_lock(|cache| {
|
||||
let now = OffsetDateTime::now_utc();
|
||||
Self::replace_cached_group_members(cache, group, &[], now);
|
||||
cache.delete_group(group, now);
|
||||
cache.delete_group_policy(group, now);
|
||||
});
|
||||
|
||||
self.cache.with_write_lock(|cache| {
|
||||
let now = OffsetDateTime::now_utc();
|
||||
self.remove_group_from_memberships_map_unlocked(cache, group, now);
|
||||
cache.delete_group(group, now);
|
||||
cache.delete_group_policy(group, now);
|
||||
});
|
||||
|
||||
return Ok(updated_at);
|
||||
}
|
||||
|
||||
self.remove_members_from_group_at(group, members, false, updated_at).await
|
||||
Ok(updated_at)
|
||||
}
|
||||
|
||||
fn remove_group_from_memberships_map_unlocked(&self, cache: &mut LockedCache, group: &str, now: OffsetDateTime) {
|
||||
let user_group_memberships = Arc::clone(&cache.state().user_group_memberships);
|
||||
for (k, v) in user_group_memberships.iter() {
|
||||
if v.contains(group) {
|
||||
let mut m = v.clone();
|
||||
m.remove(group);
|
||||
cache.add_or_update_user_group_membership(k, &m, now);
|
||||
}
|
||||
}
|
||||
async fn save_group_and_publish(&self, group: &str, edit: &dyn GroupEdit, gi: GroupInfo) -> Result<()> {
|
||||
edit.save(&gi).await?;
|
||||
self.cache
|
||||
.with_write_lock(|cache| self.publish_group_unlocked(cache, group, &gi));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn update_group_memberships_map_unlocked(&self, cache: &mut LockedCache, group: &str, gi: &GroupInfo, now: OffsetDateTime) {
|
||||
let user_group_memberships = Arc::clone(&cache.state().user_group_memberships);
|
||||
for member in gi.members.iter() {
|
||||
let mut m = user_group_memberships.get(member).cloned().unwrap_or_default();
|
||||
m.insert(group.to_string());
|
||||
cache.add_or_update_user_group_membership(member, &m, now);
|
||||
fn publish_group_unlocked(&self, cache: &mut LockedCache, group: &str, gi: &GroupInfo) {
|
||||
let now = OffsetDateTime::now_utc();
|
||||
cache.add_or_update_group(group, gi, now);
|
||||
Self::replace_cached_group_members(cache, group, &gi.members, now);
|
||||
}
|
||||
|
||||
fn replace_cached_group_members(cache: &mut LockedCache, group: &str, members: &[String], now: OffsetDateTime) {
|
||||
// This node may have missed peer edits, so reconcile against the complete record,
|
||||
// including memberships that were already stale before this edit began.
|
||||
let mut missing: HashSet<_> = members.iter().collect();
|
||||
let reverse = Arc::clone(&cache.state().user_group_memberships);
|
||||
for (user, groups) in reverse.iter() {
|
||||
if groups.contains(group) && !missing.remove(user) {
|
||||
let mut groups = groups.clone();
|
||||
groups.remove(group);
|
||||
if groups.is_empty() {
|
||||
cache.delete_user_group_membership(user, now);
|
||||
} else {
|
||||
cache.add_or_update_user_group_membership(user, &groups, now);
|
||||
}
|
||||
}
|
||||
}
|
||||
for user in missing {
|
||||
let mut groups = reverse.get(user).cloned().unwrap_or_default();
|
||||
groups.insert(group.to_owned());
|
||||
cache.add_or_update_user_group_membership(user, &groups, now);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2283,7 +2267,7 @@ where
|
||||
|
||||
self.cache.with_write_lock(|cache| {
|
||||
let now = OffsetDateTime::now_utc();
|
||||
self.remove_group_from_memberships_map_unlocked(cache, group, now);
|
||||
Self::replace_cached_group_members(cache, group, &[], now);
|
||||
cache.delete_group(group, now);
|
||||
cache.delete_group_policy(group, now);
|
||||
});
|
||||
@@ -2293,12 +2277,8 @@ where
|
||||
|
||||
let gi = m[group].clone();
|
||||
|
||||
self.cache.with_write_lock(|cache| {
|
||||
let now = OffsetDateTime::now_utc();
|
||||
cache.add_or_update_group(group, &gi, now);
|
||||
self.remove_group_from_memberships_map_unlocked(cache, group, now);
|
||||
self.update_group_memberships_map_unlocked(cache, group, &gi, now);
|
||||
});
|
||||
self.cache
|
||||
.with_write_lock(|cache| self.publish_group_unlocked(cache, group, &gi));
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -104,8 +104,12 @@ pub(crate) mod crate_boundary {
|
||||
}
|
||||
|
||||
pub(crate) mod object_store {
|
||||
pub(crate) use super::storage_contracts::{HTTPPreconditions, ListOperations, ObjectInfoOrErr, ObjectOperations};
|
||||
pub(crate) use super::storage_contracts::{
|
||||
HTTPPreconditions, ListOperations, NamespaceLocking, ObjectInfoOrErr, ObjectOperations,
|
||||
};
|
||||
pub(crate) use rustfs_ecstore::api::config::com::read_config_preserve_empty_with_metadata_opts;
|
||||
pub(crate) use rustfs_ecstore::api::object::WriteCommitGuard;
|
||||
pub(crate) use rustfs_ecstore::api::set_disk::get_lock_acquire_timeout;
|
||||
}
|
||||
|
||||
pub(crate) mod runtime {
|
||||
|
||||
@@ -21,6 +21,14 @@ use serde::{Deserialize, Serialize, de::DeserializeOwned};
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use time::OffsetDateTime;
|
||||
|
||||
/// A backend-owned transaction for one named group. Persistence must fail if its lock is lost.
|
||||
#[async_trait::async_trait]
|
||||
pub trait GroupEdit: Send + Sync {
|
||||
async fn save(&self, group: &GroupInfo) -> Result<()>;
|
||||
/// Remove the policy mapping, then the group record, after the caller verified emptiness.
|
||||
async fn delete_empty(&self) -> Result<()>;
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
pub trait Store: Clone + Send + Sync + 'static {
|
||||
fn has_watcher(&self) -> bool;
|
||||
@@ -34,6 +42,11 @@ pub trait Store: Clone + Send + Sync + 'static {
|
||||
async fn delete_iam_config(&self, path: impl AsRef<str> + Send) -> Result<()>;
|
||||
|
||||
async fn save_user_identity(&self, name: &str, user_type: UserType, item: UserIdentity, ttl: Option<usize>) -> Result<()>;
|
||||
/// Create a regular identity or replace one without changing its persisted status.
|
||||
/// Return whether a previous secret was replaced. Backends without conditional writes fail closed.
|
||||
async fn save_regular_user_preserving_status(&self, _name: &str, _item: &UserIdentity) -> Result<bool> {
|
||||
Err(crate::error::Error::IAMActionNotAllowed)
|
||||
}
|
||||
async fn delete_user_identity(&self, name: &str, user_type: UserType) -> Result<()>;
|
||||
async fn load_user_identity(&self, name: &str, user_type: UserType) -> Result<UserIdentity>;
|
||||
|
||||
@@ -41,6 +54,12 @@ pub trait Store: Clone + Send + Sync + 'static {
|
||||
async fn load_users(&self, user_type: UserType, m: &mut HashMap<String, UserIdentity>) -> Result<()>;
|
||||
async fn load_secret_key(&self, name: &str, user_type: UserType) -> Result<String>;
|
||||
|
||||
/// Serialize edits to this group across all writers sharing the backend, then read its
|
||||
/// durable state. Only confirmed absence yields `None`. Keep the returned guard through
|
||||
/// save/delete and local cache publication; dropping it releases the transaction lock.
|
||||
async fn begin_group_edit(&self, _name: &str) -> Result<(Box<dyn GroupEdit>, Option<GroupInfo>)> {
|
||||
Err(crate::error::Error::other("backend does not support serialized group edits"))
|
||||
}
|
||||
async fn save_group_info(&self, name: &str, item: GroupInfo) -> Result<()>;
|
||||
async fn delete_group_info(&self, name: &str) -> Result<()>;
|
||||
async fn load_group(&self, name: &str, m: &mut HashMap<String, GroupInfo>) -> Result<()>;
|
||||
|
||||
+234
-42
@@ -12,8 +12,9 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::{GroupInfo, MappedPolicy, Store, UserType};
|
||||
use super::{GroupEdit, GroupInfo, MappedPolicy, Store, UserType};
|
||||
use crate::error::{Error, Result, is_err_config_not_found, is_err_no_such_group};
|
||||
use crate::storage_api::object_store::WriteCommitGuard;
|
||||
use crate::{
|
||||
IAM_CONFIG_ROOT_PREFIX, IamStorageError, IamStore, classify_iam_system_path_failure_reason, delete_iam_config,
|
||||
read_iam_config_no_lock, read_iam_config_with_metadata, save_iam_config, save_iam_config_with_opts,
|
||||
@@ -41,8 +42,8 @@ use tokio_util::sync::CancellationToken;
|
||||
use tracing::{debug, error, warn};
|
||||
|
||||
use crate::storage_api::object_store::{
|
||||
HTTPPreconditions, ListOperations, ObjectInfoOrErr as StorageObjectInfoOrErr, ObjectOperations,
|
||||
read_config_preserve_empty_with_metadata_opts,
|
||||
HTTPPreconditions, ListOperations, NamespaceLocking, ObjectInfoOrErr as StorageObjectInfoOrErr, ObjectOperations,
|
||||
get_lock_acquire_timeout, read_config_preserve_empty_with_metadata_opts,
|
||||
};
|
||||
|
||||
pub static IAM_CONFIG_PREFIX: LazyLock<String> = LazyLock::new(|| format!("{IAM_CONFIG_ROOT_PREFIX}/iam"));
|
||||
@@ -85,6 +86,29 @@ fn get_group_info_path(group: &str) -> String {
|
||||
path_join_buf(&[&IAM_CONFIG_GROUPS_PREFIX, group, IAM_GROUP_MEMBERS_FILE])
|
||||
}
|
||||
|
||||
fn unchanged_status_precondition(
|
||||
previous: &UserIdentity,
|
||||
item: &UserIdentity,
|
||||
etag: Option<String>,
|
||||
) -> Result<(HTTPPreconditions, bool)> {
|
||||
if previous.credentials.status != item.credentials.status
|
||||
|| previous.credentials.is_temp()
|
||||
|| previous.credentials.is_service_account()
|
||||
{
|
||||
return Err(Error::IAMActionNotAllowed);
|
||||
}
|
||||
let tag = etag
|
||||
.filter(|tag| !tag.is_empty() && tag != "*")
|
||||
.ok_or_else(|| Error::other("missing IAM identity revision"))?;
|
||||
Ok((
|
||||
HTTPPreconditions {
|
||||
if_match: Some(tag),
|
||||
..Default::default()
|
||||
},
|
||||
previous.credentials.secret_key != item.credentials.secret_key,
|
||||
))
|
||||
}
|
||||
|
||||
fn get_policy_doc_path(name: &str) -> String {
|
||||
path_join_buf(&[&IAM_CONFIG_POLICIES_PREFIX, name, IAM_POLICY_FILE])
|
||||
}
|
||||
@@ -143,7 +167,6 @@ const SVC_ACC_LIST_KEY: &str = "service-accounts/";
|
||||
const GROUPS_LIST_KEY: &str = "groups/";
|
||||
const POLICIES_LIST_KEY: &str = "policies/";
|
||||
const STS_LIST_KEY: &str = "sts/";
|
||||
const POLICY_DB_PREFIX: &str = "policydb/";
|
||||
const POLICY_DB_USERS_LIST_KEY: &str = "policydb/users/";
|
||||
const POLICY_DB_STS_USERS_LIST_KEY: &str = "policydb/sts-users/";
|
||||
const POLICY_DB_GROUPS_LIST_KEY: &str = "policydb/groups/";
|
||||
@@ -173,17 +196,6 @@ struct LazyRewriteEntry {
|
||||
static IAM_LAZY_REWRITE_TRACKER: LazyLock<Mutex<HashMap<String, LazyRewriteEntry>>> =
|
||||
LazyLock::new(|| Mutex::new(HashMap::new()));
|
||||
|
||||
// split_path splits a path into a top-level directory and a child item. The
|
||||
// parent directory retains the trailing slash.
|
||||
fn split_path(s: &str, last_index: bool) -> (&str, &str) {
|
||||
let i = if last_index { s.rfind('/') } else { s.find('/') };
|
||||
|
||||
match i {
|
||||
Some(index) => (&s[..index + 1], &s[index + 1..]),
|
||||
None => (s, ""),
|
||||
}
|
||||
}
|
||||
|
||||
/// Attempts to decrypt an at-rest IAM config blob using the same key sources as
|
||||
/// the IAM load path: the RustFS master keys (`RUSTFS_IAM_MASTER_KEY[_OLD_KEYS]`)
|
||||
/// and the MinIO-compatible legacy keys derived from the root credentials
|
||||
@@ -220,6 +232,53 @@ pub struct ObjectStore {
|
||||
object_api: Arc<IamStore>,
|
||||
}
|
||||
|
||||
struct ObjectGroupEdit {
|
||||
api: Arc<IamStore>,
|
||||
path: String,
|
||||
policy_path: String,
|
||||
guard: WriteCommitGuard,
|
||||
}
|
||||
|
||||
impl ObjectGroupEdit {
|
||||
fn fenced(&self, mut opts: IamObjectOptions) -> Result<IamObjectOptions> {
|
||||
if self.guard.is_lock_lost() {
|
||||
return Err(Error::other("group transaction lock lost"));
|
||||
}
|
||||
if let Some(signal) = self.guard.lock_lost_signal() {
|
||||
opts.add_namespace_lock_lost_signal(signal);
|
||||
}
|
||||
Ok(opts)
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl GroupEdit for ObjectGroupEdit {
|
||||
async fn save(&self, group: &GroupInfo) -> Result<()> {
|
||||
let data = ObjectStore::prepare_data_for_storage(&serde_json::to_vec(group)?)?;
|
||||
let opts = self.fenced(IamObjectOptions {
|
||||
max_parity: true,
|
||||
..Default::default()
|
||||
})?;
|
||||
save_iam_config_with_opts(self.api.clone(), &self.path, data, &opts).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn delete_empty(&self) -> Result<()> {
|
||||
for path in [&self.policy_path, &self.path] {
|
||||
let opts = self.fenced(IamObjectOptions {
|
||||
delete_prefix: true,
|
||||
delete_prefix_object: true,
|
||||
..Default::default()
|
||||
})?;
|
||||
match self.api.delete_object(ObjectStore::BUCKET_NAME, path, opts).await {
|
||||
Ok(_) | Err(IamStorageError::FileNotFound | IamStorageError::ObjectNotFound(_, _)) => (),
|
||||
Err(err) => return Err(err.into()),
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl ObjectStore {
|
||||
const BUCKET_NAME: &'static str = ".rustfs.sys";
|
||||
|
||||
@@ -490,36 +549,41 @@ impl ObjectStore {
|
||||
|
||||
#[hotpath::measure]
|
||||
async fn list_all_iamconfig_items(&self) -> Result<HashMap<String, Vec<String>>> {
|
||||
let (tx, mut rx) = mpsc::channel::<StringOrErr>(100);
|
||||
|
||||
let ctx = CancellationToken::new();
|
||||
|
||||
self.list_iam_config_items(format!("{}/", *IAM_CONFIG_PREFIX).as_str(), ctx.clone(), tx)
|
||||
.await;
|
||||
|
||||
let mut res = HashMap::new();
|
||||
let ctx = CancellationToken::new();
|
||||
// Walk only cache partitions. Other children of config/iam (notably
|
||||
// plugin delegations) have independent lifetimes and cardinalities.
|
||||
for key in [
|
||||
USERS_LIST_KEY,
|
||||
GROUPS_LIST_KEY,
|
||||
POLICIES_LIST_KEY,
|
||||
SVC_ACC_LIST_KEY,
|
||||
STS_LIST_KEY,
|
||||
POLICY_DB_USERS_LIST_KEY,
|
||||
POLICY_DB_GROUPS_LIST_KEY,
|
||||
POLICY_DB_STS_USERS_LIST_KEY,
|
||||
] {
|
||||
let (tx, mut rx) = mpsc::channel::<StringOrErr>(100);
|
||||
self.list_iam_config_items(&format!("{}/{key}", *IAM_CONFIG_PREFIX), ctx.clone(), tx)
|
||||
.await;
|
||||
while let Some(v) = rx.recv().await {
|
||||
if let Some(err) = v.err {
|
||||
warn!(
|
||||
path_kind = "iam_config",
|
||||
operation = "list_items",
|
||||
reason = "walk_result",
|
||||
error = %err,
|
||||
"system path list failed"
|
||||
);
|
||||
record_system_path_failure("iam_config", "list_items", "walk_result");
|
||||
ctx.cancel();
|
||||
|
||||
while let Some(v) = rx.recv().await {
|
||||
if let Some(err) = v.err {
|
||||
warn!(
|
||||
path_kind = "iam_config",
|
||||
operation = "list_items",
|
||||
reason = "walk_result",
|
||||
error = %err,
|
||||
"system path list failed"
|
||||
);
|
||||
record_system_path_failure("iam_config", "list_items", "walk_result");
|
||||
ctx.cancel();
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
if let Some(item) = v.item {
|
||||
let last_index = item.starts_with(POLICY_DB_PREFIX);
|
||||
let (list_key, trimmed_item) = split_path(&item, last_index);
|
||||
res.entry(list_key.to_owned())
|
||||
.or_insert_with(Vec::new)
|
||||
.push(trimmed_item.to_owned());
|
||||
if let Some(item) = v.item {
|
||||
res.entry(key.to_owned()).or_insert_with(Vec::new).push(item);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -904,6 +968,34 @@ impl Store for ObjectStore {
|
||||
e
|
||||
})
|
||||
}
|
||||
async fn save_regular_user_preserving_status(&self, name: &str, item: &UserIdentity) -> Result<bool> {
|
||||
let path = get_user_identity_path(name, UserType::Reg);
|
||||
let existing =
|
||||
read_config_preserve_empty_with_metadata_opts(self.object_api.clone(), &path, &LoadMode::Locked.read_opts()).await;
|
||||
let (conditions, rotated) = match existing.map_err(Error::from) {
|
||||
Ok((data, metadata)) => {
|
||||
let previous: UserIdentity = Self::decode_iam_config_strict(&data)?;
|
||||
unchanged_status_precondition(&previous, item, metadata.etag)?
|
||||
}
|
||||
Err(err) if is_err_config_not_found(&err) => (
|
||||
HTTPPreconditions {
|
||||
if_none_match: Some("*".into()),
|
||||
..Default::default()
|
||||
},
|
||||
false,
|
||||
),
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
let opts = IamObjectOptions {
|
||||
max_parity: true,
|
||||
http_preconditions: Some(conditions),
|
||||
..Default::default()
|
||||
};
|
||||
let data = Self::prepare_data_for_storage(&serde_json::to_vec(item)?)?;
|
||||
save_iam_config_with_opts(self.object_api.clone(), &path, data, &opts).await?;
|
||||
Ok(rotated)
|
||||
}
|
||||
|
||||
async fn delete_user_identity(&self, name: &str, user_type: UserType) -> Result<()> {
|
||||
self.delete_iam_config(get_user_identity_path(name, user_type))
|
||||
.await
|
||||
@@ -960,6 +1052,28 @@ impl Store for ObjectStore {
|
||||
Ok(u.credentials.secret_key)
|
||||
}
|
||||
|
||||
async fn begin_group_edit(&self, name: &str) -> Result<(Box<dyn GroupEdit>, Option<GroupInfo>)> {
|
||||
let path = get_group_info_path(name);
|
||||
// Take the transaction lock before any object locks. A distinct key lets ordinary
|
||||
// config reads, writes and deletes keep taking their own namespace locks.
|
||||
let lock = self.object_api.new_ns_lock(Self::BUCKET_NAME, &format!("{path}.txn")).await?;
|
||||
let guard = WriteCommitGuard::acquire(&lock, get_lock_acquire_timeout())
|
||||
.await
|
||||
.map_err(Error::other)?;
|
||||
let group = match self.load_iam_config_strict(&path).await {
|
||||
Ok(gi) => Some(gi),
|
||||
Err(Error::ConfigNotFound) => None,
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
let edit = ObjectGroupEdit {
|
||||
api: self.object_api.clone(),
|
||||
path,
|
||||
policy_path: get_mapped_policy_path(name, UserType::Reg, true),
|
||||
guard,
|
||||
};
|
||||
Ok((Box::new(edit), group))
|
||||
}
|
||||
|
||||
async fn save_group_info(&self, name: &str, item: GroupInfo) -> Result<()> {
|
||||
self.save_iam_config(item, get_group_info_path(name)).await
|
||||
}
|
||||
@@ -1413,6 +1527,12 @@ mod tests {
|
||||
)
|
||||
.await;
|
||||
let store = ObjectStore::new(env.ecstore);
|
||||
store
|
||||
.save_iam_config("fixture", "config/iam/plugin-delegations/v1/fixture")
|
||||
.await
|
||||
.expect("seed unrelated partition");
|
||||
let listed = store.list_all_iamconfig_items().await.expect("list IAM partitions");
|
||||
assert!(!listed.contains_key("plugin-delegations/"));
|
||||
store
|
||||
.save_group_info("reload-group", GroupInfo::new(vec!["reload-user".to_string()]))
|
||||
.await
|
||||
@@ -1504,6 +1624,78 @@ mod tests {
|
||||
assert!(matches!(err, Error::NoSuchUser(name) if name == "missing-sts"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn restricted_user_upsert_requires_unchanged_status_and_exact_revision() {
|
||||
let old = super::UserIdentity::from(Credentials {
|
||||
status: "off".into(),
|
||||
secret_key: "old-fixture".into(),
|
||||
..Default::default()
|
||||
});
|
||||
let mut new = old.clone();
|
||||
new.credentials.secret_key = "new-fixture".into();
|
||||
let (predicate, rotated) =
|
||||
super::unchanged_status_precondition(&old, &new, Some("rev1".into())).expect("status remains disabled");
|
||||
assert!(rotated);
|
||||
assert_eq!(predicate.if_match.as_deref(), Some("rev1"));
|
||||
assert!(super::unchanged_status_precondition(&old, &new, None).is_err());
|
||||
new.credentials.status = "on".into();
|
||||
assert!(matches!(
|
||||
super::unchanged_status_precondition(&old, &new, Some("rev1".into())),
|
||||
Err(crate::error::Error::IAMActionNotAllowed)
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn restricted_user_upsert_checks_durable_status_before_replacing_credentials() {
|
||||
use crate::store::{Store, UserType};
|
||||
let _credentials = test_cred();
|
||||
let temp = tempfile::tempdir().expect("IAM fixture directory");
|
||||
let env = Box::pin(
|
||||
rustfs_test_utils::TestECStoreEnv::builder()
|
||||
.base_dir(temp.path())
|
||||
.init_bucket_metadata(false)
|
||||
.build(),
|
||||
)
|
||||
.await;
|
||||
let store = ObjectStore::new(env.ecstore);
|
||||
let name = "status-fixture";
|
||||
let mut entry = super::UserIdentity::from(Credentials {
|
||||
access_key: name.into(),
|
||||
status: "off".into(),
|
||||
secret_key: "first-fixture".into(),
|
||||
..Default::default()
|
||||
});
|
||||
assert!(
|
||||
!store
|
||||
.save_regular_user_preserving_status(name, &entry)
|
||||
.await
|
||||
.expect("create absent account")
|
||||
);
|
||||
entry.credentials.status = "on".into();
|
||||
entry.credentials.secret_key = "second-fixture".into();
|
||||
assert!(matches!(
|
||||
store.save_regular_user_preserving_status(name, &entry).await,
|
||||
Err(Error::IAMActionNotAllowed)
|
||||
));
|
||||
let persisted = store
|
||||
.load_user_identity(name, UserType::Reg)
|
||||
.await
|
||||
.expect("reload denied write");
|
||||
assert_eq!(persisted.credentials.status, "off");
|
||||
assert_eq!(persisted.credentials.secret_key, "first-fixture");
|
||||
entry.credentials.status = "off".into();
|
||||
assert!(
|
||||
store
|
||||
.save_regular_user_preserving_status(name, &entry)
|
||||
.await
|
||||
.expect("rotate while disabled")
|
||||
);
|
||||
let persisted = store.load_user_identity(name, UserType::Reg).await.expect("reload rotation");
|
||||
assert_eq!(persisted.credentials.status, "off");
|
||||
assert_eq!(persisted.credentials.secret_key, "second-fixture");
|
||||
}
|
||||
|
||||
fn test_cred() -> Credentials {
|
||||
if let Some(cred) = crate::root_credentials::credentials() {
|
||||
return cred;
|
||||
|
||||
+461
-67
@@ -76,6 +76,8 @@ pub const STATUS_ENABLED: &str = "enabled";
|
||||
pub const STATUS_DISABLED: &str = "disabled";
|
||||
|
||||
pub const POLICYNAME: &str = "policy";
|
||||
/// Signed subset of the OIDC policy selection granted independently of IAM mappings.
|
||||
pub const OIDC_EXPLICIT_POLICIES_CLAIM: &str = "x-rustfs-internal-oidc-explicit-policies";
|
||||
pub const PLUGIN_ATTEMPT_CREDENTIAL_CLAIM: &str = "rustfs:plugin-attempt-v1";
|
||||
pub const SESSION_POLICY_NAME: &str = "sessionPolicy";
|
||||
pub const SESSION_POLICY_NAME_EXTRACTED: &str = "sessionPolicy-extracted";
|
||||
@@ -616,6 +618,9 @@ impl<T: Store> IamSys<T> {
|
||||
/// replicated edit's source time) instead of the local clock; see
|
||||
/// `IamCache::set_policy_at` (backlog#2291).
|
||||
pub async fn set_policy_at(&self, name: &str, policy: Policy, updated_at: OffsetDateTime) -> Result<OffsetDateTime> {
|
||||
if name.ends_with(rustfs_utils::path::GLOBAL_DIR_SUFFIX) {
|
||||
return Err(IamError::InvalidArgument);
|
||||
}
|
||||
let updated_at = self.store.set_policy_at(name, policy, updated_at).await?;
|
||||
|
||||
if !self.has_watcher() {
|
||||
@@ -975,6 +980,7 @@ impl<T: Store> IamSys<T> {
|
||||
) -> Result<Credentials> {
|
||||
if !opts.access_key.is_empty()
|
||||
&& (opts.access_key.trim() == "."
|
||||
|| opts.access_key.ends_with(rustfs_utils::path::GLOBAL_DIR_SUFFIX)
|
||||
|| opts.access_key.trim() == ".."
|
||||
|| opts.access_key.chars().any(|ch| ch.is_control() || matches!(ch, '/' | '\\')))
|
||||
{
|
||||
@@ -1291,6 +1297,22 @@ impl<T: Store> IamSys<T> {
|
||||
access_key: &str,
|
||||
args: &AddOrUpdateUserReq,
|
||||
updated_at: OffsetDateTime,
|
||||
) -> Result<OffsetDateTime> {
|
||||
self.upsert_regular_user(access_key, args, updated_at, true).await
|
||||
}
|
||||
|
||||
/// Create a regular user, or update its credentials without changing the stored account status.
|
||||
pub async fn create_user_preserving_status(&self, access_key: &str, args: &AddOrUpdateUserReq) -> Result<OffsetDateTime> {
|
||||
self.upsert_regular_user(access_key, args, OffsetDateTime::now_utc(), false)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn upsert_regular_user(
|
||||
&self,
|
||||
access_key: &str,
|
||||
args: &AddOrUpdateUserReq,
|
||||
updated_at: OffsetDateTime,
|
||||
allow_status_change: bool,
|
||||
) -> Result<OffsetDateTime> {
|
||||
let runtime = tokio::runtime::Handle::try_current().map_err(Error::other)?;
|
||||
let iam = self.clone();
|
||||
@@ -1298,7 +1320,10 @@ impl<T: Store> IamSys<T> {
|
||||
let args = args.clone();
|
||||
#[cfg(test)]
|
||||
let notification_probe = crate::LOAD_USER_NOTIFICATION_PROBE.try_with(Arc::clone).ok();
|
||||
let operation = async move { iam.create_user_at_inner(&access_key, &args, updated_at).await };
|
||||
let operation = async move {
|
||||
iam.create_user_at_inner(&access_key, &args, updated_at, allow_status_change)
|
||||
.await
|
||||
};
|
||||
#[cfg(test)]
|
||||
let task = runtime.spawn(async move {
|
||||
if let Some(probe) = notification_probe {
|
||||
@@ -1316,12 +1341,13 @@ impl<T: Store> IamSys<T> {
|
||||
access_key: &str,
|
||||
args: &AddOrUpdateUserReq,
|
||||
updated_at: OffsetDateTime,
|
||||
allow_status_change: bool,
|
||||
) -> Result<OffsetDateTime> {
|
||||
if !is_access_key_valid(access_key) {
|
||||
return Err(IamError::InvalidAccessKeyLength);
|
||||
}
|
||||
|
||||
if contains_reserved_chars(access_key) {
|
||||
if contains_reserved_chars(access_key) || access_key.ends_with(rustfs_utils::path::GLOBAL_DIR_SUFFIX) {
|
||||
return Err(IamError::ContainsReservedChars);
|
||||
}
|
||||
|
||||
@@ -1331,7 +1357,7 @@ impl<T: Store> IamSys<T> {
|
||||
|
||||
let (updated_at, secret_changed, parent_guard) = self
|
||||
.store
|
||||
.add_user_at_with_secret_change(access_key, args, updated_at)
|
||||
.add_user_at_with_secret_change(access_key, args, updated_at, allow_status_change)
|
||||
.await?;
|
||||
if secret_changed {
|
||||
self.revoke_sts_sessions_for_parent(access_key).await?;
|
||||
@@ -1495,7 +1521,7 @@ impl<T: Store> IamSys<T> {
|
||||
users: Vec<String>,
|
||||
updated_at: OffsetDateTime,
|
||||
) -> Result<OffsetDateTime> {
|
||||
if contains_reserved_chars(group) {
|
||||
if contains_reserved_chars(group) || group.ends_with(rustfs_utils::path::GLOBAL_DIR_SUFFIX) {
|
||||
return Err(IamError::GroupNameContainsReservedChars);
|
||||
}
|
||||
let updated_at = self.store.add_users_to_group_at(group, users, updated_at).await?;
|
||||
@@ -1666,16 +1692,10 @@ impl<T: Store> IamSys<T> {
|
||||
parent_user: &str,
|
||||
groups: &Option<Vec<String>>,
|
||||
) -> Result<Option<HashMap<String, Value>>> {
|
||||
let Some(policy_names) = self.verified_federated_policy_names(claims, parent_user, groups).await? else {
|
||||
let Some((resolved, _)) = self.verified_federated_policy(claims, parent_user, groups).await? else {
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
let (resolved, _) = self.store.merge_policies(&policy_names.join(",")).await;
|
||||
let resolved_names = MappedPolicy::new(&resolved).to_slice();
|
||||
if policy_names.iter().any(|policy_name| !resolved_names.contains(policy_name)) {
|
||||
return Err(IamError::InvalidArgument);
|
||||
}
|
||||
|
||||
let boundary = match claims.get(SESSION_POLICY_NAME) {
|
||||
Some(Value::String(encoded)) => {
|
||||
decode_session_policy(encoded)?;
|
||||
@@ -1692,6 +1712,9 @@ impl<T: Store> IamSys<T> {
|
||||
if claims.get(OIDC_VIRTUAL_PARENT_CLAIM).and_then(Value::as_str) == Some(parent_user) {
|
||||
verified_claims.insert(OIDC_VIRTUAL_PARENT_CLAIM.to_string(), Value::String(parent_user.to_string()));
|
||||
}
|
||||
if let Some(explicit) = claims.get(OIDC_EXPLICIT_POLICIES_CLAIM) {
|
||||
verified_claims.insert(OIDC_EXPLICIT_POLICIES_CLAIM.to_owned(), explicit.clone());
|
||||
}
|
||||
if let Some(boundary) = boundary {
|
||||
verified_claims.insert(FEDERATED_POLICY_BOUNDARY_CLAIM.to_string(), Value::String(boundary));
|
||||
}
|
||||
@@ -1873,20 +1896,24 @@ impl<T: Store> IamSys<T> {
|
||||
Ok(policy_names)
|
||||
}
|
||||
|
||||
fn signed_oidc_policy_names(claims: &HashMap<String, Value>, parent_user: &str) -> Result<Vec<String>> {
|
||||
let policy_names = Self::safe_claim_policy_names(claims, parent_user);
|
||||
if policy_names.is_empty() {
|
||||
async fn resolve_oidc_policy(&self, names: &[String]) -> Result<(String, Policy)> {
|
||||
if names.is_empty() || names.iter().any(|name| !is_safe_claim_policy_name(name)) {
|
||||
return Err(IamError::InvalidArgument);
|
||||
}
|
||||
Ok(policy_names)
|
||||
let (resolved, policy) = self.store.merge_policies(&names.join(",")).await;
|
||||
let resolved_names = MappedPolicy::new(&resolved).to_slice();
|
||||
if names.iter().any(|name| !resolved_names.contains(name)) {
|
||||
return Err(IamError::InvalidArgument);
|
||||
}
|
||||
Ok((resolved, policy))
|
||||
}
|
||||
|
||||
async fn verified_federated_policy_names(
|
||||
async fn verified_federated_policy(
|
||||
&self,
|
||||
claims: &HashMap<String, Value>,
|
||||
parent_user: &str,
|
||||
groups: &Option<Vec<String>>,
|
||||
) -> Result<Option<Vec<String>>> {
|
||||
) -> Result<Option<(String, Policy)>> {
|
||||
if !is_rustfs_oidc_claims(claims) {
|
||||
return Ok(None);
|
||||
}
|
||||
@@ -1894,12 +1921,72 @@ impl<T: Store> IamSys<T> {
|
||||
return Err(IamError::InvalidArgument);
|
||||
}
|
||||
|
||||
let policy_names = if is_verified_federated_session(claims, parent_user) {
|
||||
Self::signed_oidc_policy_names(claims, parent_user)?
|
||||
} else {
|
||||
self.oidc_policy_names(claims, parent_user, groups).await?
|
||||
let explicit = claims.get(OIDC_EXPLICIT_POLICIES_CLAIM);
|
||||
if explicit.is_none() {
|
||||
// Earlier grouped credentials cannot distinguish an explicit Deny from a detached grant.
|
||||
if claims.contains_key("groups") || groups.as_ref().is_some_and(|groups| !groups.is_empty()) {
|
||||
return Err(IamError::InvalidArgument);
|
||||
}
|
||||
if !is_verified_federated_session(claims, parent_user) && !has_verified_federated_policy(claims) {
|
||||
let names = self.oidc_policy_names(claims, parent_user, groups).await?;
|
||||
return self.resolve_oidc_policy(&names).await.map(Some);
|
||||
}
|
||||
}
|
||||
let issued: Vec<String> = claims
|
||||
.get(POLICYNAME)
|
||||
.and_then(Value::as_str)
|
||||
.ok_or(IamError::InvalidArgument)?
|
||||
.split(',')
|
||||
.map(str::trim)
|
||||
.map(str::to_owned)
|
||||
.collect();
|
||||
if issued.iter().any(|name| !is_safe_claim_policy_name(name)) {
|
||||
return Err(IamError::InvalidArgument);
|
||||
}
|
||||
let Some(explicit) = explicit else {
|
||||
return self.resolve_oidc_policy(&issued).await.map(Some);
|
||||
};
|
||||
Ok(Some(policy_names))
|
||||
if !is_verified_federated_session(claims, parent_user) {
|
||||
return Err(IamError::InvalidArgument);
|
||||
}
|
||||
let explicit: HashSet<&str> = explicit
|
||||
.as_array()
|
||||
.ok_or(IamError::InvalidArgument)?
|
||||
.iter()
|
||||
.map(|v| {
|
||||
v.as_str()
|
||||
.filter(|name| issued.iter().any(|n| n == name))
|
||||
.ok_or(IamError::InvalidArgument)
|
||||
})
|
||||
.collect::<Result<_>>()?;
|
||||
// Use the issuer's signed group list, including on replicated and derived credentials.
|
||||
let signed_groups: Option<Vec<String>> = claims
|
||||
.get("groups")
|
||||
.map(serde::Deserialize::deserialize)
|
||||
.transpose()
|
||||
.map_err(|_| IamError::InvalidArgument)?;
|
||||
if signed_groups.is_none() && groups.as_ref().is_some_and(|groups| !groups.is_empty()) {
|
||||
return Err(IamError::InvalidArgument);
|
||||
}
|
||||
let current = self.store.sts_policy_db_get(parent_user, &signed_groups).await?;
|
||||
let active: Vec<_> = issued
|
||||
.iter()
|
||||
.filter(|name| explicit.contains(name.as_str()) || current.contains(name))
|
||||
.cloned()
|
||||
.collect();
|
||||
let (resolved, mut policy) = self.resolve_oidc_policy(&active).await?;
|
||||
// Later attachments can restrict the credential, but cannot widen its issuance ceiling.
|
||||
let new: Vec<_> = current.into_iter().filter(|name| !issued.contains(name)).collect();
|
||||
if !new.is_empty() {
|
||||
let (_, restrictions) = self.resolve_oidc_policy(&new).await?;
|
||||
policy.statements.extend(
|
||||
restrictions
|
||||
.statements
|
||||
.into_iter()
|
||||
.filter(|s| s.effect == rustfs_policy::policy::Effect::Deny),
|
||||
);
|
||||
}
|
||||
Ok(Some((resolved, policy)))
|
||||
}
|
||||
|
||||
/// Compatibility wrapper for service-account authorization entry points.
|
||||
@@ -1929,6 +2016,9 @@ impl<T: Store> IamSys<T> {
|
||||
let state = Self::policy_plugin_state().await;
|
||||
if marked {
|
||||
if args.is_owner
|
||||
// Plugin-attempt sessions are confined to their allowlisted work;
|
||||
// they cannot use a default permission to manage the parent identity.
|
||||
|| self_service_deny_check(args)
|
||||
|| args.claims.get(PLUGIN_ATTEMPT_CREDENTIAL_CLAIM) != Some(&Value::Bool(true))
|
||||
|| !matches!(state, PolicyPluginState::Disabled)
|
||||
{
|
||||
@@ -2077,10 +2167,10 @@ impl<T: Store> IamSys<T> {
|
||||
session_policy,
|
||||
} => {
|
||||
let session_ok = evaluate_prepared_session_policy(session_policy, args).await;
|
||||
if let Some(ok) = session_ok {
|
||||
return Ok(ok && (*is_owner || combined_policy.is_allowed(args).await));
|
||||
if session_ok == Some(false) && is_rustfs_oidc_claims(args.claims) {
|
||||
return Err(Error::IAMActionNotAllowed);
|
||||
}
|
||||
*is_owner || combined_policy.is_allowed(args).await
|
||||
session_ok != Some(false) && (*is_owner || derived_policy_allowed(combined_policy, args).await?)
|
||||
}
|
||||
PreparedIamMode::ServiceAccount {
|
||||
is_owner,
|
||||
@@ -2101,12 +2191,18 @@ impl<T: Store> IamSys<T> {
|
||||
true
|
||||
};
|
||||
|
||||
if !boundary_allowed {
|
||||
return Err(Error::IAMActionNotAllowed);
|
||||
}
|
||||
let parent_allowed =
|
||||
(*bypass_parent_policy || *is_owner || combined_policy.is_allowed(&parent_args).await) && boundary_allowed;
|
||||
*bypass_parent_policy || *is_owner || derived_policy_allowed(combined_policy, &parent_args).await?;
|
||||
match mode {
|
||||
PreparedServicePolicyMode::Inherited => parent_allowed,
|
||||
PreparedServicePolicyMode::SessionBound => {
|
||||
let session_ok = evaluate_prepared_session_policy(session_policy, args).await;
|
||||
if session_ok == Some(false) && is_rustfs_oidc_claims(args.claims) {
|
||||
return Err(Error::IAMActionNotAllowed);
|
||||
}
|
||||
if let Some(ok) = session_ok {
|
||||
return Ok(ok && parent_allowed);
|
||||
}
|
||||
@@ -2126,9 +2222,24 @@ impl<T: Store> IamSys<T> {
|
||||
};
|
||||
|
||||
if policies.is_empty() {
|
||||
// The policy lookup also returns an empty set for an inactive user.
|
||||
// Only a live identity gets the default for its own credential changes.
|
||||
let mode = if self_service_deny_check(args)
|
||||
&& self
|
||||
.store
|
||||
.get_user(args.account)
|
||||
.await
|
||||
.is_some_and(|u| u.credentials.is_valid())
|
||||
{
|
||||
PreparedIamMode::Regular {
|
||||
combined_policy: Policy::default(),
|
||||
}
|
||||
} else {
|
||||
PreparedIamMode::Deny
|
||||
};
|
||||
return PreparedIamAuth {
|
||||
needs_existing_object_tag: false,
|
||||
mode: PreparedIamMode::Deny,
|
||||
mode,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -2155,24 +2266,19 @@ impl<T: Store> IamSys<T> {
|
||||
}
|
||||
|
||||
pub(crate) async fn prepare_sts_auth(&self, args: &Args<'_>, parent_user: &str) -> PreparedIamAuth {
|
||||
let federated_policy_names = match self
|
||||
.verified_federated_policy_names(args.claims, parent_user, args.groups)
|
||||
.await
|
||||
{
|
||||
Ok(policy_names) => policy_names,
|
||||
let federated_policy = match self.verified_federated_policy(args.claims, parent_user, args.groups).await {
|
||||
Ok(policy) => policy,
|
||||
Err(_) => {
|
||||
return PreparedIamAuth {
|
||||
needs_existing_object_tag: false,
|
||||
mode: PreparedIamMode::Deny,
|
||||
mode: PreparedIamMode::TerminalDeny,
|
||||
};
|
||||
}
|
||||
};
|
||||
let federated_session = federated_policy_names.is_some();
|
||||
let verified_federated_session = is_verified_federated_session(args.claims, parent_user);
|
||||
let is_owner = !is_rustfs_oidc_claims(args.claims) && crate::is_root_access_key(parent_user);
|
||||
let role_arn = args.get_role_arn();
|
||||
|
||||
let (effective_groups, groups_source, policies) = if federated_session {
|
||||
let (effective_groups, groups_source, mut policy_names) = if federated_policy.is_some() {
|
||||
(args.groups.clone(), "oidc_claims", Vec::new())
|
||||
} else if is_owner {
|
||||
(None, "owner", Vec::new())
|
||||
@@ -2204,21 +2310,27 @@ impl<T: Store> IamSys<T> {
|
||||
}
|
||||
},
|
||||
};
|
||||
let p = self.policy_db_get(parent_user, &effective_groups).await.unwrap_or_default();
|
||||
let p = match self.policy_db_get(parent_user, &effective_groups).await {
|
||||
Ok(policies) => policies,
|
||||
Err(_) if self_service_deny_check(args) => {
|
||||
return PreparedIamAuth {
|
||||
needs_existing_object_tag: false,
|
||||
mode: PreparedIamMode::Deny,
|
||||
};
|
||||
}
|
||||
Err(_) => Vec::new(),
|
||||
};
|
||||
(effective_groups, groups_source, p)
|
||||
};
|
||||
|
||||
let mut policy_names = if let Some(policy_names) = federated_policy_names {
|
||||
policy_names
|
||||
} else {
|
||||
policies
|
||||
};
|
||||
if !is_owner && policy_names.is_empty() && !is_rustfs_oidc_claims(args.claims) {
|
||||
policy_names = Self::safe_claim_policy_names(args.claims, parent_user);
|
||||
}
|
||||
|
||||
let mut combined_policy = Policy::default();
|
||||
if !is_owner {
|
||||
if let Some((_, policy)) = federated_policy {
|
||||
combined_policy = policy;
|
||||
} else if !is_owner {
|
||||
if policy_names.is_empty() {
|
||||
if args.deny_only {
|
||||
combined_policy = Policy::default();
|
||||
@@ -2237,7 +2349,7 @@ impl<T: Store> IamSys<T> {
|
||||
requested_policies = %requested_policies,
|
||||
"prepare_sts_auth: no STS policy names resolved"
|
||||
);
|
||||
if args.deny_only && !verified_federated_session {
|
||||
if args.deny_only && !self_service_deny_check(args) {
|
||||
combined_policy = Policy::default();
|
||||
} else {
|
||||
return PreparedIamAuth {
|
||||
@@ -2251,7 +2363,7 @@ impl<T: Store> IamSys<T> {
|
||||
.iter()
|
||||
.any(|policy_name| !resolved_policy_names.iter().any(|resolved| resolved == policy_name));
|
||||
if has_unresolved_policy_names {
|
||||
if verified_federated_session {
|
||||
if self_service_deny_check(args) {
|
||||
return PreparedIamAuth {
|
||||
needs_existing_object_tag: false,
|
||||
mode: PreparedIamMode::Deny,
|
||||
@@ -2328,14 +2440,14 @@ impl<T: Store> IamSys<T> {
|
||||
if is_rustfs_oidc_claims(args.claims) && !has_verified_federated_policy {
|
||||
return PreparedIamAuth {
|
||||
needs_existing_object_tag: false,
|
||||
mode: PreparedIamMode::Deny,
|
||||
mode: PreparedIamMode::TerminalDeny,
|
||||
};
|
||||
}
|
||||
let federated_boundary = if has_verified_federated_policy {
|
||||
if !is_valid_verified_federated_policy(args.claims, parent_user) {
|
||||
return PreparedIamAuth {
|
||||
needs_existing_object_tag: false,
|
||||
mode: PreparedIamMode::Deny,
|
||||
mode: PreparedIamMode::TerminalDeny,
|
||||
};
|
||||
}
|
||||
match federated_policy_boundary(args.claims) {
|
||||
@@ -2343,7 +2455,7 @@ impl<T: Store> IamSys<T> {
|
||||
Err(_) => {
|
||||
return PreparedIamAuth {
|
||||
needs_existing_object_tag: false,
|
||||
mode: PreparedIamMode::Deny,
|
||||
mode: PreparedIamMode::TerminalDeny,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -2362,10 +2474,8 @@ impl<T: Store> IamSys<T> {
|
||||
let is_owner = !is_rustfs_oidc_claims(args.claims) && crate::is_root_access_key(parent_user);
|
||||
let role_arn = args.get_role_arn();
|
||||
|
||||
let svc_policies = if is_owner || bypass_parent_policy {
|
||||
let svc_policies = if is_owner || bypass_parent_policy || has_verified_federated_policy {
|
||||
Vec::new()
|
||||
} else if has_verified_federated_policy {
|
||||
Self::safe_claim_policy_names(args.claims, parent_user)
|
||||
} else if role_arn.is_some() {
|
||||
let Ok(arn) = ARN::parse(role_arn.unwrap_or_default()) else {
|
||||
tracing::warn!(
|
||||
@@ -2389,7 +2499,7 @@ impl<T: Store> IamSys<T> {
|
||||
policies
|
||||
};
|
||||
|
||||
if !is_owner && !bypass_parent_policy && svc_policies.is_empty() {
|
||||
if !is_owner && !bypass_parent_policy && !has_verified_federated_policy && svc_policies.is_empty() {
|
||||
return PreparedIamAuth {
|
||||
needs_existing_object_tag: false,
|
||||
mode: PreparedIamMode::Deny,
|
||||
@@ -2398,12 +2508,17 @@ impl<T: Store> IamSys<T> {
|
||||
|
||||
let combined_policy = if is_owner || bypass_parent_policy {
|
||||
Policy::default()
|
||||
} else if has_verified_federated_policy {
|
||||
let Ok(Some((_, policy))) = self.verified_federated_policy(args.claims, parent_user, args.groups).await else {
|
||||
return PreparedIamAuth {
|
||||
needs_existing_object_tag: false,
|
||||
mode: PreparedIamMode::TerminalDeny,
|
||||
};
|
||||
};
|
||||
policy
|
||||
} else {
|
||||
let (a, c) = self.store.merge_policies(&svc_policies.join(",")).await;
|
||||
let resolved = MappedPolicy::new(&a).to_slice();
|
||||
if a.is_empty()
|
||||
|| has_verified_federated_policy && svc_policies.iter().any(|policy_name| !resolved.contains(policy_name))
|
||||
{
|
||||
if a.is_empty() {
|
||||
return PreparedIamAuth {
|
||||
needs_existing_object_tag: false,
|
||||
mode: PreparedIamMode::Deny,
|
||||
@@ -2474,8 +2589,26 @@ async fn prepared_session_policy_needs_existing_object_tag_for_args(policy: &Pre
|
||||
}
|
||||
}
|
||||
|
||||
fn self_service_deny_check(args: &Args<'_>) -> bool {
|
||||
use rustfs_policy::policy::action::{Action, AdminAction};
|
||||
args.deny_only
|
||||
&& matches!(
|
||||
args.action,
|
||||
Action::AdminAction(
|
||||
AdminAction::ChangeOwnPasswordAdminAction
|
||||
| AdminAction::CreateUserAdminAction
|
||||
| AdminAction::ManageOwnMFAAdminAction
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
fn prepare_session_policy(args: &Args<'_>, empty_is_none: bool) -> PreparedSessionPolicy {
|
||||
let Some(policy_str) = extract_session_policy_text(args.claims) else {
|
||||
if self_service_deny_check(args)
|
||||
&& (args.claims.contains_key(SESSION_POLICY_NAME) || args.claims.contains_key(SESSION_POLICY_NAME_EXTRACTED))
|
||||
{
|
||||
return PreparedSessionPolicy::DenyAll;
|
||||
}
|
||||
return PreparedSessionPolicy::None;
|
||||
};
|
||||
|
||||
@@ -2548,6 +2681,7 @@ pub fn remove_verified_federated_policy(claims: &mut HashMap<String, Value>) ->
|
||||
let removed = claims.remove(VERIFIED_FEDERATED_POLICY_CLAIM).is_some();
|
||||
claims.remove(FEDERATED_POLICY_BOUNDARY_CLAIM);
|
||||
claims.remove(OIDC_VIRTUAL_PARENT_CLAIM);
|
||||
claims.remove(OIDC_EXPLICIT_POLICIES_CLAIM);
|
||||
removed
|
||||
}
|
||||
|
||||
@@ -2556,6 +2690,10 @@ fn merge_derived_service_account_claims(
|
||||
source_claims: &HashMap<String, Value>,
|
||||
verified_federated_policy: Option<HashMap<String, Value>>,
|
||||
) -> Result<()> {
|
||||
if verified_federated_policy.is_some() {
|
||||
// Only the parent's signed groups may participate in federated policy resolution.
|
||||
target_claims.remove("groups");
|
||||
}
|
||||
for (key, value) in source_claims {
|
||||
if key != "exp" {
|
||||
target_claims.insert(key.clone(), value.clone());
|
||||
@@ -2622,6 +2760,24 @@ fn extract_session_policy_text(claims: &HashMap<String, Value>) -> Option<String
|
||||
String::from_utf8(bytes).ok()
|
||||
}
|
||||
|
||||
async fn derived_policy_allowed(policy: &Policy, args: &Args<'_>) -> Result<bool> {
|
||||
let allowed = policy.is_allowed(args).await;
|
||||
// A bucket policy may supplement a missing Allow, but not override a live OIDC Deny.
|
||||
if !allowed
|
||||
&& is_rustfs_oidc_claims(args.claims)
|
||||
&& (args.deny_only
|
||||
|| !policy
|
||||
.is_allowed(&Args {
|
||||
deny_only: true,
|
||||
..args.clone()
|
||||
})
|
||||
.await)
|
||||
{
|
||||
return Err(Error::IAMActionNotAllowed);
|
||||
}
|
||||
Ok(allowed)
|
||||
}
|
||||
|
||||
async fn evaluate_prepared_session_policy(policy: &PreparedSessionPolicy, args: &Args<'_>) -> Option<bool> {
|
||||
match policy {
|
||||
PreparedSessionPolicy::None => None,
|
||||
@@ -2708,7 +2864,7 @@ mod tests {
|
||||
use crate::cache::{Cache, CacheEntity};
|
||||
use crate::error::Error;
|
||||
use crate::manager::get_default_policies;
|
||||
use crate::store::{GroupInfo, MappedPolicy, Store, UserType};
|
||||
use crate::store::{GroupEdit, GroupInfo, MappedPolicy, Store, UserType};
|
||||
use rustfs_credentials::{Credentials, init_global_action_credentials};
|
||||
use rustfs_policy::auth::{UserIdentity, get_new_credentials_with_metadata};
|
||||
use rustfs_policy::policy::Args;
|
||||
@@ -3444,10 +3600,30 @@ mod tests {
|
||||
derived_save_started: Arc<tokio::sync::Notify>,
|
||||
release_derived_save: Arc<tokio::sync::Notify>,
|
||||
block_group_save: Arc<std::sync::atomic::AtomicBool>,
|
||||
group_locks: Arc<Mutex<HashMap<String, Arc<tokio::sync::Mutex<()>>>>>,
|
||||
saved_groups: Arc<Mutex<HashMap<String, GroupInfo>>>,
|
||||
group_save_started: Arc<tokio::sync::Notify>,
|
||||
group_save_release: Arc<tokio::sync::Notify>,
|
||||
}
|
||||
|
||||
struct MockGroupEdit {
|
||||
store: StsTestMockStore,
|
||||
name: String,
|
||||
_guard: tokio::sync::OwnedMutexGuard<()>,
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl GroupEdit for MockGroupEdit {
|
||||
async fn save(&self, group: &GroupInfo) -> Result<()> {
|
||||
self.store.save_group_info(&self.name, group.clone()).await
|
||||
}
|
||||
|
||||
async fn delete_empty(&self) -> Result<()> {
|
||||
self.store.delete_mapped_policy(&self.name, UserType::Reg, true).await?;
|
||||
self.store.delete_group_info(&self.name).await
|
||||
}
|
||||
}
|
||||
|
||||
impl StsTestMockStore {
|
||||
fn new(empty_policies: bool) -> Self {
|
||||
Self {
|
||||
@@ -3468,6 +3644,12 @@ mod tests {
|
||||
derived_save_started: Arc::new(tokio::sync::Notify::new()),
|
||||
release_derived_save: Arc::new(tokio::sync::Notify::new()),
|
||||
block_group_save: Arc::new(std::sync::atomic::AtomicBool::new(false)),
|
||||
group_locks: Default::default(),
|
||||
saved_groups: Arc::new(Mutex::new(if empty_policies {
|
||||
HashMap::new()
|
||||
} else {
|
||||
HashMap::from([("testgroup".to_owned(), GroupInfo::new(vec!["sts-fallback-test-parent".to_owned()]))])
|
||||
})),
|
||||
group_save_started: Arc::new(tokio::sync::Notify::new()),
|
||||
group_save_release: Arc::new(tokio::sync::Notify::new()),
|
||||
}
|
||||
@@ -3610,15 +3792,36 @@ mod tests {
|
||||
Err(Error::InvalidArgument)
|
||||
}
|
||||
|
||||
async fn save_group_info(&self, _name: &str, _item: GroupInfo) -> Result<()> {
|
||||
async fn begin_group_edit(&self, name: &str) -> Result<(Box<dyn GroupEdit>, Option<GroupInfo>)> {
|
||||
let lock = self
|
||||
.group_locks
|
||||
.lock()
|
||||
.expect("group locks")
|
||||
.entry(name.to_owned())
|
||||
.or_default()
|
||||
.clone();
|
||||
let guard = lock.lock_owned().await;
|
||||
Ok((
|
||||
Box::new(MockGroupEdit {
|
||||
store: self.clone(),
|
||||
name: name.to_owned(),
|
||||
_guard: guard,
|
||||
}),
|
||||
self.saved_groups.lock().expect("saved groups").get(name).cloned(),
|
||||
))
|
||||
}
|
||||
|
||||
async fn save_group_info(&self, name: &str, item: GroupInfo) -> Result<()> {
|
||||
if self.block_group_save.load(std::sync::atomic::Ordering::SeqCst) {
|
||||
self.group_save_started.notify_one();
|
||||
self.group_save_release.notified().await;
|
||||
}
|
||||
self.saved_groups.lock().expect("saved groups").insert(name.to_owned(), item);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn delete_group_info(&self, _name: &str) -> Result<()> {
|
||||
async fn delete_group_info(&self, name: &str) -> Result<()> {
|
||||
self.saved_groups.lock().expect("saved groups").remove(name);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -3671,11 +3874,14 @@ mod tests {
|
||||
Err(Error::InvalidArgument)
|
||||
}
|
||||
|
||||
async fn delete_mapped_policy(&self, name: &str, user_type: UserType, _is_group: bool) -> Result<()> {
|
||||
async fn delete_mapped_policy(&self, name: &str, user_type: UserType, is_group: bool) -> Result<()> {
|
||||
self.deleted_mapped_policies
|
||||
.lock()
|
||||
.expect("deleted_mapped_policies mutex poisoned")
|
||||
.push((name.to_string(), user_type));
|
||||
if is_group {
|
||||
return Ok(());
|
||||
}
|
||||
Err(Error::InvalidArgument)
|
||||
}
|
||||
|
||||
@@ -3784,9 +3990,7 @@ mod tests {
|
||||
users.insert(PARENT_USER.to_string(), parent_identity);
|
||||
users.extend(saved_users);
|
||||
|
||||
let group = GroupInfo::new(vec![PARENT_USER.to_string()]);
|
||||
let mut groups = HashMap::new();
|
||||
groups.insert(GROUP_NAME.to_string(), group);
|
||||
let groups = self.saved_groups.lock().expect("saved groups").clone();
|
||||
|
||||
let group_policy = MappedPolicy::new("readwrite");
|
||||
let mut group_policies = HashMap::new();
|
||||
@@ -3819,6 +4023,82 @@ mod tests {
|
||||
IamSys::new(cache)
|
||||
}
|
||||
|
||||
async fn shared_group_caches() -> (StsTestMockStore, Arc<IamCache<StsTestMockStore>>, Arc<IamCache<StsTestMockStore>>) {
|
||||
let store = StsTestMockStore::new(false);
|
||||
let (first, second) = temp_env::async_with_vars([("RUSTFS_SKIP_BACKGROUND_TASK", Some("1"))], async {
|
||||
(
|
||||
IamCache::new(store.clone()).await.expect("first cache"),
|
||||
IamCache::new(store.clone()).await.expect("second cache"),
|
||||
)
|
||||
})
|
||||
.await;
|
||||
(store, first, second)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn group_edit_status_and_removal_survive_shared_backend_reload() {
|
||||
let (store, first, second) = shared_group_caches().await;
|
||||
let group = "testgroup";
|
||||
let member = "sts-fallback-test-parent";
|
||||
second
|
||||
.add_users_to_group("unrelated", vec![member.to_owned()])
|
||||
.await
|
||||
.expect("seed unrelated membership");
|
||||
store.block_group_save.store(true, std::sync::atomic::Ordering::SeqCst);
|
||||
let mut removal = Box::pin(first.remove_users_from_group(group, vec![member.to_owned()]));
|
||||
assert!(futures::poll!(&mut removal).is_pending());
|
||||
store.block_group_save.store(false, std::sync::atomic::Ordering::SeqCst);
|
||||
let mut status = Box::pin(second.set_group_status(group, false));
|
||||
assert!(futures::poll!(&mut status).is_pending(), "status must wait for the other writer");
|
||||
store.group_save_release.notify_one();
|
||||
removal.await.expect("commit removal");
|
||||
status.await.expect("commit status");
|
||||
|
||||
store.load_all(&first.cache).await.expect("reload durable groups");
|
||||
for node in [&first, &second] {
|
||||
let state = node.cache.snapshot();
|
||||
assert!(state.groups[group].members.is_empty());
|
||||
assert_eq!(state.groups[group].status, "disabled");
|
||||
assert_eq!(state.user_group_memberships[member], HashSet::from(["unrelated".to_owned()]));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn group_edit_delete_orders_emptiness_check_with_add() {
|
||||
let (store, first, second) = shared_group_caches().await;
|
||||
let group = "testgroup";
|
||||
let member = "sts-fallback-test-parent";
|
||||
second
|
||||
.remove_users_from_group(group, vec![member.to_owned()])
|
||||
.await
|
||||
.expect("empty group");
|
||||
store.block_group_save.store(true, std::sync::atomic::Ordering::SeqCst);
|
||||
let mut add = Box::pin(first.add_users_to_group(group, vec![member.to_owned()]));
|
||||
assert!(futures::poll!(&mut add).is_pending());
|
||||
let mut delete = Box::pin(second.remove_users_from_group(group, vec![]));
|
||||
assert!(futures::poll!(&mut delete).is_pending(), "delete must wait before checking emptiness");
|
||||
store.group_save_release.notify_one();
|
||||
add.await.expect("restore membership");
|
||||
assert!(matches!(delete.await, Err(Error::GroupNotEmpty)));
|
||||
assert!(store.deleted_mapped_policies.lock().expect("policy deletions").is_empty());
|
||||
store.block_group_save.store(false, std::sync::atomic::Ordering::SeqCst);
|
||||
second.set_group_status(group, false).await.expect("refresh from peer add");
|
||||
assert!(second.cache.snapshot().user_group_memberships[member].contains(group));
|
||||
first
|
||||
.remove_users_from_group(group, vec![member.to_owned()])
|
||||
.await
|
||||
.expect("empty group");
|
||||
second
|
||||
.remove_users_from_group(group, vec![])
|
||||
.await
|
||||
.expect("delete empty group");
|
||||
assert!(matches!(first.set_group_status(group, true).await, Err(Error::NoSuchGroup(_))));
|
||||
assert!(!second.cache.snapshot().user_group_memberships.contains_key(member));
|
||||
assert!(!store.saved_groups.lock().expect("durable groups").contains_key(group));
|
||||
}
|
||||
|
||||
async fn assert_group_write_during_reload_is_published(remove: bool) {
|
||||
let iam_sys = Arc::new(temp_env::async_with_vars([("RUSTFS_SKIP_BACKGROUND_TASK", Some("1"))], test_iam_sys()).await);
|
||||
let member = "sts-fallback-test-parent";
|
||||
@@ -4014,6 +4294,17 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn service_account_rejects_storage_directory_suffix() {
|
||||
let iam = test_iam_sys().await;
|
||||
let key = format!("example{}", rustfs_utils::path::GLOBAL_DIR_SUFFIX);
|
||||
let err = iam
|
||||
.new_service_account("parent", None, service_account_opts(&key, "test-secret"))
|
||||
.await
|
||||
.expect_err("directory marker cannot identify an IAM account");
|
||||
assert_eq!(err, Error::InvalidArgument);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn new_service_account_rejects_parent_access_key() {
|
||||
ensure_test_global_credentials();
|
||||
@@ -4204,6 +4495,11 @@ mod tests {
|
||||
policy: None,
|
||||
status: rustfs_madmin::AccountStatus::Enabled,
|
||||
};
|
||||
let virtual_parent = base64_simd::URL_SAFE_NO_PAD.encode_to_string([42; 32]);
|
||||
assert!(matches!(
|
||||
iam.create_user(&virtual_parent, &args).await,
|
||||
Err(IamError::AccessKeyAlreadyExists)
|
||||
));
|
||||
for service in [false, true] {
|
||||
for expired in [false, true] {
|
||||
let key = format!("collision-{service}-{expired}");
|
||||
@@ -4267,6 +4563,20 @@ mod tests {
|
||||
.expect("regular user update remains supported");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn create_user_rejects_invalid_storage_component() {
|
||||
let iam = test_iam_sys().await;
|
||||
let args = AddOrUpdateUserReq {
|
||||
secret_key: "exampleSecret123".to_string(),
|
||||
policy: None,
|
||||
status: rustfs_madmin::AccountStatus::Enabled,
|
||||
};
|
||||
for key in ["bad\0key", "not/a/component", "not\\a\\component"] {
|
||||
assert!(matches!(iam.create_user(key, &args).await, Err(Error::InvalidArgument)));
|
||||
}
|
||||
iam.create_user("valid-key", &args).await.expect("ordinary key");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn add_user_secret_rotation_revokes_sessions_but_identical_secret_does_not() {
|
||||
let iam = test_iam_sys().await;
|
||||
@@ -5748,13 +6058,15 @@ mod tests {
|
||||
("sub".to_string(), Value::String("subject-123".to_string())),
|
||||
("parent".to_string(), Value::String(parent_user.to_string())),
|
||||
(OIDC_VIRTUAL_PARENT_CLAIM.to_string(), Value::String(parent_user.to_string())),
|
||||
(POLICYNAME.to_string(), Value::String("readwrite".to_string())),
|
||||
(POLICYNAME.to_string(), Value::String("readwrite,readonly".to_string())),
|
||||
(OIDC_EXPLICIT_POLICIES_CLAIM.to_string(), serde_json::json!(["readonly"])),
|
||||
("groups".to_string(), serde_json::json!(["testgroup"])),
|
||||
]);
|
||||
let conditions = HashMap::new();
|
||||
let sts_args = Args {
|
||||
account: "OIDCGROUPCALLER01",
|
||||
groups: &groups,
|
||||
action: Action::S3Action(S3Action::GetObjectAction),
|
||||
action: Action::S3Action(S3Action::PutObjectAction),
|
||||
bucket: "bucket",
|
||||
conditions: &conditions,
|
||||
is_owner: false,
|
||||
@@ -5789,7 +6101,10 @@ mod tests {
|
||||
.await
|
||||
.expect("group-authorized OIDC caller should create a service account");
|
||||
assert_eq!(replication_claims.get(VERIFIED_FEDERATED_POLICY_CLAIM), Some(&Value::Bool(true)));
|
||||
assert_eq!(replication_claims.get(POLICYNAME), Some(&Value::String("readwrite".to_string())));
|
||||
assert_eq!(
|
||||
replication_claims.get(OIDC_EXPLICIT_POLICIES_CLAIM),
|
||||
claims.get(OIDC_EXPLICIT_POLICIES_CLAIM)
|
||||
);
|
||||
|
||||
let service_claims = iam_sys
|
||||
.get_claims_for_svc_acc(&service_account.access_key)
|
||||
@@ -5805,6 +6120,83 @@ mod tests {
|
||||
iam_sys.is_allowed(&service_args).await,
|
||||
"OIDC service account must resolve the policy names selected at STS issuance"
|
||||
);
|
||||
|
||||
let now = OffsetDateTime::now_utc();
|
||||
let deny_read = Policy::parse_config(br#"{"Version":"2012-10-17","Statement":[{"Effect":"Deny","Action":"s3:GetObject","Resource":"arn:aws:s3:::bucket/*"}]}"#).expect("parse Deny");
|
||||
iam_sys
|
||||
.store
|
||||
.cache
|
||||
.add_or_update_policy_doc("deny-read", &PolicyDoc::new(deny_read), now);
|
||||
for (mapping, enabled, read, write) in [
|
||||
("", true, true, false),
|
||||
("readwrite", false, true, false),
|
||||
("readwrite,deny-read", true, false, true),
|
||||
("readwrite,missing-deny", true, false, false),
|
||||
] {
|
||||
if mapping.is_empty() {
|
||||
iam_sys.store.cache.delete_group_policy("testgroup", now);
|
||||
} else {
|
||||
iam_sys
|
||||
.store
|
||||
.cache
|
||||
.add_or_update_group_policy("testgroup", &MappedPolicy::new(mapping), now);
|
||||
}
|
||||
let mut group = GroupInfo::new(Vec::new());
|
||||
if !enabled {
|
||||
group.status = STATUS_DISABLED.to_owned();
|
||||
}
|
||||
iam_sys.store.cache.add_or_update_group("testgroup", &group, now);
|
||||
for (action, expected) in [(S3Action::GetObjectAction, read), (S3Action::PutObjectAction, write)] {
|
||||
let sts = Args {
|
||||
action: Action::S3Action(action),
|
||||
..sts_args.clone()
|
||||
};
|
||||
let service = Args {
|
||||
action: sts.action,
|
||||
..service_args.clone()
|
||||
};
|
||||
let prepared = iam_sys.prepare_sts_auth(&sts, parent_user).await;
|
||||
assert_eq!(
|
||||
iam_sys.eval_prepared(&prepared, &sts).await,
|
||||
expected,
|
||||
"STS: {mapping}, enabled={enabled}"
|
||||
);
|
||||
assert_eq!(
|
||||
iam_sys.is_allowed(&service).await,
|
||||
expected,
|
||||
"service account: {mapping}, enabled={enabled}"
|
||||
);
|
||||
if mapping == "readwrite,deny-read" && action == S3Action::GetObjectAction {
|
||||
let child = iam_sys.prepare_service_account_auth(&service, parent_user).await;
|
||||
for (prepared, args) in [(&prepared, &sts), (&child, &service)] {
|
||||
assert!(
|
||||
matches!(iam_sys.try_eval_prepared(prepared, args).await, Err(Error::IAMActionNotAllowed)),
|
||||
"a newly mapped Deny must stop bucket policy fallback"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
iam_sys.store.cache.delete_group_policy("testgroup", now);
|
||||
for (source, account) in [(&claims, sts_args.account), (&service_claims, service_args.account)] {
|
||||
let mut legacy = source.clone();
|
||||
legacy.remove(OIDC_EXPLICIT_POLICIES_CLAIM);
|
||||
let args = Args {
|
||||
claims: &legacy,
|
||||
account,
|
||||
deny_only: true,
|
||||
..sts_args.clone()
|
||||
};
|
||||
let prepared = if account == sts_args.account {
|
||||
iam_sys.prepare_sts_auth(&args, parent_user).await
|
||||
} else {
|
||||
iam_sys.prepare_service_account_auth(&args, parent_user).await
|
||||
};
|
||||
assert!(
|
||||
!iam_sys.eval_prepared(&prepared, &args).await,
|
||||
"ambiguous legacy grouped credentials must be reissued"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -5832,6 +6224,8 @@ mod tests {
|
||||
("parent".to_string(), Value::String(parent_user.to_string())),
|
||||
(OIDC_VIRTUAL_PARENT_CLAIM.to_string(), Value::String(parent_user.to_string())),
|
||||
(POLICYNAME.to_string(), Value::String("readonly,deny-delete".to_string())),
|
||||
(OIDC_EXPLICIT_POLICIES_CLAIM.to_string(), serde_json::json!(["readonly", "deny-delete"])),
|
||||
("groups".to_string(), serde_json::json!(["testgroup"])),
|
||||
]);
|
||||
let groups = Some(vec!["testgroup".to_string()]);
|
||||
let conditions = HashMap::new();
|
||||
@@ -6036,7 +6430,7 @@ mod tests {
|
||||
|
||||
let prepared = iam_sys.prepare_sts_auth(&args, parent_user).await;
|
||||
|
||||
assert!(matches!(prepared.mode, PreparedIamMode::Deny));
|
||||
assert!(matches!(prepared.mode, PreparedIamMode::TerminalDeny));
|
||||
assert!(!iam_sys.eval_prepared(&prepared, &args).await);
|
||||
}
|
||||
|
||||
|
||||
@@ -33,23 +33,18 @@ default = []
|
||||
diagnostic-internode-http2-ping = []
|
||||
diagnostic-read-version-caller = []
|
||||
hotpath = [
|
||||
"hotpath/hotpath",
|
||||
"hotpath/tokio",
|
||||
"rustfs-s3-ops/hotpath",
|
||||
]
|
||||
hotpath-alloc = [
|
||||
"hotpath",
|
||||
"hotpath/hotpath-alloc",
|
||||
"rustfs-s3-ops/hotpath-alloc",
|
||||
]
|
||||
hotpath-cpu = [
|
||||
"hotpath",
|
||||
"hotpath/hotpath-cpu",
|
||||
"rustfs-s3-ops/hotpath-cpu",
|
||||
]
|
||||
|
||||
[dependencies]
|
||||
hotpath.workspace = true
|
||||
metrics = { workspace = true }
|
||||
rustfs-s3-ops = { workspace = true }
|
||||
num_cpus = { workspace = true }
|
||||
|
||||
@@ -59,6 +59,7 @@ use std::sync::{
|
||||
LazyLock, Mutex,
|
||||
atomic::{AtomicBool, AtomicU64, Ordering},
|
||||
};
|
||||
use std::time::Duration;
|
||||
|
||||
/// Global switch for detailed per-stage PUT metrics (path label, stage durations).
|
||||
/// When `false`, `record_put_object_path` and `record_put_object_stage_duration`
|
||||
@@ -449,6 +450,125 @@ pub const PUT_RENAME_QUORUM_FANOUT_STATE_PANIC: &str = "panic";
|
||||
pub const PUT_RENAME_DISK_WAIT_COMPLETION_POSITION_QUORUM_FIRST: &str = "quorum_first";
|
||||
pub const PUT_RENAME_DISK_WAIT_COMPLETION_POSITION_QUORUM_TAIL: &str = "quorum_tail";
|
||||
pub const PUT_RENAME_DISK_WAIT_COMPLETION_POSITION_ERROR: &str = "error";
|
||||
pub const ECSTORE_RENAME_RPC_KIND_ORDINARY: &str = "ordinary";
|
||||
pub const ECSTORE_RENAME_RPC_KIND_QUOTA_FENCE: &str = "quota_fence";
|
||||
pub const ECSTORE_RENAME_RPC_KIND_INCARNATION: &str = "incarnation";
|
||||
pub const ECSTORE_RENAME_RPC_KIND_MULTIPART: &str = "multipart";
|
||||
pub const ECSTORE_RENAME_RPC_KIND_SCANNER_LEASE: &str = "scanner_lease";
|
||||
pub const ECSTORE_RENAME_RPC_KIND_POOL_META_CAS: &str = "pool_meta_cas";
|
||||
pub const ECSTORE_RENAME_RPC_KINDS: [&str; 6] = [
|
||||
ECSTORE_RENAME_RPC_KIND_ORDINARY,
|
||||
ECSTORE_RENAME_RPC_KIND_QUOTA_FENCE,
|
||||
ECSTORE_RENAME_RPC_KIND_INCARNATION,
|
||||
ECSTORE_RENAME_RPC_KIND_MULTIPART,
|
||||
ECSTORE_RENAME_RPC_KIND_SCANNER_LEASE,
|
||||
ECSTORE_RENAME_RPC_KIND_POOL_META_CAS,
|
||||
];
|
||||
pub const ECSTORE_RENAME_PEER_WIDTHS: [&str; 18] = [
|
||||
"0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "gt16",
|
||||
];
|
||||
pub const ECSTORE_RENAME_COVERAGE: [&str; 3] = ["none", "partial", "full"];
|
||||
pub const ECSTORE_RENAME_BATCH_EXECUTION: [&str; 10] = [
|
||||
"planned_groups",
|
||||
"planned_items",
|
||||
"stream_groups",
|
||||
"stream_items",
|
||||
"completed_items",
|
||||
"fallback_groups",
|
||||
"fallback_items",
|
||||
"not_dispatched_items",
|
||||
"may_have_published_items",
|
||||
"terminal_missing_items",
|
||||
];
|
||||
pub const ECSTORE_RENAME_ADMISSION: [&str; 10] = [
|
||||
"unary_attempts",
|
||||
"unary_items",
|
||||
"unary_wait_ns_total",
|
||||
"unary_wait_ns_max",
|
||||
"unary_timeouts",
|
||||
"batch_attempts",
|
||||
"batch_items",
|
||||
"batch_wait_ns_total",
|
||||
"batch_wait_ns_max",
|
||||
"batch_timeouts",
|
||||
];
|
||||
pub const ECSTORE_RENAME_QUARANTINE: [&str; 12] = [
|
||||
"unary_events",
|
||||
"unary_items",
|
||||
"unary_hold_ns_total",
|
||||
"unary_hold_ns_max",
|
||||
"unary_released_items",
|
||||
"unary_no_runtime_items",
|
||||
"batch_events",
|
||||
"batch_items",
|
||||
"batch_hold_ns_total",
|
||||
"batch_hold_ns_max",
|
||||
"batch_released_items",
|
||||
"batch_no_runtime_items",
|
||||
];
|
||||
pub const ECSTORE_RENAME_BATCH_TIMING: [&str; 4] = [
|
||||
"client_stream_to_first",
|
||||
"client_first_to_last",
|
||||
"server_dispatch_to_first",
|
||||
"server_first_to_last",
|
||||
];
|
||||
|
||||
static ECSTORE_RENAME_RPC_COUNTS: [AtomicU64; 6] = [const { AtomicU64::new(0) }; 6];
|
||||
static ECSTORE_RENAME_REMOTE_WIDTH_COUNTS: [AtomicU64; 18] = [const { AtomicU64::new(0) }; 18];
|
||||
static ECSTORE_RENAME_BATCHABLE_WIDTH_COUNTS: [AtomicU64; 18] = [const { AtomicU64::new(0) }; 18];
|
||||
static ECSTORE_RENAME_COVERAGE_COUNTS: [AtomicU64; 3] = [const { AtomicU64::new(0) }; 3];
|
||||
static ECSTORE_RENAME_BATCH_EXECUTION_COUNTS: [AtomicU64; 10] = [const { AtomicU64::new(0) }; 10];
|
||||
static ECSTORE_RENAME_ADMISSION_COUNTS: [AtomicU64; 10] = [const { AtomicU64::new(0) }; 10];
|
||||
static ECSTORE_RENAME_QUARANTINE_COUNTS: [AtomicU64; 12] = [const { AtomicU64::new(0) }; 12];
|
||||
static ECSTORE_RENAME_BATCH_TIMING_COUNTS: [AtomicU64; 4] = [const { AtomicU64::new(0) }; 4];
|
||||
static ECSTORE_RENAME_BATCH_TIMING_TOTAL_NS: [AtomicU64; 4] = [const { AtomicU64::new(0) }; 4];
|
||||
static ECSTORE_RENAME_BATCH_TIMING_MAX_NS: [AtomicU64; 4] = [const { AtomicU64::new(0) }; 4];
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
pub struct EcstoreRenameDiagnosticsSnapshot {
|
||||
pub rpc_counts: [u64; 6],
|
||||
pub remote_width_counts: [u64; 18],
|
||||
pub batchable_width_counts: [u64; 18],
|
||||
pub coverage_counts: [u64; 3],
|
||||
pub batch_execution_counts: [u64; 10],
|
||||
pub admission_counts: [u64; 10],
|
||||
pub quarantine_counts: [u64; 12],
|
||||
pub batch_timing_counts: [u64; 4],
|
||||
pub batch_timing_total_ns: [u64; 4],
|
||||
pub batch_timing_max_ns: [u64; 4],
|
||||
}
|
||||
|
||||
impl EcstoreRenameDiagnosticsSnapshot {
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.rpc_counts
|
||||
.iter()
|
||||
.chain(self.remote_width_counts.iter())
|
||||
.chain(self.batchable_width_counts.iter())
|
||||
.chain(self.coverage_counts.iter())
|
||||
.chain(self.batch_execution_counts.iter())
|
||||
.chain(self.admission_counts.iter())
|
||||
.chain(self.quarantine_counts.iter())
|
||||
.chain(self.batch_timing_counts.iter())
|
||||
.chain(self.batch_timing_total_ns.iter())
|
||||
.chain(self.batch_timing_max_ns.iter())
|
||||
.all(|count| *count == 0)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn snapshot_ecstore_rename_diagnostics() -> EcstoreRenameDiagnosticsSnapshot {
|
||||
EcstoreRenameDiagnosticsSnapshot {
|
||||
rpc_counts: std::array::from_fn(|index| ECSTORE_RENAME_RPC_COUNTS[index].load(Ordering::Relaxed)),
|
||||
remote_width_counts: std::array::from_fn(|index| ECSTORE_RENAME_REMOTE_WIDTH_COUNTS[index].load(Ordering::Relaxed)),
|
||||
batchable_width_counts: std::array::from_fn(|index| ECSTORE_RENAME_BATCHABLE_WIDTH_COUNTS[index].load(Ordering::Relaxed)),
|
||||
coverage_counts: std::array::from_fn(|index| ECSTORE_RENAME_COVERAGE_COUNTS[index].load(Ordering::Relaxed)),
|
||||
batch_execution_counts: std::array::from_fn(|index| ECSTORE_RENAME_BATCH_EXECUTION_COUNTS[index].load(Ordering::Relaxed)),
|
||||
admission_counts: std::array::from_fn(|index| ECSTORE_RENAME_ADMISSION_COUNTS[index].load(Ordering::Relaxed)),
|
||||
quarantine_counts: std::array::from_fn(|index| ECSTORE_RENAME_QUARANTINE_COUNTS[index].load(Ordering::Relaxed)),
|
||||
batch_timing_counts: std::array::from_fn(|index| ECSTORE_RENAME_BATCH_TIMING_COUNTS[index].load(Ordering::Relaxed)),
|
||||
batch_timing_total_ns: std::array::from_fn(|index| ECSTORE_RENAME_BATCH_TIMING_TOTAL_NS[index].load(Ordering::Relaxed)),
|
||||
batch_timing_max_ns: std::array::from_fn(|index| ECSTORE_RENAME_BATCH_TIMING_MAX_NS[index].load(Ordering::Relaxed)),
|
||||
}
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn get_stage_metrics_enabled() -> bool {
|
||||
@@ -2663,6 +2783,169 @@ pub fn record_put_rename_quorum_wait_fanout(
|
||||
}
|
||||
}
|
||||
|
||||
/// Records a rename RPC after its canonical request digest has been attached.
|
||||
///
|
||||
/// This uses the PUT-stage diagnostics switch because it is temporary hot-path
|
||||
/// evidence, but the metric is ECStore-scoped: scanner and pool metadata
|
||||
/// renames are not S3 PutObject operations.
|
||||
#[inline(always)]
|
||||
pub fn record_ecstore_rename_remote_rpc(kind: &'static str) {
|
||||
if !put_stage_metrics_enabled() {
|
||||
return;
|
||||
}
|
||||
let Some(index) = ECSTORE_RENAME_RPC_KINDS.iter().position(|candidate| *candidate == kind) else {
|
||||
return;
|
||||
};
|
||||
ECSTORE_RENAME_RPC_COUNTS[index].fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
fn rename_peer_width_index(width: usize) -> usize {
|
||||
width.min(17)
|
||||
}
|
||||
|
||||
/// Records the per-peer opportunity for a future body-bound rename batch.
|
||||
#[inline(always)]
|
||||
pub fn record_ecstore_rename_batch_candidate_peer(remote_disks: usize, batchable_disks: usize) {
|
||||
if !put_stage_metrics_enabled() || remote_disks == 0 {
|
||||
return;
|
||||
}
|
||||
ECSTORE_RENAME_REMOTE_WIDTH_COUNTS[rename_peer_width_index(remote_disks)].fetch_add(1, Ordering::Relaxed);
|
||||
ECSTORE_RENAME_BATCHABLE_WIDTH_COUNTS[rename_peer_width_index(batchable_disks)].fetch_add(1, Ordering::Relaxed);
|
||||
let coverage_index = if batchable_disks == 0 {
|
||||
0
|
||||
} else if batchable_disks == remote_disks {
|
||||
2
|
||||
} else {
|
||||
1
|
||||
};
|
||||
ECSTORE_RENAME_COVERAGE_COUNTS[coverage_index].fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
fn record_ecstore_rename_batch_execution(index: usize, count: usize) {
|
||||
if put_stage_metrics_enabled() {
|
||||
let count = u64::try_from(count).unwrap_or(u64::MAX);
|
||||
ECSTORE_RENAME_BATCH_EXECUTION_COUNTS[index].fetch_add(count, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn record_ecstore_rename_batch_planned(items: usize) {
|
||||
record_ecstore_rename_batch_execution(0, 1);
|
||||
record_ecstore_rename_batch_execution(1, items);
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn record_ecstore_rename_batch_stream_started(items: usize) {
|
||||
record_ecstore_rename_batch_execution(2, 1);
|
||||
record_ecstore_rename_batch_execution(3, items);
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn record_ecstore_rename_batch_item_completed() {
|
||||
record_ecstore_rename_batch_execution(4, 1);
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn record_ecstore_rename_batch_fallback(items: usize) {
|
||||
record_ecstore_rename_batch_execution(5, 1);
|
||||
record_ecstore_rename_batch_execution(6, items);
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn record_ecstore_rename_batch_not_dispatched(items: usize) {
|
||||
record_ecstore_rename_batch_execution(7, items);
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn record_ecstore_rename_batch_may_have_published(items: usize) {
|
||||
record_ecstore_rename_batch_execution(8, items);
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn record_ecstore_rename_batch_terminal_missing(items: usize) {
|
||||
record_ecstore_rename_batch_execution(9, items);
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn record_ecstore_rename_admission(batch: bool, items: usize, wait: Duration, timed_out: bool) {
|
||||
if !put_stage_metrics_enabled() {
|
||||
return;
|
||||
}
|
||||
let base = usize::from(batch) * 5;
|
||||
let items = u64::try_from(items).unwrap_or(u64::MAX);
|
||||
let wait_ns = u64::try_from(wait.as_nanos()).unwrap_or(u64::MAX);
|
||||
ECSTORE_RENAME_ADMISSION_COUNTS[base].fetch_add(1, Ordering::Relaxed);
|
||||
ECSTORE_RENAME_ADMISSION_COUNTS[base + 1].fetch_add(items, Ordering::Relaxed);
|
||||
ECSTORE_RENAME_ADMISSION_COUNTS[base + 2].fetch_add(wait_ns, Ordering::Relaxed);
|
||||
ECSTORE_RENAME_ADMISSION_COUNTS[base + 3].fetch_max(wait_ns, Ordering::Relaxed);
|
||||
if timed_out {
|
||||
ECSTORE_RENAME_ADMISSION_COUNTS[base + 4].fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn record_ecstore_rename_quarantine(batch: bool, items: usize, hold: Duration) {
|
||||
if !put_stage_metrics_enabled() {
|
||||
return;
|
||||
}
|
||||
let base = usize::from(batch) * 6;
|
||||
let items = u64::try_from(items).unwrap_or(u64::MAX);
|
||||
let hold_ns = u64::try_from(hold.as_nanos()).unwrap_or(u64::MAX);
|
||||
ECSTORE_RENAME_QUARANTINE_COUNTS[base].fetch_add(1, Ordering::Relaxed);
|
||||
ECSTORE_RENAME_QUARANTINE_COUNTS[base + 1].fetch_add(items, Ordering::Relaxed);
|
||||
ECSTORE_RENAME_QUARANTINE_COUNTS[base + 2].fetch_add(hold_ns, Ordering::Relaxed);
|
||||
ECSTORE_RENAME_QUARANTINE_COUNTS[base + 3].fetch_max(hold_ns, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn record_ecstore_rename_quarantine_released(batch: bool, items: usize) {
|
||||
if put_stage_metrics_enabled() {
|
||||
let base = usize::from(batch) * 6;
|
||||
ECSTORE_RENAME_QUARANTINE_COUNTS[base + 4].fetch_add(u64::try_from(items).unwrap_or(u64::MAX), Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn record_ecstore_rename_quarantine_no_runtime(batch: bool, items: usize) {
|
||||
if put_stage_metrics_enabled() {
|
||||
let base = usize::from(batch) * 6;
|
||||
ECSTORE_RENAME_QUARANTINE_COUNTS[base + 5].fetch_add(u64::try_from(items).unwrap_or(u64::MAX), Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
fn record_ecstore_rename_batch_timing(index: usize, duration: Duration) {
|
||||
if !put_stage_metrics_enabled() {
|
||||
return;
|
||||
}
|
||||
let duration_ns = u64::try_from(duration.as_nanos()).unwrap_or(u64::MAX);
|
||||
ECSTORE_RENAME_BATCH_TIMING_COUNTS[index].fetch_add(1, Ordering::Relaxed);
|
||||
ECSTORE_RENAME_BATCH_TIMING_TOTAL_NS[index].fetch_add(duration_ns, Ordering::Relaxed);
|
||||
ECSTORE_RENAME_BATCH_TIMING_MAX_NS[index].fetch_max(duration_ns, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn record_ecstore_rename_batch_client_stream_to_first(duration: Duration) {
|
||||
record_ecstore_rename_batch_timing(0, duration);
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn record_ecstore_rename_batch_client_first_to_last(duration: Duration) {
|
||||
record_ecstore_rename_batch_timing(1, duration);
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn record_ecstore_rename_batch_server_dispatch_to_first(duration: Duration) {
|
||||
record_ecstore_rename_batch_timing(2, duration);
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn record_ecstore_rename_batch_server_first_to_last(duration: Duration) {
|
||||
record_ecstore_rename_batch_timing(3, duration);
|
||||
}
|
||||
|
||||
/// Record generic internal operation stage duration (non-PUT paths).
|
||||
/// Use this for metacache walks, listing, lifecycle, and other background
|
||||
/// operations that are NOT part of the PUT object hot path.
|
||||
@@ -4585,6 +4868,85 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ecstore_rename_batch_candidate_metrics_are_gated_and_bounded() {
|
||||
let _guard = METRICS_FLAG_LOCK.lock().unwrap_or_else(|e| e.into_inner());
|
||||
set_put_stage_metrics_enabled(false);
|
||||
let disabled_before = snapshot_ecstore_rename_diagnostics();
|
||||
record_ecstore_rename_remote_rpc(ECSTORE_RENAME_RPC_KIND_ORDINARY);
|
||||
record_ecstore_rename_batch_candidate_peer(4, 4);
|
||||
record_ecstore_rename_admission(false, 1, Duration::from_nanos(10), false);
|
||||
record_ecstore_rename_quarantine(true, 4, Duration::from_nanos(30));
|
||||
record_ecstore_rename_batch_client_stream_to_first(Duration::from_nanos(10));
|
||||
assert_eq!(snapshot_ecstore_rename_diagnostics(), disabled_before);
|
||||
|
||||
set_put_stage_metrics_enabled(true);
|
||||
let before = snapshot_ecstore_rename_diagnostics();
|
||||
for kind in ECSTORE_RENAME_RPC_KINDS {
|
||||
record_ecstore_rename_remote_rpc(kind);
|
||||
}
|
||||
record_ecstore_rename_batch_candidate_peer(4, 4);
|
||||
record_ecstore_rename_batch_candidate_peer(4, 3);
|
||||
record_ecstore_rename_batch_candidate_peer(4, 0);
|
||||
record_ecstore_rename_batch_candidate_peer(20, 20);
|
||||
record_ecstore_rename_batch_planned(4);
|
||||
record_ecstore_rename_batch_stream_started(4);
|
||||
record_ecstore_rename_batch_item_completed();
|
||||
record_ecstore_rename_batch_fallback(3);
|
||||
record_ecstore_rename_batch_not_dispatched(2);
|
||||
record_ecstore_rename_batch_may_have_published(4);
|
||||
record_ecstore_rename_batch_terminal_missing(1);
|
||||
record_ecstore_rename_admission(false, 1, Duration::from_nanos(10), false);
|
||||
record_ecstore_rename_admission(true, 4, Duration::from_nanos(20), true);
|
||||
record_ecstore_rename_quarantine(true, 4, Duration::from_nanos(30));
|
||||
record_ecstore_rename_quarantine_released(true, 3);
|
||||
record_ecstore_rename_quarantine_no_runtime(true, 1);
|
||||
record_ecstore_rename_batch_client_stream_to_first(Duration::from_nanos(10));
|
||||
record_ecstore_rename_batch_client_first_to_last(Duration::from_nanos(20));
|
||||
record_ecstore_rename_batch_server_dispatch_to_first(Duration::from_nanos(30));
|
||||
record_ecstore_rename_batch_server_first_to_last(Duration::from_nanos(40));
|
||||
set_put_stage_metrics_enabled(false);
|
||||
|
||||
let after = snapshot_ecstore_rename_diagnostics();
|
||||
for index in 0..ECSTORE_RENAME_RPC_KINDS.len() {
|
||||
assert_eq!(after.rpc_counts[index] - before.rpc_counts[index], 1);
|
||||
}
|
||||
assert_eq!(after.remote_width_counts[4] - before.remote_width_counts[4], 3);
|
||||
assert_eq!(after.remote_width_counts[17] - before.remote_width_counts[17], 1);
|
||||
assert_eq!(after.batchable_width_counts[0] - before.batchable_width_counts[0], 1);
|
||||
assert_eq!(after.batchable_width_counts[3] - before.batchable_width_counts[3], 1);
|
||||
assert_eq!(after.batchable_width_counts[4] - before.batchable_width_counts[4], 1);
|
||||
assert_eq!(after.batchable_width_counts[17] - before.batchable_width_counts[17], 1);
|
||||
assert_eq!(after.coverage_counts[0] - before.coverage_counts[0], 1);
|
||||
assert_eq!(after.coverage_counts[1] - before.coverage_counts[1], 1);
|
||||
assert_eq!(after.coverage_counts[2] - before.coverage_counts[2], 2);
|
||||
assert_eq!(
|
||||
std::array::from_fn::<_, 10, _>(|index| {
|
||||
after.batch_execution_counts[index] - before.batch_execution_counts[index]
|
||||
}),
|
||||
[1, 4, 1, 4, 1, 1, 3, 2, 4, 1]
|
||||
);
|
||||
assert_eq!(
|
||||
std::array::from_fn::<_, 10, _>(|index| after.admission_counts[index] - before.admission_counts[index]),
|
||||
[1, 1, 10, 10, 0, 1, 4, 20, 20, 1]
|
||||
);
|
||||
assert_eq!(
|
||||
std::array::from_fn::<_, 12, _>(|index| after.quarantine_counts[index] - before.quarantine_counts[index]),
|
||||
[0, 0, 0, 0, 0, 0, 1, 4, 30, 30, 3, 1]
|
||||
);
|
||||
assert_eq!(
|
||||
std::array::from_fn::<_, 4, _>(|index| after.batch_timing_counts[index] - before.batch_timing_counts[index]),
|
||||
[1, 1, 1, 1]
|
||||
);
|
||||
assert_eq!(
|
||||
std::array::from_fn::<_, 4, _>(|index| { after.batch_timing_total_ns[index] - before.batch_timing_total_ns[index] }),
|
||||
[10, 20, 30, 40]
|
||||
);
|
||||
for (actual, expected) in after.batch_timing_max_ns.into_iter().zip([10, 20, 30, 40]) {
|
||||
assert!(actual >= expected);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_put_object_diagnostic_buckets() {
|
||||
assert_eq!(put_object_size_bucket(0), "unknown");
|
||||
|
||||
@@ -58,8 +58,6 @@ zeroize = { workspace = true, features = ["derive"] }
|
||||
|
||||
# Configuration and storage
|
||||
url = { workspace = true }
|
||||
tempfile = { workspace = true }
|
||||
|
||||
# Caching
|
||||
moka = { workspace = true, features = ["future"] }
|
||||
|
||||
|
||||
@@ -65,7 +65,7 @@ use aws_sdk_kms::error::{ProvideErrorMetadata, SdkError};
|
||||
use aws_sdk_kms::types::{DataKeySpec, KeySpec as AwsKeySpec, KeyState as AwsKeyState, KeyUsageType, Tag};
|
||||
use aws_smithy_runtime_api::client::orchestrator::HttpResponse;
|
||||
use aws_smithy_types::Blob;
|
||||
use jiff::Zoned;
|
||||
use jiff::ZonedDateTime;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
use super::{BackendCapabilities, ExpiredKeyRemoval, KmsBackend, empty_key_page, list_keys_page_size};
|
||||
@@ -234,12 +234,12 @@ fn map_key_usage(usage: Option<&KeyUsageType>) -> KeyUsage {
|
||||
}
|
||||
}
|
||||
|
||||
fn to_zoned(value: Option<&aws_smithy_types::DateTime>) -> Option<Zoned> {
|
||||
fn to_zoned(value: Option<&aws_smithy_types::DateTime>) -> Option<ZonedDateTime> {
|
||||
let value = value?;
|
||||
let nanos = i32::try_from(value.subsec_nanos()).ok()?;
|
||||
jiff::Timestamp::new(value.secs(), nanos)
|
||||
.ok()
|
||||
.map(|timestamp| timestamp.to_zoned(jiff::tz::TimeZone::UTC))
|
||||
.map(|timestamp| timestamp.to_zoned_datetime(jiff::tz::TimeZone::UTC))
|
||||
}
|
||||
|
||||
/// `None` for an empty context so the request omits the field entirely rather
|
||||
@@ -593,7 +593,7 @@ impl KmsBackend for AwsKmsBackend {
|
||||
encrypted_key: ciphertext.into_inner(),
|
||||
nonce: Vec::new(),
|
||||
encryption_context: request.encryption_context,
|
||||
created_at: Zoned::now(),
|
||||
created_at: ZonedDateTime::now(),
|
||||
// AWS addresses backing keys internally and never exposes a version.
|
||||
master_key_version: None,
|
||||
// AWS binds the context into the blob itself; this field describes
|
||||
@@ -876,7 +876,7 @@ impl KmsBackend for AwsKmsBackend {
|
||||
/// report what AWS has already done: a key that is gone counts as removed,
|
||||
/// a key that left `PendingDeletion` counts as a state change, and a key
|
||||
/// still pending is never expired by RustFS regardless of the deadline.
|
||||
async fn remove_expired_key(&self, key_id: &str, _now: &Zoned) -> Result<ExpiredKeyRemoval> {
|
||||
async fn remove_expired_key(&self, key_id: &str, _now: &ZonedDateTime) -> Result<ExpiredKeyRemoval> {
|
||||
match self.describe(key_id).await {
|
||||
Ok(metadata) => Ok(match metadata.key_state {
|
||||
KeyState::PendingDeletion => ExpiredKeyRemoval::NotExpired,
|
||||
@@ -1492,7 +1492,7 @@ mod tests {
|
||||
let (http_client, backend) = scripted_backend(vec![ok_event(key_metadata_json("test-key", "PendingDeletion"))]);
|
||||
assert_eq!(
|
||||
backend
|
||||
.remove_expired_key("test-key", &Zoned::now())
|
||||
.remove_expired_key("test-key", &ZonedDateTime::now())
|
||||
.await
|
||||
.expect("observation should succeed"),
|
||||
ExpiredKeyRemoval::NotExpired
|
||||
@@ -1502,7 +1502,7 @@ mod tests {
|
||||
let (_http, backend) = scripted_backend(vec![error_event(400, "NotFoundException", "key does not exist")]);
|
||||
assert_eq!(
|
||||
backend
|
||||
.remove_expired_key("test-key", &Zoned::now())
|
||||
.remove_expired_key("test-key", &ZonedDateTime::now())
|
||||
.await
|
||||
.expect("a key AWS already removed is a completed removal"),
|
||||
ExpiredKeyRemoval::Removed
|
||||
@@ -1511,7 +1511,7 @@ mod tests {
|
||||
let (_http, backend) = scripted_backend(vec![ok_event(key_metadata_json("test-key", "Enabled"))]);
|
||||
assert_eq!(
|
||||
backend
|
||||
.remove_expired_key("test-key", &Zoned::now())
|
||||
.remove_expired_key("test-key", &ZonedDateTime::now())
|
||||
.await
|
||||
.expect("observation should succeed"),
|
||||
ExpiredKeyRemoval::StateChanged
|
||||
|
||||
@@ -35,7 +35,7 @@ use aes_gcm::{
|
||||
use argon2::{Algorithm, Argon2, Params, Version};
|
||||
use async_trait::async_trait;
|
||||
use base64_simd::STANDARD as BASE64;
|
||||
use jiff::Zoned;
|
||||
use jiff::ZonedDateTime;
|
||||
use rand::RngExt;
|
||||
use serde::de::{self, IgnoredAny, MapAccess, Visitor};
|
||||
use serde::{Deserialize, Serialize};
|
||||
@@ -582,14 +582,14 @@ struct StoredMasterKey {
|
||||
description: Option<String>,
|
||||
metadata: HashMap<String, String>,
|
||||
#[serde(with = "crate::time_serde::zoned")]
|
||||
created_at: Zoned,
|
||||
created_at: ZonedDateTime,
|
||||
#[serde(with = "crate::time_serde::option_zoned")]
|
||||
rotated_at: Option<Zoned>,
|
||||
rotated_at: Option<ZonedDateTime>,
|
||||
created_by: Option<String>,
|
||||
/// Scheduled deletion deadline; absent on records written before deadline
|
||||
/// persistence landed, so it must stay optional for backward compatibility.
|
||||
#[serde(default, with = "crate::time_serde::option_zoned")]
|
||||
deletion_date: Option<Zoned>,
|
||||
deletion_date: Option<ZonedDateTime>,
|
||||
/// Encrypted key material (32 bytes encoded in base64 for AES-256)
|
||||
encrypted_key_material: String,
|
||||
/// Nonce used for encryption
|
||||
@@ -685,10 +685,10 @@ impl<'de> Deserialize<'de> for StoredMasterKey {
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct ZonedValue(#[serde(with = "crate::time_serde::zoned")] Zoned);
|
||||
struct ZonedValue(#[serde(with = "crate::time_serde::zoned")] ZonedDateTime);
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct OptionalZonedValue(#[serde(with = "crate::time_serde::option_zoned")] Option<Zoned>);
|
||||
struct OptionalZonedValue(#[serde(with = "crate::time_serde::option_zoned")] Option<ZonedDateTime>);
|
||||
|
||||
struct StoredMasterKeyVisitor;
|
||||
|
||||
@@ -1556,7 +1556,7 @@ impl LocalKmsClient {
|
||||
encrypted_key,
|
||||
nonce,
|
||||
encryption_context: request.encryption_context.clone(),
|
||||
created_at: Zoned::now(),
|
||||
created_at: ZonedDateTime::now(),
|
||||
master_key_version: None,
|
||||
context_binding,
|
||||
};
|
||||
@@ -1596,7 +1596,7 @@ impl LocalKmsClient {
|
||||
encrypted_key,
|
||||
nonce,
|
||||
encryption_context: request.encryption_context.clone(),
|
||||
created_at: Zoned::now(),
|
||||
created_at: ZonedDateTime::now(),
|
||||
// Local rotation is rejected, so the key has a single material version.
|
||||
master_key_version: None,
|
||||
context_binding,
|
||||
@@ -1860,7 +1860,7 @@ impl LocalKmsClient {
|
||||
let mut master_key = self.load_master_key(key_id).await?;
|
||||
ensure_key_status_permits(key_id, &master_key.status, StateGatedOperation::ScheduleDeletion)?;
|
||||
master_key.status = KeyStatus::PendingDeletion;
|
||||
master_key.deletion_date = Some(Zoned::now() + Duration::from_secs(pending_window_days as u64 * 86400));
|
||||
master_key.deletion_date = Some(ZonedDateTime::now() + Duration::from_secs(pending_window_days as u64 * 86400));
|
||||
|
||||
// Preserve the existing key material (see enable_key): scheduling deletion must not
|
||||
// regenerate the master key, or cancelling the deletion later would recover a key that
|
||||
@@ -1985,7 +1985,7 @@ impl KmsBackend for LocalKmsBackend {
|
||||
key_state: KeyState::Enabled,
|
||||
key_usage: request.key_usage,
|
||||
description: request.description,
|
||||
creation_date: Zoned::now(),
|
||||
creation_date: ZonedDateTime::now(),
|
||||
deletion_date: None,
|
||||
origin: "KMS".to_string(),
|
||||
key_manager: "CUSTOMER".to_string(),
|
||||
@@ -2113,7 +2113,7 @@ impl KmsBackend for LocalKmsBackend {
|
||||
Ok((_stored, key_material)) => {
|
||||
let mut tombstone = master_key.clone();
|
||||
tombstone.status = KeyStatus::Deleted;
|
||||
tombstone.deletion_date = Some(Zoned::now());
|
||||
tombstone.deletion_date = Some(ZonedDateTime::now());
|
||||
self.client.save_master_key(&tombstone, &key_material).await?;
|
||||
}
|
||||
Err(error) => {
|
||||
@@ -2136,7 +2136,7 @@ impl KmsBackend for LocalKmsBackend {
|
||||
key_usage: master_key.usage,
|
||||
key_state: KeyState::PendingDeletion, // AWS KMS compatibility
|
||||
creation_date: master_key.created_at,
|
||||
deletion_date: Some(Zoned::now()),
|
||||
deletion_date: Some(ZonedDateTime::now()),
|
||||
key_manager: "CUSTOMER".to_string(),
|
||||
origin: "AWS_KMS".to_string(),
|
||||
tags: master_key.metadata,
|
||||
@@ -2162,7 +2162,7 @@ impl KmsBackend for LocalKmsBackend {
|
||||
)));
|
||||
}
|
||||
|
||||
let deletion_date = Zoned::now() + Duration::from_secs(days as u64 * 86400);
|
||||
let deletion_date = ZonedDateTime::now() + Duration::from_secs(days as u64 * 86400);
|
||||
master_key.status = KeyStatus::PendingDeletion;
|
||||
master_key.deletion_date = Some(deletion_date.clone());
|
||||
|
||||
@@ -2317,7 +2317,7 @@ impl KmsBackend for LocalKmsBackend {
|
||||
.with_update_key_metadata(true)
|
||||
}
|
||||
|
||||
async fn remove_expired_key(&self, key_id: &str, now: &Zoned) -> Result<ExpiredKeyRemoval> {
|
||||
async fn remove_expired_key(&self, key_id: &str, now: &ZonedDateTime) -> Result<ExpiredKeyRemoval> {
|
||||
// The per-key write lock serializes this against a concurrent
|
||||
// cancellation, closing the check-then-remove race.
|
||||
let _write_guard = self.client.lock_key_for_write(key_id).await;
|
||||
@@ -3088,12 +3088,12 @@ mod tests {
|
||||
description: Option<String>,
|
||||
metadata: HashMap<String, String>,
|
||||
#[serde(with = "crate::time_serde::zoned")]
|
||||
created_at: Zoned,
|
||||
created_at: ZonedDateTime,
|
||||
#[serde(with = "crate::time_serde::option_zoned")]
|
||||
rotated_at: Option<Zoned>,
|
||||
rotated_at: Option<ZonedDateTime>,
|
||||
created_by: Option<String>,
|
||||
#[serde(default, with = "crate::time_serde::option_zoned")]
|
||||
deletion_date: Option<Zoned>,
|
||||
deletion_date: Option<ZonedDateTime>,
|
||||
encrypted_key_material: String,
|
||||
nonce: Vec<u8>,
|
||||
#[serde(default)]
|
||||
@@ -4284,7 +4284,7 @@ mod tests {
|
||||
let (_stored, key_material) = client.decode_stored_key(&key_id).await.expect("decode stored key");
|
||||
let mut tombstone = client.load_master_key(&key_id).await.expect("load key");
|
||||
tombstone.status = KeyStatus::Deleted;
|
||||
tombstone.deletion_date = Some(Zoned::now());
|
||||
tombstone.deletion_date = Some(ZonedDateTime::now());
|
||||
client
|
||||
.save_master_key(&tombstone, &key_material)
|
||||
.await
|
||||
@@ -4292,7 +4292,7 @@ mod tests {
|
||||
|
||||
// The sweep primitive completes the crashed removal...
|
||||
let outcome = backend
|
||||
.remove_expired_key(&key_id, &Zoned::now())
|
||||
.remove_expired_key(&key_id, &ZonedDateTime::now())
|
||||
.await
|
||||
.expect("tombstone completion");
|
||||
assert_eq!(outcome, crate::backends::ExpiredKeyRemoval::Removed);
|
||||
@@ -4300,7 +4300,7 @@ mod tests {
|
||||
|
||||
// ...and stays idempotent once the key is gone.
|
||||
let outcome = backend
|
||||
.remove_expired_key(&key_id, &Zoned::now())
|
||||
.remove_expired_key(&key_id, &ZonedDateTime::now())
|
||||
.await
|
||||
.expect("repeat removal");
|
||||
assert_eq!(outcome, crate::backends::ExpiredKeyRemoval::Removed);
|
||||
|
||||
@@ -17,7 +17,7 @@
|
||||
use crate::error::{KmsError, Result};
|
||||
use crate::types::*;
|
||||
use async_trait::async_trait;
|
||||
use jiff::Zoned;
|
||||
use jiff::ZonedDateTime;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashMap;
|
||||
|
||||
@@ -608,7 +608,7 @@ pub trait KmsBackend: Send + Sync {
|
||||
///
|
||||
/// The default rejects the operation for backends without deletion
|
||||
/// support.
|
||||
async fn remove_expired_key(&self, _key_id: &str, _now: &Zoned) -> Result<ExpiredKeyRemoval> {
|
||||
async fn remove_expired_key(&self, _key_id: &str, _now: &ZonedDateTime) -> Result<ExpiredKeyRemoval> {
|
||||
Err(KmsError::unsupported_capability("backend without deletion support", "remove_expired_key"))
|
||||
}
|
||||
|
||||
@@ -934,7 +934,7 @@ mod tests {
|
||||
#[tokio::test]
|
||||
async fn default_remove_expired_key_is_unsupported() {
|
||||
let error = MinimalBackend
|
||||
.remove_expired_key("any-key", &jiff::Zoned::now())
|
||||
.remove_expired_key("any-key", &jiff::ZonedDateTime::now())
|
||||
.await
|
||||
.expect_err("backends without deletion support must reject expired-key removal");
|
||||
assert!(
|
||||
|
||||
@@ -34,7 +34,7 @@ use aes_gcm::{
|
||||
aead::{Aead, KeyInit, Payload},
|
||||
};
|
||||
use async_trait::async_trait;
|
||||
use jiff::Zoned;
|
||||
use jiff::ZonedDateTime;
|
||||
use rand::RngExt;
|
||||
use std::collections::HashMap;
|
||||
use tracing::debug;
|
||||
@@ -65,7 +65,7 @@ pub struct StaticKmsBackend {
|
||||
/// across a restart and differs between nodes of one cluster, because there
|
||||
/// is no creation event to anchor it to; callers must treat it as "when this
|
||||
/// node loaded the key", not as the key's birth date.
|
||||
created_at: Zoned,
|
||||
created_at: ZonedDateTime,
|
||||
}
|
||||
|
||||
impl StaticKmsBackend {
|
||||
@@ -89,7 +89,7 @@ impl StaticKmsBackend {
|
||||
Ok(Self {
|
||||
key_id: static_config.key_id.clone(),
|
||||
key: key.into(),
|
||||
created_at: Zoned::now(),
|
||||
created_at: ZonedDateTime::now(),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -149,7 +149,7 @@ impl StaticKmsBackend {
|
||||
encrypted_key: encrypted,
|
||||
nonce: nonce_bytes.to_vec(),
|
||||
encryption_context: request.encryption_context.clone(),
|
||||
created_at: Zoned::now(),
|
||||
created_at: ZonedDateTime::now(),
|
||||
// The static backend has a single fixed key with no rotation.
|
||||
master_key_version: None,
|
||||
// The context is already bound as AAD by the Static cipher path
|
||||
@@ -200,7 +200,7 @@ impl StaticKmsBackend {
|
||||
encrypted_key: encrypted,
|
||||
nonce: nonce_bytes.to_vec(),
|
||||
encryption_context: request.encryption_context.clone(),
|
||||
created_at: Zoned::now(),
|
||||
created_at: ZonedDateTime::now(),
|
||||
// The static backend has a single fixed key with no rotation.
|
||||
master_key_version: None,
|
||||
// The context is already bound as AAD by the Static cipher path
|
||||
|
||||
@@ -35,7 +35,7 @@ use crate::policy::{self, AttemptError, OpClass, OwnWrite, OwnWrites, RetryPolic
|
||||
use crate::types::*;
|
||||
use async_trait::async_trait;
|
||||
use base64_simd::STANDARD as BASE64;
|
||||
use jiff::Zoned;
|
||||
use jiff::ZonedDateTime;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashMap;
|
||||
use std::future::Future;
|
||||
@@ -90,7 +90,7 @@ struct VaultKeyData {
|
||||
/// Key usage type
|
||||
usage: KeyUsage,
|
||||
/// Key creation timestamp
|
||||
created_at: Zoned,
|
||||
created_at: ZonedDateTime,
|
||||
/// Key status
|
||||
status: KeyStatus,
|
||||
/// Key version
|
||||
@@ -104,7 +104,7 @@ struct VaultKeyData {
|
||||
/// Scheduled deletion deadline; absent on records written before deadline
|
||||
/// persistence landed, so it must stay optional for backward compatibility.
|
||||
#[serde(default)]
|
||||
deletion_date: Option<Zoned>,
|
||||
deletion_date: Option<ZonedDateTime>,
|
||||
/// When the key's material last became current through a rotation.
|
||||
///
|
||||
/// Written by [`VaultKmsClient::rotate_key`] as part of the same
|
||||
@@ -117,7 +117,7 @@ struct VaultKeyData {
|
||||
/// See [`crate::deletion_worker`] for why collapsing the two is safe for the
|
||||
/// rotation-age gauge.
|
||||
#[serde(default)]
|
||||
rotated_at: Option<Zoned>,
|
||||
rotated_at: Option<ZonedDateTime>,
|
||||
/// Encrypted key material (base64 encoded)
|
||||
encrypted_key_material: String,
|
||||
/// Version that pre-versioning envelopes (no `master_key_version`) resolve to.
|
||||
@@ -381,7 +381,7 @@ struct VaultKeyVersionRecord {
|
||||
/// Encrypted key material (base64 encoded)
|
||||
encrypted_key_material: String,
|
||||
/// When this version's material was created
|
||||
created_at: Zoned,
|
||||
created_at: ZonedDateTime,
|
||||
}
|
||||
|
||||
/// Sub-path (under each key path) reserved for immutable version records.
|
||||
@@ -1305,7 +1305,7 @@ impl VaultKmsClient {
|
||||
encrypted_key,
|
||||
nonce,
|
||||
encryption_context: request.encryption_context.clone(),
|
||||
created_at: Zoned::now(),
|
||||
created_at: ZonedDateTime::now(),
|
||||
master_key_version: Some(key_data.version),
|
||||
context_binding,
|
||||
};
|
||||
@@ -1347,7 +1347,7 @@ impl VaultKmsClient {
|
||||
encrypted_key,
|
||||
nonce,
|
||||
encryption_context: request.encryption_context.clone(),
|
||||
created_at: Zoned::now(),
|
||||
created_at: ZonedDateTime::now(),
|
||||
master_key_version: Some(key_data.version),
|
||||
context_binding,
|
||||
};
|
||||
@@ -1718,7 +1718,7 @@ impl VaultKmsClient {
|
||||
let key_data = VaultKeyData {
|
||||
algorithm: algorithm.to_string(),
|
||||
usage: attributes.map_or(KeyUsage::EncryptDecrypt, |request| request.key_usage.clone()),
|
||||
created_at: Zoned::now(),
|
||||
created_at: ZonedDateTime::now(),
|
||||
status: KeyStatus::Active,
|
||||
version: 1,
|
||||
description: attributes.and_then(|request| request.description.clone()),
|
||||
@@ -2003,7 +2003,7 @@ impl VaultKmsClient {
|
||||
let record = VaultKeyVersionRecord {
|
||||
version: new_version,
|
||||
encrypted_key_material: new_material.clone(),
|
||||
created_at: Zoned::now(),
|
||||
created_at: ZonedDateTime::now(),
|
||||
};
|
||||
if !self.try_create_key_version_record(key_id, &record).await? {
|
||||
// A record for the next version already exists: an interrupted rotation
|
||||
@@ -2027,7 +2027,7 @@ impl VaultKmsClient {
|
||||
// the recorded time never runs ahead of the current version.
|
||||
key_data.version = new_version;
|
||||
key_data.encrypted_key_material = new_material;
|
||||
key_data.rotated_at = Some(Zoned::now());
|
||||
key_data.rotated_at = Some(ZonedDateTime::now());
|
||||
// Fresh material, fresh AES-GCM nonce budget: the wrap ceiling is per
|
||||
// key material version, so the counter restarts with the same commit
|
||||
// that makes the new material current.
|
||||
@@ -2114,7 +2114,7 @@ impl VaultKmsBackend {
|
||||
/// record whose key material is missing: persisting it would cement the
|
||||
/// empty-material state under a fresh document version — a damaged key
|
||||
/// must go through an explicit repair operation, not a lifecycle update.
|
||||
async fn mark_key_pending_deletion(&self, key_id: &str, deletion_date: &Zoned) -> Result<()> {
|
||||
async fn mark_key_pending_deletion(&self, key_id: &str, deletion_date: &ZonedDateTime) -> Result<()> {
|
||||
self.client
|
||||
.update_key_data_with_cas(key_id, |key_data| {
|
||||
ensure_key_status_permits(key_id, &key_data.status, StateGatedOperation::ScheduleDeletion)?;
|
||||
@@ -2144,7 +2144,7 @@ impl KmsBackend for VaultKmsBackend {
|
||||
key_state: KeyState::Enabled,
|
||||
key_usage: request.key_usage,
|
||||
description: request.description,
|
||||
creation_date: Zoned::now(),
|
||||
creation_date: ZonedDateTime::now(),
|
||||
deletion_date: None,
|
||||
origin: "VAULT".to_string(),
|
||||
key_manager: "VAULT".to_string(),
|
||||
@@ -2287,7 +2287,7 @@ impl KmsBackend for VaultKmsBackend {
|
||||
.await?;
|
||||
self.client.delete_key(key_id).await?;
|
||||
key_metadata.key_state = KeyState::PendingDeletion;
|
||||
key_metadata.deletion_date = Some(Zoned::now());
|
||||
key_metadata.deletion_date = Some(ZonedDateTime::now());
|
||||
|
||||
// No deletion date: the key is permanently gone, not scheduled.
|
||||
None
|
||||
@@ -2306,7 +2306,7 @@ impl KmsBackend for VaultKmsBackend {
|
||||
)));
|
||||
}
|
||||
|
||||
let deletion_date = Zoned::now() + Duration::from_secs(days as u64 * 86400);
|
||||
let deletion_date = ZonedDateTime::now() + Duration::from_secs(days as u64 * 86400);
|
||||
self.mark_key_pending_deletion(key_id, &deletion_date).await?;
|
||||
key_metadata.key_state = KeyState::PendingDeletion;
|
||||
key_metadata.deletion_date = Some(deletion_date.clone());
|
||||
@@ -2415,7 +2415,7 @@ impl KmsBackend for VaultKmsBackend {
|
||||
.with_production_supported(true)
|
||||
}
|
||||
|
||||
async fn remove_expired_key(&self, key_id: &str, now: &Zoned) -> Result<ExpiredKeyRemoval> {
|
||||
async fn remove_expired_key(&self, key_id: &str, now: &ZonedDateTime) -> Result<ExpiredKeyRemoval> {
|
||||
// Tombstone under check-and-set with per-attempt re-validation: a
|
||||
// cancellation landing between the read and the write makes the write
|
||||
// conflict, and the re-read then observes the cancelled state and
|
||||
@@ -2513,7 +2513,7 @@ mod tests {
|
||||
VaultKeyData {
|
||||
algorithm: "AES_256".to_string(),
|
||||
usage: KeyUsage::EncryptDecrypt,
|
||||
created_at: Zoned::now(),
|
||||
created_at: ZonedDateTime::now(),
|
||||
status: KeyStatus::Active,
|
||||
version: 1,
|
||||
description: None,
|
||||
@@ -3197,7 +3197,7 @@ mod tests {
|
||||
let key_data = VaultKeyData {
|
||||
algorithm: "AES_256".to_string(),
|
||||
usage: KeyUsage::EncryptDecrypt,
|
||||
created_at: Zoned::now(),
|
||||
created_at: ZonedDateTime::now(),
|
||||
status: KeyStatus::Active,
|
||||
version: 1,
|
||||
description: None,
|
||||
@@ -3235,8 +3235,8 @@ mod tests {
|
||||
let mut key_data = healthy_key_data();
|
||||
key_data.wrap_budget_reserved = 7_000_000;
|
||||
key_data.baseline_version = Some(2);
|
||||
key_data.rotated_at = Some(Zoned::now());
|
||||
key_data.deletion_date = Some(Zoned::now());
|
||||
key_data.rotated_at = Some(ZonedDateTime::now());
|
||||
key_data.deletion_date = Some(ZonedDateTime::now());
|
||||
key_data.description = Some("described".to_string());
|
||||
|
||||
let value = serde_json::to_value(&key_data).expect("serialize key data");
|
||||
@@ -3683,11 +3683,11 @@ mod tests {
|
||||
/// of deadline persistence turns this test red.
|
||||
#[test]
|
||||
fn vault_key_data_deletion_date_round_trips_and_stays_backward_compatible() {
|
||||
let deadline = Zoned::now() + Duration::from_secs(7 * 86400);
|
||||
let deadline = ZonedDateTime::now() + Duration::from_secs(7 * 86400);
|
||||
let key_data = VaultKeyData {
|
||||
algorithm: "AES_256".to_string(),
|
||||
usage: KeyUsage::EncryptDecrypt,
|
||||
created_at: Zoned::now(),
|
||||
created_at: ZonedDateTime::now(),
|
||||
status: KeyStatus::PendingDeletion,
|
||||
version: 1,
|
||||
description: None,
|
||||
@@ -3703,7 +3703,7 @@ mod tests {
|
||||
let mut value = serde_json::to_value(&key_data).expect("serialize");
|
||||
let restored: VaultKeyData = serde_json::from_value(value.clone()).expect("round trip");
|
||||
assert_eq!(
|
||||
restored.deletion_date.as_ref().map(Zoned::timestamp),
|
||||
restored.deletion_date.as_ref().map(ZonedDateTime::timestamp),
|
||||
Some(deadline.timestamp()),
|
||||
"deletion deadline must survive the KV2 round trip"
|
||||
);
|
||||
@@ -4055,10 +4055,10 @@ mod tests {
|
||||
/// cancellation and destroyed the key.
|
||||
#[tokio::test]
|
||||
async fn wired_expired_key_sweep_yields_to_concurrent_cancellation() {
|
||||
let now = Zoned::now() + Duration::from_secs(3600);
|
||||
let now = ZonedDateTime::now() + Duration::from_secs(3600);
|
||||
let mut pending = healthy_key_data();
|
||||
pending.status = KeyStatus::PendingDeletion;
|
||||
pending.deletion_date = Some(Zoned::now());
|
||||
pending.deletion_date = Some(ZonedDateTime::now());
|
||||
|
||||
let vault = ScriptedVault::serve(vec![
|
||||
ScriptedResponse::ok(kv2_metadata_read_data(1)),
|
||||
@@ -4103,7 +4103,7 @@ mod tests {
|
||||
async fn wired_cancel_deletion_after_sweep_tombstone_fails_closed() {
|
||||
let mut pending = healthy_key_data();
|
||||
pending.status = KeyStatus::PendingDeletion;
|
||||
pending.deletion_date = Some(Zoned::now());
|
||||
pending.deletion_date = Some(ZonedDateTime::now());
|
||||
let mut tombstoned = healthy_key_data();
|
||||
tombstoned.status = KeyStatus::Deleted;
|
||||
|
||||
@@ -4192,7 +4192,7 @@ mod tests {
|
||||
async fn wired_immediate_deletion_of_a_pending_key_destroys_it() {
|
||||
let mut pending = healthy_key_data();
|
||||
pending.status = KeyStatus::PendingDeletion;
|
||||
pending.deletion_date = Some(Zoned::now() + Duration::from_secs(7 * 86400));
|
||||
pending.deletion_date = Some(ZonedDateTime::now() + Duration::from_secs(7 * 86400));
|
||||
|
||||
let vault = ScriptedVault::serve(vec![
|
||||
// describe_key: key info plus stored metadata.
|
||||
@@ -4328,7 +4328,7 @@ mod tests {
|
||||
async fn wired_schedule_deletion_regates_after_conflict() {
|
||||
let mut already_pending = healthy_key_data();
|
||||
already_pending.status = KeyStatus::PendingDeletion;
|
||||
already_pending.deletion_date = Some(Zoned::now() + Duration::from_secs(7 * 86400));
|
||||
already_pending.deletion_date = Some(ZonedDateTime::now() + Duration::from_secs(7 * 86400));
|
||||
|
||||
let vault = ScriptedVault::serve(vec![
|
||||
ScriptedResponse::ok(kv2_read_data(&healthy_key_data())),
|
||||
@@ -4747,7 +4747,7 @@ mod tests {
|
||||
let record_v2 = VaultKeyVersionRecord {
|
||||
version: 2,
|
||||
encrypted_key_material: base64_simd::STANDARD.encode_to_string(material_v2),
|
||||
created_at: Zoned::now(),
|
||||
created_at: ZonedDateTime::now(),
|
||||
};
|
||||
// A well-formed envelope wrapped under version 2 — under a reverted
|
||||
// guard this decrypt would *succeed*, which is exactly the masked
|
||||
@@ -4763,7 +4763,7 @@ mod tests {
|
||||
encrypted_key,
|
||||
nonce,
|
||||
encryption_context: HashMap::new(),
|
||||
created_at: Zoned::now(),
|
||||
created_at: ZonedDateTime::now(),
|
||||
master_key_version: Some(2),
|
||||
context_binding: None,
|
||||
};
|
||||
@@ -4843,7 +4843,7 @@ mod tests {
|
||||
let record_v2 = VaultKeyVersionRecord {
|
||||
version: 2,
|
||||
encrypted_key_material: adopted_material.clone(),
|
||||
created_at: Zoned::now(),
|
||||
created_at: ZonedDateTime::now(),
|
||||
};
|
||||
|
||||
let (vault, client) = scripted_client(vec![
|
||||
@@ -5042,7 +5042,7 @@ mod tests {
|
||||
/// Reverting either half turns this test red.
|
||||
#[tokio::test]
|
||||
async fn wired_rotate_persists_rotation_time_and_describe_reports_it() {
|
||||
let created_at = Zoned::now() - Duration::from_secs(365 * 86400);
|
||||
let created_at = ZonedDateTime::now() - Duration::from_secs(365 * 86400);
|
||||
let mut key_data = healthy_key_data();
|
||||
key_data.created_at = created_at.clone();
|
||||
key_data.version = 2;
|
||||
@@ -5069,7 +5069,7 @@ mod tests {
|
||||
let persisted: VaultKeyData =
|
||||
serde_json::from_value(committed["data"].clone()).expect("the committed record must deserialize");
|
||||
assert_eq!(
|
||||
persisted.rotated_at.as_ref().map(Zoned::timestamp),
|
||||
persisted.rotated_at.as_ref().map(ZonedDateTime::timestamp),
|
||||
Some(reported.timestamp()),
|
||||
"the rotation must persist the time it reports: {committed}"
|
||||
);
|
||||
@@ -5091,12 +5091,12 @@ mod tests {
|
||||
.await
|
||||
.expect("describe the rotated key");
|
||||
assert_eq!(
|
||||
described.rotated_at.as_ref().map(Zoned::timestamp),
|
||||
described.rotated_at.as_ref().map(ZonedDateTime::timestamp),
|
||||
Some(reported.timestamp()),
|
||||
"describe must report the persisted rotation time"
|
||||
);
|
||||
assert_ne!(
|
||||
described.rotated_at.as_ref().map(Zoned::timestamp),
|
||||
described.rotated_at.as_ref().map(ZonedDateTime::timestamp),
|
||||
Some(created_at.timestamp()),
|
||||
"a rotated key must not be aged from its creation"
|
||||
);
|
||||
@@ -5109,7 +5109,7 @@ mod tests {
|
||||
#[tokio::test]
|
||||
async fn wired_describe_key_invents_no_rotation_time_for_legacy_records() {
|
||||
let mut key_data = healthy_key_data();
|
||||
key_data.created_at = Zoned::now() - Duration::from_secs(365 * 86400);
|
||||
key_data.created_at = ZonedDateTime::now() - Duration::from_secs(365 * 86400);
|
||||
key_data.version = 4;
|
||||
key_data.baseline_version = Some(1);
|
||||
|
||||
@@ -5143,7 +5143,7 @@ mod tests {
|
||||
/// rest of their contents intact.
|
||||
#[test]
|
||||
fn vault_key_data_rotated_at_round_trips_and_stays_backward_compatible() {
|
||||
let rotated_at = Zoned::now();
|
||||
let rotated_at = ZonedDateTime::now();
|
||||
let mut key_data = healthy_key_data();
|
||||
key_data.rotated_at = Some(rotated_at.clone());
|
||||
key_data.baseline_version = Some(1);
|
||||
@@ -5153,7 +5153,7 @@ mod tests {
|
||||
let mut value = serde_json::to_value(&key_data).expect("serialize");
|
||||
let restored: VaultKeyData = serde_json::from_value(value.clone()).expect("round trip");
|
||||
assert_eq!(
|
||||
restored.rotated_at.as_ref().map(Zoned::timestamp),
|
||||
restored.rotated_at.as_ref().map(ZonedDateTime::timestamp),
|
||||
Some(rotated_at.timestamp()),
|
||||
"the rotation time must survive the KV2 round trip"
|
||||
);
|
||||
@@ -5190,7 +5190,7 @@ mod tests {
|
||||
encrypted_key,
|
||||
nonce,
|
||||
encryption_context: HashMap::new(),
|
||||
created_at: Zoned::now(),
|
||||
created_at: ZonedDateTime::now(),
|
||||
master_key_version: None,
|
||||
context_binding: None,
|
||||
};
|
||||
@@ -5250,7 +5250,7 @@ mod tests {
|
||||
encrypted_key,
|
||||
nonce,
|
||||
encryption_context: HashMap::new(),
|
||||
created_at: Zoned::now(),
|
||||
created_at: ZonedDateTime::now(),
|
||||
master_key_version: None,
|
||||
context_binding: None,
|
||||
};
|
||||
@@ -5302,7 +5302,7 @@ mod tests {
|
||||
encrypted_key,
|
||||
nonce,
|
||||
encryption_context: HashMap::new(),
|
||||
created_at: Zoned::now(),
|
||||
created_at: ZonedDateTime::now(),
|
||||
master_key_version: None,
|
||||
context_binding: None,
|
||||
};
|
||||
|
||||
@@ -33,7 +33,7 @@ use crate::policy::{self, AttemptError, OpClass, OwnWrite, OwnWrites, RetryPolic
|
||||
use crate::types::*;
|
||||
use async_trait::async_trait;
|
||||
use base64_simd::STANDARD as BASE64;
|
||||
use jiff::Zoned;
|
||||
use jiff::ZonedDateTime;
|
||||
use moka::future::Cache;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::{BTreeMap, HashMap};
|
||||
@@ -141,8 +141,8 @@ struct TransitKeyMetadata {
|
||||
description: Option<String>,
|
||||
tags: HashMap<String, String>,
|
||||
key_state: KeyState,
|
||||
created_at: Zoned,
|
||||
deletion_date: Option<Zoned>,
|
||||
created_at: ZonedDateTime,
|
||||
deletion_date: Option<ZonedDateTime>,
|
||||
origin: String,
|
||||
created_by: Option<String>,
|
||||
current_version: u32,
|
||||
@@ -160,8 +160,8 @@ struct TransitKeyMetadataPersisted {
|
||||
description: Option<String>,
|
||||
tags: HashMap<String, String>,
|
||||
key_state: KeyState,
|
||||
created_at: Zoned,
|
||||
deletion_date: Option<Zoned>,
|
||||
created_at: ZonedDateTime,
|
||||
deletion_date: Option<ZonedDateTime>,
|
||||
origin: String,
|
||||
created_by: Option<String>,
|
||||
current_version: u32,
|
||||
@@ -336,7 +336,7 @@ impl TransitKeyMetadata {
|
||||
description: request.description.clone(),
|
||||
tags: request.tags.clone(),
|
||||
key_state: KeyState::Enabled,
|
||||
created_at: Zoned::now(),
|
||||
created_at: ZonedDateTime::now(),
|
||||
deletion_date: None,
|
||||
origin: request.origin.clone().unwrap_or_else(|| "VAULT_TRANSIT".to_string()),
|
||||
created_by: None,
|
||||
@@ -360,7 +360,7 @@ impl TransitKeyMetadata {
|
||||
description: None,
|
||||
tags: HashMap::new(),
|
||||
key_state: KeyState::Enabled,
|
||||
created_at: Zoned::now(),
|
||||
created_at: ZonedDateTime::now(),
|
||||
deletion_date: None,
|
||||
origin: "VAULT_TRANSIT".to_string(),
|
||||
created_by: None,
|
||||
@@ -1055,7 +1055,7 @@ impl VaultTransitKmsClient {
|
||||
encrypted_key: encrypted_key.into_bytes(),
|
||||
nonce: Vec::new(),
|
||||
encryption_context: request.encryption_context.clone(),
|
||||
created_at: Zoned::now(),
|
||||
created_at: ZonedDateTime::now(),
|
||||
// Transit ciphertext already self-describes its key version
|
||||
// ("vault:vN:..."), so the envelope never carries one.
|
||||
master_key_version: None,
|
||||
@@ -1100,7 +1100,7 @@ impl VaultTransitKmsClient {
|
||||
encrypted_key: encrypted.into_bytes(),
|
||||
nonce: Vec::new(),
|
||||
encryption_context: request.encryption_context.clone(),
|
||||
created_at: Zoned::now(),
|
||||
created_at: ZonedDateTime::now(),
|
||||
// Transit ciphertext already self-describes its key version
|
||||
// ("vault:vN:..."), so the envelope never carries one.
|
||||
master_key_version: None,
|
||||
@@ -1543,7 +1543,7 @@ impl VaultTransitKmsClient {
|
||||
pending_window_days: u32,
|
||||
_context: Option<&OperationContext>,
|
||||
) -> Result<()> {
|
||||
let deletion_date = Zoned::now() + Duration::from_secs(pending_window_days as u64 * 86400);
|
||||
let deletion_date = ZonedDateTime::now() + Duration::from_secs(pending_window_days as u64 * 86400);
|
||||
self.mutate_key_metadata(key_id, |metadata| {
|
||||
ensure_key_state_permits(key_id, &metadata.key_state, StateGatedOperation::ScheduleDeletion)?;
|
||||
metadata.key_state = KeyState::PendingDeletion;
|
||||
@@ -1592,7 +1592,7 @@ impl VaultTransitKmsClient {
|
||||
description: metadata.description,
|
||||
metadata: metadata.tags,
|
||||
created_at: metadata.created_at,
|
||||
rotated_at: Some(Zoned::now()),
|
||||
rotated_at: Some(ZonedDateTime::now()),
|
||||
created_by: metadata.created_by,
|
||||
deletion_date: None,
|
||||
})
|
||||
@@ -1810,7 +1810,7 @@ impl KmsBackend for VaultTransitKmsBackend {
|
||||
// Transit key is destroyed leaves a key that is already unusable.
|
||||
// Re-checked against every fresh snapshot: a concurrent
|
||||
// cancellation that commits first fails this removal.
|
||||
let now = Zoned::now();
|
||||
let now = ZonedDateTime::now();
|
||||
self.client
|
||||
.mutate_key_metadata(&key_id, |metadata| match metadata.key_state {
|
||||
KeyState::PendingDeletion | KeyState::Unavailable => {
|
||||
@@ -1844,7 +1844,7 @@ impl KmsBackend for VaultTransitKmsBackend {
|
||||
)));
|
||||
}
|
||||
|
||||
let scheduled = Zoned::now() + Duration::from_secs(days as u64 * 86400);
|
||||
let scheduled = ZonedDateTime::now() + Duration::from_secs(days as u64 * 86400);
|
||||
self.client
|
||||
.mutate_key_metadata(&key_id, |metadata| {
|
||||
// Re-run the gate against every fresh snapshot: the check
|
||||
@@ -1934,7 +1934,7 @@ impl KmsBackend for VaultTransitKmsBackend {
|
||||
.with_production_supported(true)
|
||||
}
|
||||
|
||||
async fn remove_expired_key(&self, key_id: &str, now: &Zoned) -> Result<ExpiredKeyRemoval> {
|
||||
async fn remove_expired_key(&self, key_id: &str, now: &ZonedDateTime) -> Result<ExpiredKeyRemoval> {
|
||||
// The transit key's existence anchors "already removed": once it is
|
||||
// gone only stale scheduling metadata can remain, so clean that up.
|
||||
match self.client.read_transit_key(key_id).await {
|
||||
@@ -3220,7 +3220,7 @@ mod tests {
|
||||
encrypted_key: b"vault:v1:AAAA".to_vec(),
|
||||
nonce: Vec::new(),
|
||||
encryption_context: HashMap::new(),
|
||||
created_at: Zoned::now(),
|
||||
created_at: ZonedDateTime::now(),
|
||||
master_key_version: Some(1),
|
||||
context_binding: None,
|
||||
};
|
||||
@@ -3297,7 +3297,7 @@ mod tests {
|
||||
async fn wired_immediate_deletion_of_a_pending_key_destroys_it() {
|
||||
let mut pending = TransitKeyMetadata::from_create_request(&CreateKeyRequest::default());
|
||||
pending.key_state = KeyState::PendingDeletion;
|
||||
pending.deletion_date = Some(Zoned::now() + Duration::from_secs(7 * 86400));
|
||||
pending.deletion_date = Some(ZonedDateTime::now() + Duration::from_secs(7 * 86400));
|
||||
let vault = ScriptedVault::serve(vec![
|
||||
// Current key metadata for the response.
|
||||
ScriptedResponse::ok(transit_key_read_data("wired-key")),
|
||||
@@ -3354,7 +3354,7 @@ mod tests {
|
||||
async fn wired_expired_sweep_backs_off_when_cancel_wins_the_cas_race() {
|
||||
let mut pending = TransitKeyMetadata::from_create_request(&CreateKeyRequest::default());
|
||||
pending.key_state = KeyState::PendingDeletion;
|
||||
pending.deletion_date = Some(Zoned::now());
|
||||
pending.deletion_date = Some(ZonedDateTime::now());
|
||||
let cancelled = TransitKeyMetadata::from_create_request(&CreateKeyRequest::default());
|
||||
let vault = ScriptedVault::serve(vec![
|
||||
// The transit key still exists.
|
||||
@@ -3378,7 +3378,7 @@ mod tests {
|
||||
.await
|
||||
.expect("vault transit backend should build");
|
||||
|
||||
let now = Zoned::now() + Duration::from_secs(3600);
|
||||
let now = ZonedDateTime::now() + Duration::from_secs(3600);
|
||||
let outcome = backend
|
||||
.remove_expired_key("wired-key", &now)
|
||||
.await
|
||||
|
||||
@@ -75,7 +75,7 @@ use crate::error::{KmsError, Result};
|
||||
use crate::manager::KmsManager;
|
||||
use crate::service::ObjectEncryptionService;
|
||||
use crate::types::{CreateKeyRequest, EncryptionAlgorithm, EncryptionMetadata, KeyUsage};
|
||||
use jiff::Zoned;
|
||||
use jiff::ZonedDateTime;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::collections::HashMap;
|
||||
@@ -272,10 +272,10 @@ pub struct DrillRecoveryEvidence {
|
||||
pub struct DrillRpoEvidence {
|
||||
/// When the export sealed the bundle.
|
||||
#[serde(with = "crate::time_serde::zoned")]
|
||||
pub snapshot_completed_at: Zoned,
|
||||
pub snapshot_completed_at: ZonedDateTime,
|
||||
/// When the disaster struck.
|
||||
#[serde(with = "crate::time_serde::zoned")]
|
||||
pub disaster_at: Zoned,
|
||||
pub disaster_at: ZonedDateTime,
|
||||
/// The data-loss window: work committed inside it is not in the bundle.
|
||||
pub rpo_window_millis: u64,
|
||||
/// Keys created after the fence closed, which the bundle cannot contain.
|
||||
@@ -322,7 +322,7 @@ pub struct DrillEvidence {
|
||||
pub dataset: DrillDataset,
|
||||
/// When the run started.
|
||||
#[serde(with = "crate::time_serde::zoned")]
|
||||
pub started_at: Zoned,
|
||||
pub started_at: ZonedDateTime,
|
||||
/// Bundle evidence.
|
||||
pub bundle: DrillBundleEvidence,
|
||||
/// Recovery evidence.
|
||||
@@ -508,7 +508,7 @@ impl PhaseClock {
|
||||
/// still produces the record of *how* it failed.
|
||||
pub async fn run_local_drill(kek: &BackupKek, request: &DrillRequest) -> Result<DrillEvidence> {
|
||||
request.validate()?;
|
||||
let started_at = Zoned::now();
|
||||
let started_at = ZonedDateTime::now();
|
||||
let mut clock = PhaseClock::new();
|
||||
let mut findings = Vec::new();
|
||||
|
||||
@@ -557,7 +557,7 @@ pub async fn run_local_drill(kek: &BackupKek, request: &DrillRequest) -> Result<
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
let snapshot_completed_at = Zoned::now();
|
||||
let snapshot_completed_at = ZonedDateTime::now();
|
||||
clock.finish(DrillPhase::Export);
|
||||
|
||||
// Past the fence: this key belongs to the window the bundle cannot
|
||||
@@ -576,7 +576,7 @@ pub async fn run_local_drill(kek: &BackupKek, request: &DrillRequest) -> Result<
|
||||
let bundle_digest_before = tree_digest(&bundle_dir).await?;
|
||||
|
||||
let destroyed_files = apply_disaster(&live_dir, request.disaster).await?;
|
||||
let disaster_at = Zoned::now();
|
||||
let disaster_at = ZonedDateTime::now();
|
||||
clock.finish(DrillPhase::Disaster);
|
||||
|
||||
// Recovery step one is always the same: preserve what survived, then
|
||||
@@ -980,7 +980,7 @@ fn walk_files(root: &Path) -> std::io::Result<Vec<PathBuf>> {
|
||||
Ok(files)
|
||||
}
|
||||
|
||||
fn elapsed_millis(from: &Zoned, to: &Zoned) -> u64 {
|
||||
fn elapsed_millis(from: &ZonedDateTime, to: &ZonedDateTime) -> u64 {
|
||||
let nanos = to.timestamp().as_nanosecond() - from.timestamp().as_nanosecond();
|
||||
u64::try_from(nanos / 1_000_000).unwrap_or(0)
|
||||
}
|
||||
|
||||
@@ -63,7 +63,7 @@ use aes_gcm::{
|
||||
Aes256Gcm, Key, Nonce,
|
||||
aead::{Aead, KeyInit, Payload},
|
||||
};
|
||||
use jiff::Zoned;
|
||||
use jiff::ZonedDateTime;
|
||||
use rand::RngExt;
|
||||
use serde::Deserialize;
|
||||
use sha2::{Digest, Sha256};
|
||||
@@ -454,7 +454,7 @@ async fn build_and_write_bundle(
|
||||
// Normalized to UTC so the stored spelling is host-independent: the
|
||||
// local zone's name (or a POSIX TZ string in minimal containers) has
|
||||
// no business inside a portable bundle.
|
||||
created_at: Zoned::now().with_time_zone(jiff::tz::TimeZone::UTC),
|
||||
created_at: ZonedDateTime::now().with_time_zone(jiff::tz::TimeZone::UTC),
|
||||
rustfs_version: request.rustfs_version.clone(),
|
||||
deployment_identity: request.deployment_identity.clone(),
|
||||
backend: BackupBackendKind::Local,
|
||||
|
||||
@@ -22,7 +22,7 @@
|
||||
|
||||
use crate::backup::capability::{AtRestProtection, BackupBackendKind, BackupResponsibility};
|
||||
use crate::backup::error::BackupError;
|
||||
use jiff::Zoned;
|
||||
use jiff::ZonedDateTime;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::collections::BTreeSet;
|
||||
@@ -503,7 +503,7 @@ pub struct BackupManifest {
|
||||
pub backup_id: String,
|
||||
/// Creation time of the bundle.
|
||||
#[serde(with = "crate::time_serde::zoned")]
|
||||
pub created_at: Zoned,
|
||||
pub created_at: ZonedDateTime,
|
||||
/// RustFS version that produced the bundle.
|
||||
pub rustfs_version: String,
|
||||
/// Opaque identity of the producing deployment, used by restore preflight
|
||||
|
||||
@@ -239,7 +239,7 @@ impl KmsCache {
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::types::{KeyState, KeyUsage};
|
||||
use jiff::Zoned;
|
||||
use jiff::ZonedDateTime;
|
||||
use metrics_util::MetricKind;
|
||||
use metrics_util::debugging::{DebugValue, DebuggingRecorder};
|
||||
use std::time::Duration;
|
||||
@@ -264,7 +264,7 @@ mod tests {
|
||||
key_state: KeyState::Enabled,
|
||||
key_usage: KeyUsage::EncryptDecrypt,
|
||||
description: None,
|
||||
creation_date: Zoned::now(),
|
||||
creation_date: ZonedDateTime::now(),
|
||||
deletion_date: None,
|
||||
origin: "KMS".to_string(),
|
||||
key_manager: "CUSTOMER".to_string(),
|
||||
@@ -342,7 +342,7 @@ mod tests {
|
||||
key_state: KeyState::Enabled,
|
||||
key_usage: KeyUsage::EncryptDecrypt,
|
||||
description: Some("Test key".to_string()),
|
||||
creation_date: Zoned::now(),
|
||||
creation_date: ZonedDateTime::now(),
|
||||
deletion_date: None,
|
||||
origin: "KMS".to_string(),
|
||||
key_manager: "CUSTOMER".to_string(),
|
||||
@@ -376,7 +376,7 @@ mod tests {
|
||||
key_state: KeyState::Enabled,
|
||||
key_usage: KeyUsage::EncryptDecrypt,
|
||||
description: Some("TTL test key".to_string()),
|
||||
creation_date: Zoned::now(),
|
||||
creation_date: ZonedDateTime::now(),
|
||||
deletion_date: None,
|
||||
origin: "KMS".to_string(),
|
||||
key_manager: "CUSTOMER".to_string(),
|
||||
@@ -406,7 +406,7 @@ mod tests {
|
||||
key_state: KeyState::Enabled,
|
||||
key_usage: KeyUsage::EncryptDecrypt,
|
||||
description: None,
|
||||
creation_date: Zoned::now(),
|
||||
creation_date: ZonedDateTime::now(),
|
||||
deletion_date: None,
|
||||
origin: "KMS".to_string(),
|
||||
key_manager: "CUSTOMER".to_string(),
|
||||
|
||||
@@ -27,7 +27,7 @@ use crate::backends::{ExpiredKeyRemoval, KmsBackend};
|
||||
use crate::error::Result;
|
||||
use crate::types::{KeyInfo, KeyStatus, ListKeysRequest, OperationContext};
|
||||
use async_trait::async_trait;
|
||||
use jiff::Zoned;
|
||||
use jiff::ZonedDateTime;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
@@ -117,7 +117,7 @@ struct KeyCensus {
|
||||
}
|
||||
|
||||
impl KeyCensus {
|
||||
fn observe(&mut self, key: &KeyInfo, now: &Zoned) {
|
||||
fn observe(&mut self, key: &KeyInfo, now: &ZonedDateTime) {
|
||||
match key.status {
|
||||
KeyStatus::PendingDeletion => self.pending_deletion += 1,
|
||||
KeyStatus::Deleted => self.tombstones += 1,
|
||||
@@ -145,7 +145,7 @@ impl KeyCensus {
|
||||
|
||||
/// Seconds from `earlier` to `now`, clamped at zero so clock skew (or a
|
||||
/// timestamp persisted by a node running ahead) cannot produce a negative age.
|
||||
fn seconds_between(earlier: &Zoned, now: &Zoned) -> f64 {
|
||||
fn seconds_between(earlier: &ZonedDateTime, now: &ZonedDateTime) -> f64 {
|
||||
(now.timestamp().as_second() - earlier.timestamp().as_second()).max(0) as f64
|
||||
}
|
||||
|
||||
@@ -273,7 +273,7 @@ impl DeletionWorker {
|
||||
}
|
||||
_ = ticker.tick() => {}
|
||||
}
|
||||
let report = self.sweep(&Zoned::now()).await;
|
||||
let report = self.sweep(&ZonedDateTime::now()).await;
|
||||
if !report.removed.is_empty() || !report.blocked.is_empty() || report.failed > 0 || report.unreadable > 0 {
|
||||
info!(
|
||||
removed = ?report.removed,
|
||||
@@ -293,7 +293,7 @@ impl DeletionWorker {
|
||||
/// Also feeds the lifecycle gauges: every key it lists is counted into a
|
||||
/// [`KeyCensus`] on the way past, so the observability comes out of the
|
||||
/// pages the sweep already had to fetch.
|
||||
pub(crate) async fn sweep(&self, now: &Zoned) -> SweepReport {
|
||||
pub(crate) async fn sweep(&self, now: &ZonedDateTime) -> SweepReport {
|
||||
let mut report = SweepReport::default();
|
||||
let mut census = KeyCensus::default();
|
||||
let mut marker: Option<String> = None;
|
||||
@@ -359,7 +359,7 @@ impl DeletionWorker {
|
||||
}
|
||||
|
||||
/// Handle one key that is on its way out; reports whether it was removed.
|
||||
async fn process_key(&self, key_id: &str, now: &Zoned, report: &mut SweepReport) -> bool {
|
||||
async fn process_key(&self, key_id: &str, now: &ZonedDateTime, report: &mut SweepReport) -> bool {
|
||||
// Never remove a key that live configuration still points at. The
|
||||
// default key check is built in; broader references (bucket
|
||||
// encryption settings, ...) come from the injected checker.
|
||||
@@ -464,8 +464,8 @@ mod tests {
|
||||
DeletionWorker::new(backend, None, None, "local")
|
||||
}
|
||||
|
||||
fn after_window() -> Zoned {
|
||||
Zoned::now() + Duration::from_secs(8 * 86400)
|
||||
fn after_window() -> ZonedDateTime {
|
||||
ZonedDateTime::now() + Duration::from_secs(8 * 86400)
|
||||
}
|
||||
|
||||
async fn assert_key_gone(backend: &LocalKmsBackend, key_id: &str) {
|
||||
@@ -488,7 +488,7 @@ mod tests {
|
||||
let worker = worker(backend.clone());
|
||||
|
||||
// Not yet due: nothing happens.
|
||||
let report = worker.sweep(&Zoned::now()).await;
|
||||
let report = worker.sweep(&ZonedDateTime::now()).await;
|
||||
assert!(report.removed.is_empty());
|
||||
assert_eq!(report.skipped, 1);
|
||||
assert_eq!(report.failed, 0);
|
||||
@@ -635,7 +635,7 @@ mod tests {
|
||||
let worker = DeletionWorker::new(backend.clone(), None, None, "local").with_audit_sink(Some(sink.clone()));
|
||||
|
||||
// A key that is not yet due was never at risk, so it produces no record.
|
||||
worker.sweep(&Zoned::now()).await;
|
||||
worker.sweep(&ZonedDateTime::now()).await;
|
||||
assert!(
|
||||
sink.records.lock().expect("sink lock").is_empty(),
|
||||
"a key the sweep declined to touch must not be audited as a deletion"
|
||||
@@ -780,7 +780,7 @@ mod tests {
|
||||
.sum()
|
||||
}
|
||||
|
||||
fn key_info(key_id: &str, status: KeyStatus, created_at: Zoned, rotated_at: Option<Zoned>) -> KeyInfo {
|
||||
fn key_info(key_id: &str, status: KeyStatus, created_at: ZonedDateTime, rotated_at: Option<ZonedDateTime>) -> KeyInfo {
|
||||
KeyInfo {
|
||||
key_id: key_id.to_string(),
|
||||
description: None,
|
||||
@@ -801,7 +801,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn census_ages_from_rotation_and_ignores_departing_keys() {
|
||||
let now = Zoned::now();
|
||||
let now = ZonedDateTime::now();
|
||||
let day = Duration::from_secs(86400);
|
||||
let mut census = KeyCensus::default();
|
||||
|
||||
@@ -833,7 +833,7 @@ mod tests {
|
||||
/// material never wraps again, so its consumed nonce budget is moot.
|
||||
#[test]
|
||||
fn census_takes_the_max_wrap_reservation_of_usable_keys_only() {
|
||||
let now = Zoned::now();
|
||||
let now = ZonedDateTime::now();
|
||||
let mut census = KeyCensus::default();
|
||||
|
||||
census.observe(&key_info("uncounted", KeyStatus::Active, now.clone(), None), &now);
|
||||
@@ -859,7 +859,7 @@ mod tests {
|
||||
fn wrap_budget_gauge_is_aggregate_and_carries_no_key_label() {
|
||||
let (snapshot, ()) = record_metrics(|| {
|
||||
Box::pin(async {
|
||||
let now = Zoned::now();
|
||||
let now = ZonedDateTime::now();
|
||||
let mut census = KeyCensus::default();
|
||||
let mut wrapped = key_info("wrapped-key-id", KeyStatus::Active, now.clone(), None);
|
||||
wrapped.wrap_budget_reserved = Some(2_000_000);
|
||||
@@ -897,7 +897,7 @@ mod tests {
|
||||
// Three days in, still inside the seven-day window: the sweep
|
||||
// observes the scheduled key instead of removing it.
|
||||
let report = worker(backend.clone())
|
||||
.sweep(&(Zoned::now() + Duration::from_secs(3 * 86400)))
|
||||
.sweep(&(ZonedDateTime::now() + Duration::from_secs(3 * 86400)))
|
||||
.await;
|
||||
assert_eq!(report.skipped, 1);
|
||||
assert!(report.removed.is_empty());
|
||||
@@ -950,7 +950,7 @@ mod tests {
|
||||
|
||||
// Well inside the seven-day window: every key is observed as
|
||||
// pending rather than removed.
|
||||
let report = worker(backend.clone()).sweep(&Zoned::now()).await;
|
||||
let report = worker(backend.clone()).sweep(&ZonedDateTime::now()).await;
|
||||
assert_eq!(report.skipped, total, "every scheduled key must be inspected");
|
||||
assert!(report.removed.is_empty());
|
||||
})
|
||||
|
||||
@@ -21,7 +21,7 @@
|
||||
use crate::error::{KmsError, Result};
|
||||
use crate::persisted_observability::{BoundedUnknownFieldName, UnknownFieldSummary};
|
||||
use async_trait::async_trait;
|
||||
use jiff::Zoned;
|
||||
use jiff::ZonedDateTime;
|
||||
use rand::Rng;
|
||||
use serde::de::{self, IgnoredAny, MapAccess, Visitor};
|
||||
use serde::{Deserialize, Serialize};
|
||||
@@ -66,7 +66,7 @@ pub struct DataKeyEnvelope {
|
||||
pub nonce: Vec<u8>,
|
||||
pub encryption_context: HashMap<String, String>,
|
||||
#[serde(with = "crate::time_serde::zoned")]
|
||||
pub created_at: Zoned,
|
||||
pub created_at: ZonedDateTime,
|
||||
/// KEK version that wrapped `encrypted_key`; `None` on pre-versioning envelopes.
|
||||
///
|
||||
/// Optional and omitted when `None` so envelopes from non-rotating backends stay
|
||||
@@ -205,7 +205,7 @@ impl<'de> Deserialize<'de> for DataKeyEnvelope {
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct ZonedValue(#[serde(with = "crate::time_serde::zoned")] Zoned);
|
||||
struct ZonedValue(#[serde(with = "crate::time_serde::zoned")] ZonedDateTime);
|
||||
|
||||
struct DataKeyEnvelopeVisitor;
|
||||
|
||||
@@ -591,7 +591,7 @@ mod tests {
|
||||
encrypted_key: vec![1, 2, 3, 4],
|
||||
nonce: vec![5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16],
|
||||
encryption_context: HashMap::from([("bucket".to_string(), "b".to_string())]),
|
||||
created_at: Zoned::now(),
|
||||
created_at: ZonedDateTime::now(),
|
||||
master_key_version: None,
|
||||
context_binding: None,
|
||||
};
|
||||
@@ -625,7 +625,7 @@ mod tests {
|
||||
encrypted_key: vec![1, 2, 3, 4],
|
||||
nonce: vec![5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16],
|
||||
encryption_context: HashMap::new(),
|
||||
created_at: Zoned::now(),
|
||||
created_at: ZonedDateTime::now(),
|
||||
master_key_version: None,
|
||||
context_binding: Some(CONTEXT_BINDING_AAD_V1),
|
||||
};
|
||||
@@ -676,7 +676,7 @@ mod tests {
|
||||
map.insert("bucket".to_string(), "test-bucket".to_string());
|
||||
map
|
||||
},
|
||||
created_at: Zoned::now(),
|
||||
created_at: ZonedDateTime::now(),
|
||||
master_key_version: None,
|
||||
context_binding: None,
|
||||
};
|
||||
@@ -694,7 +694,7 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_data_key_envelope_backward_compatibility() {
|
||||
// Test deserialization with current Zoned format (with timezone annotation)
|
||||
// Test deserialization with current ZonedDateTime format (with timezone annotation)
|
||||
let envelope_json = r#"{
|
||||
"key_id": "test-key-id",
|
||||
"master_key_id": "master-key-id",
|
||||
@@ -812,7 +812,7 @@ mod tests {
|
||||
encrypted_key: vec![1, 2, 3, 4],
|
||||
nonce: vec![5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16],
|
||||
encryption_context: HashMap::new(),
|
||||
created_at: Zoned::now(),
|
||||
created_at: ZonedDateTime::now(),
|
||||
master_key_version: None,
|
||||
context_binding: None,
|
||||
};
|
||||
@@ -832,7 +832,7 @@ mod tests {
|
||||
encrypted_key: vec![1, 2, 3, 4],
|
||||
nonce: vec![5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16],
|
||||
encryption_context: HashMap::new(),
|
||||
created_at: Zoned::now(),
|
||||
created_at: ZonedDateTime::now(),
|
||||
master_key_version: Some(7),
|
||||
context_binding: None,
|
||||
};
|
||||
|
||||
@@ -28,7 +28,7 @@ use crate::types::{
|
||||
MAX_PENDING_DELETION_WINDOW_DAYS, MIN_PENDING_DELETION_WINDOW_DAYS, OperationContext, RewrapDataKeyRequest,
|
||||
RewrapDataKeyResponse, RotationDueReason,
|
||||
};
|
||||
use jiff::Zoned;
|
||||
use jiff::ZonedDateTime;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
@@ -335,7 +335,7 @@ impl KmsManager {
|
||||
let mut result = self.backend.list_keys(request).await;
|
||||
if let Ok(response) = result.as_mut() {
|
||||
let rotates = self.backend.capabilities().rotate;
|
||||
let now = Zoned::now();
|
||||
let now = ZonedDateTime::now();
|
||||
for key in &mut response.keys {
|
||||
self.apply_rotation_readiness(key, rotates, &now);
|
||||
}
|
||||
@@ -351,7 +351,7 @@ impl KmsManager {
|
||||
/// about what "overdue" means, and so a backend that cannot rotate is never
|
||||
/// the one deciding whether it should. Nothing consults the verdict before
|
||||
/// encrypting or decrypting: a key reported as due keeps serving traffic.
|
||||
fn apply_rotation_readiness(&self, key: &mut KeyInfo, backend_rotates: bool, now: &Zoned) {
|
||||
fn apply_rotation_readiness(&self, key: &mut KeyInfo, backend_rotates: bool, now: &ZonedDateTime) {
|
||||
if !backend_rotates {
|
||||
// Reported, not silently omitted: an operator chasing an overdue key
|
||||
// needs to know the answer is "this backend cannot rotate at all",
|
||||
@@ -699,7 +699,7 @@ mod tests {
|
||||
use crate::error::KmsError;
|
||||
use crate::types::{KeyMetadata, KeySpec, KeyState, KeyStatus, KeyUsage};
|
||||
use async_trait::async_trait;
|
||||
use jiff::Zoned;
|
||||
use jiff::ZonedDateTime;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Mutex;
|
||||
use std::time::Duration;
|
||||
@@ -764,7 +764,7 @@ mod tests {
|
||||
key_state: KeyState::Enabled,
|
||||
key_usage: KeyUsage::EncryptDecrypt,
|
||||
description: None,
|
||||
creation_date: Zoned::now(),
|
||||
creation_date: ZonedDateTime::now(),
|
||||
deletion_date: None,
|
||||
origin: "RUSTFS_KMS".to_string(),
|
||||
key_manager: "RUSTFS".to_string(),
|
||||
@@ -1810,7 +1810,7 @@ mod tests {
|
||||
manager
|
||||
}
|
||||
|
||||
fn aged_key(rotated_at: Option<Zoned>, created_at: Zoned) -> KeyInfo {
|
||||
fn aged_key(rotated_at: Option<ZonedDateTime>, created_at: ZonedDateTime) -> KeyInfo {
|
||||
KeyInfo {
|
||||
key_id: "key-a".to_string(),
|
||||
description: None,
|
||||
@@ -1833,7 +1833,7 @@ mod tests {
|
||||
/// a backend that cannot rotate is never told to rotate.
|
||||
#[test]
|
||||
fn rotation_readiness_reports_but_never_asks_the_impossible() {
|
||||
let now = Zoned::now();
|
||||
let now = ZonedDateTime::now();
|
||||
let long_ago = &now - jiff::Span::new().days(400);
|
||||
let recently = &now - jiff::Span::new().hours(1);
|
||||
let day = Duration::from_secs(86_400);
|
||||
@@ -1887,13 +1887,13 @@ mod tests {
|
||||
/// independently of the age policy and reported under its own reason.
|
||||
#[test]
|
||||
fn rotation_readiness_reports_an_exhausted_wrap_budget() {
|
||||
let now = Zoned::now();
|
||||
let now = ZonedDateTime::now();
|
||||
let recently = &now - jiff::Span::new().hours(1);
|
||||
let long_ago = &now - jiff::Span::new().days(400);
|
||||
let day = Duration::from_secs(86_400);
|
||||
let budget = 2_000_000;
|
||||
|
||||
let with_wraps = |manager: &KmsManager, wraps: Option<u64>, rotated_at: Option<Zoned>| {
|
||||
let with_wraps = |manager: &KmsManager, wraps: Option<u64>, rotated_at: Option<ZonedDateTime>| {
|
||||
let mut key = aged_key(rotated_at, recently.clone());
|
||||
key.wrap_budget_reserved = wraps;
|
||||
manager.apply_rotation_readiness(&mut key, true, &now);
|
||||
|
||||
@@ -595,7 +595,7 @@ mod tests {
|
||||
key_state: KeyState::Enabled,
|
||||
key_usage: KeyUsage::EncryptDecrypt,
|
||||
description: None,
|
||||
creation_date: jiff::Zoned::now(),
|
||||
creation_date: jiff::ZonedDateTime::now(),
|
||||
deletion_date: None,
|
||||
origin: "test".to_string(),
|
||||
key_manager: "test".to_string(),
|
||||
|
||||
@@ -23,7 +23,7 @@ use crate::encryption::context_aad;
|
||||
use crate::error::{KmsError, Result};
|
||||
use crate::manager::KmsManager;
|
||||
use crate::types::*;
|
||||
use jiff::Zoned;
|
||||
use jiff::ZonedDateTime;
|
||||
use md5::{Digest as Md5Digest, Md5};
|
||||
use rand::random;
|
||||
use rustfs_utils::http::object_encryption_keys::{
|
||||
@@ -543,7 +543,7 @@ impl ObjectEncryptionService {
|
||||
iv,
|
||||
tag: Some(tag),
|
||||
encryption_context: context,
|
||||
encrypted_at: Zoned::now(),
|
||||
encrypted_at: ZonedDateTime::now(),
|
||||
// Pinned to the bytes actually fed to the AEAD, so the projection
|
||||
// below can store them verbatim instead of re-deriving them.
|
||||
context_aad: Some(aad),
|
||||
@@ -702,7 +702,7 @@ impl ObjectEncryptionService {
|
||||
iv,
|
||||
tag: Some(tag),
|
||||
encryption_context: context,
|
||||
encrypted_at: Zoned::now(),
|
||||
encrypted_at: ZonedDateTime::now(),
|
||||
// Pinned to the bytes actually fed to the AEAD, so the projection
|
||||
// below can store them verbatim instead of re-deriving them.
|
||||
context_aad: Some(aad),
|
||||
@@ -953,7 +953,7 @@ impl ObjectEncryptionService {
|
||||
iv,
|
||||
tag,
|
||||
encryption_context,
|
||||
encrypted_at: Zoned::now(),
|
||||
encrypted_at: ZonedDateTime::now(),
|
||||
original_size: 0, // Not available from headers
|
||||
encrypted_data_key,
|
||||
context_aad,
|
||||
@@ -1070,7 +1070,7 @@ mod tests {
|
||||
iv: vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12],
|
||||
tag: Some(vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]),
|
||||
encryption_context: HashMap::from([("bucket".to_string(), "test-bucket".to_string())]),
|
||||
encrypted_at: Zoned::now(),
|
||||
encrypted_at: ZonedDateTime::now(),
|
||||
original_size: 100,
|
||||
encrypted_data_key: vec![1, 2, 3, 4],
|
||||
// A hand-built record with no sealed bytes to defer to.
|
||||
|
||||
@@ -1,17 +1,17 @@
|
||||
use jiff::{Timestamp, Zoned, tz::TimeZone};
|
||||
use jiff::{Timestamp, ZonedDateTime, tz::TimeZone};
|
||||
use serde::{Deserialize, Deserializer, Serializer};
|
||||
|
||||
pub(crate) mod zoned {
|
||||
use super::*;
|
||||
|
||||
pub(crate) fn serialize<S>(value: &Zoned, serializer: S) -> Result<S::Ok, S::Error>
|
||||
pub(crate) fn serialize<S>(value: &ZonedDateTime, serializer: S) -> Result<S::Ok, S::Error>
|
||||
where
|
||||
S: Serializer,
|
||||
{
|
||||
serializer.serialize_str(&value.to_string())
|
||||
}
|
||||
|
||||
pub(crate) fn deserialize<'de, D>(deserializer: D) -> Result<Zoned, D::Error>
|
||||
pub(crate) fn deserialize<'de, D>(deserializer: D) -> Result<ZonedDateTime, D::Error>
|
||||
where
|
||||
D: Deserializer<'de>,
|
||||
{
|
||||
@@ -23,7 +23,7 @@ pub(crate) mod zoned {
|
||||
pub(crate) mod option_zoned {
|
||||
use super::*;
|
||||
|
||||
pub(crate) fn serialize<S>(value: &Option<Zoned>, serializer: S) -> Result<S::Ok, S::Error>
|
||||
pub(crate) fn serialize<S>(value: &Option<ZonedDateTime>, serializer: S) -> Result<S::Ok, S::Error>
|
||||
where
|
||||
S: Serializer,
|
||||
{
|
||||
@@ -33,7 +33,7 @@ pub(crate) mod option_zoned {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn deserialize<'de, D>(deserializer: D) -> Result<Option<Zoned>, D::Error>
|
||||
pub(crate) fn deserialize<'de, D>(deserializer: D) -> Result<Option<ZonedDateTime>, D::Error>
|
||||
where
|
||||
D: Deserializer<'de>,
|
||||
{
|
||||
@@ -44,15 +44,15 @@ pub(crate) mod option_zoned {
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_zoned_compat(value: &str) -> Result<Zoned, String> {
|
||||
if let Ok(zoned) = value.parse::<Zoned>() {
|
||||
fn parse_zoned_compat(value: &str) -> Result<ZonedDateTime, String> {
|
||||
if let Ok(zoned) = value.parse::<ZonedDateTime>() {
|
||||
return Ok(zoned);
|
||||
}
|
||||
|
||||
let timestamp = value
|
||||
.parse::<Timestamp>()
|
||||
.map_err(|err| format!("failed to parse legacy timestamp '{value}': {err}"))?;
|
||||
Ok(timestamp.to_zoned(TimeZone::UTC))
|
||||
Ok(timestamp.to_zoned_datetime(TimeZone::UTC))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
+13
-13
@@ -14,7 +14,7 @@
|
||||
|
||||
//! Core type definitions for KMS operations
|
||||
|
||||
use jiff::Zoned;
|
||||
use jiff::ZonedDateTime;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashMap;
|
||||
use uuid::Uuid;
|
||||
@@ -37,7 +37,7 @@ pub struct DataKeyInfo {
|
||||
/// Associated metadata
|
||||
pub metadata: HashMap<String, String>,
|
||||
/// Key creation timestamp
|
||||
pub created_at: Zoned,
|
||||
pub created_at: ZonedDateTime,
|
||||
}
|
||||
|
||||
impl DataKeyInfo {
|
||||
@@ -61,7 +61,7 @@ impl DataKeyInfo {
|
||||
ciphertext,
|
||||
key_spec,
|
||||
metadata: HashMap::new(),
|
||||
created_at: Zoned::now(),
|
||||
created_at: ZonedDateTime::now(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -112,14 +112,14 @@ pub struct MasterKeyInfo {
|
||||
/// Associated metadata
|
||||
pub metadata: HashMap<String, String>,
|
||||
/// Key creation timestamp
|
||||
pub created_at: Zoned,
|
||||
pub created_at: ZonedDateTime,
|
||||
/// Key last rotation timestamp
|
||||
pub rotated_at: Option<Zoned>,
|
||||
pub rotated_at: Option<ZonedDateTime>,
|
||||
/// Key creator/owner
|
||||
pub created_by: Option<String>,
|
||||
/// Scheduled deletion deadline while the key is pending deletion
|
||||
#[serde(default)]
|
||||
pub deletion_date: Option<Zoned>,
|
||||
pub deletion_date: Option<ZonedDateTime>,
|
||||
}
|
||||
|
||||
impl MasterKeyInfo {
|
||||
@@ -142,7 +142,7 @@ impl MasterKeyInfo {
|
||||
status: KeyStatus::Active,
|
||||
description: None,
|
||||
metadata: HashMap::new(),
|
||||
created_at: Zoned::now(),
|
||||
created_at: ZonedDateTime::now(),
|
||||
rotated_at: None,
|
||||
created_by,
|
||||
deletion_date: None,
|
||||
@@ -174,7 +174,7 @@ impl MasterKeyInfo {
|
||||
status: KeyStatus::Active,
|
||||
description,
|
||||
metadata: HashMap::new(),
|
||||
created_at: Zoned::now(),
|
||||
created_at: ZonedDateTime::now(),
|
||||
rotated_at: None,
|
||||
created_by,
|
||||
deletion_date: None,
|
||||
@@ -247,9 +247,9 @@ pub struct KeyInfo {
|
||||
/// Key tags
|
||||
pub tags: HashMap<String, String>,
|
||||
/// Key creation timestamp
|
||||
pub created_at: Zoned,
|
||||
pub created_at: ZonedDateTime,
|
||||
/// Key last rotation timestamp
|
||||
pub rotated_at: Option<Zoned>,
|
||||
pub rotated_at: Option<ZonedDateTime>,
|
||||
/// Key creator
|
||||
pub created_by: Option<String>,
|
||||
/// Whether the key has outlived the configured rotation age.
|
||||
@@ -690,7 +690,7 @@ pub struct EncryptionMetadata {
|
||||
/// Encryption context
|
||||
pub encryption_context: HashMap<String, String>,
|
||||
/// Timestamp when encrypted
|
||||
pub encrypted_at: Zoned,
|
||||
pub encrypted_at: ZonedDateTime,
|
||||
/// Size of original data
|
||||
pub original_size: u64,
|
||||
/// Encrypted data key
|
||||
@@ -789,9 +789,9 @@ pub struct KeyMetadata {
|
||||
/// Key description
|
||||
pub description: Option<String>,
|
||||
/// Key creation timestamp
|
||||
pub creation_date: Zoned,
|
||||
pub creation_date: ZonedDateTime,
|
||||
/// Key deletion timestamp
|
||||
pub deletion_date: Option<Zoned>,
|
||||
pub deletion_date: Option<ZonedDateTime>,
|
||||
/// Key origin
|
||||
pub origin: String,
|
||||
/// Key manager
|
||||
|
||||
@@ -39,7 +39,7 @@ use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use common::{TestKms, assert_key_not_found, assert_unsupported_capability, ctx};
|
||||
use jiff::Zoned;
|
||||
use jiff::ZonedDateTime;
|
||||
use rustfs_kms::backends::local::LocalKmsBackend;
|
||||
use rustfs_kms::backends::static_kms::StaticKmsBackend;
|
||||
use rustfs_kms::backends::{ExpiredKeyRemoval, KmsBackend};
|
||||
@@ -79,8 +79,8 @@ async fn schedule(backend: &LocalKmsBackend, key_id: &str, days: u32) {
|
||||
.expect("deletion should be scheduled");
|
||||
}
|
||||
|
||||
fn days_from_now(days: u64) -> Zoned {
|
||||
Zoned::now() + Duration::from_secs(days * 86_400)
|
||||
fn days_from_now(days: u64) -> ZonedDateTime {
|
||||
ZonedDateTime::now() + Duration::from_secs(days * 86_400)
|
||||
}
|
||||
|
||||
async fn exists(backend: &LocalKmsBackend, key_id: &str) -> bool {
|
||||
@@ -422,7 +422,7 @@ async fn a_backend_without_deletion_support_reports_the_capability_gap() {
|
||||
"the static backend must not advertise deletion scheduling"
|
||||
);
|
||||
assert_unsupported_capability(
|
||||
backend.remove_expired_key(common::STATIC_KEY_ID, &Zoned::now()).await,
|
||||
backend.remove_expired_key(common::STATIC_KEY_ID, &ZonedDateTime::now()).await,
|
||||
"remove_expired_key",
|
||||
);
|
||||
}
|
||||
|
||||
@@ -39,7 +39,7 @@ hotpath-alloc = ["hotpath"]
|
||||
hotpath-cpu = ["hotpath"]
|
||||
|
||||
[dependencies]
|
||||
chrono = { workspace = true, features = ["serde"] }
|
||||
jiff = { workspace = true, features = ["serde"] }
|
||||
flate2 = { workspace = true }
|
||||
regex = { workspace = true }
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
|
||||
@@ -20,7 +20,8 @@ use crate::ingest::{IngestReport, SkipReason};
|
||||
use crate::model::{EventKind, LogEvent, LogLevel, ParseStats};
|
||||
use crate::redact;
|
||||
use crate::rules::{Finding, FindingsCollector, RuleEngine, Severity};
|
||||
use chrono::{DateTime, Duration, FixedOffset, Utc};
|
||||
use crate::time::Rfc3339;
|
||||
use jiff::{SignedDuration, ZonedDateTime};
|
||||
use regex::Regex;
|
||||
use serde::Serialize;
|
||||
use std::collections::{BTreeMap, BTreeSet, HashMap};
|
||||
@@ -35,9 +36,9 @@ const BUCKET_WIDTHS_MIN: [i64; 6] = [1, 5, 15, 60, 360, 1440];
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct AnalyzeOptions {
|
||||
/// Keep events at/after this time. Timestamp-less events always pass.
|
||||
pub since: Option<DateTime<FixedOffset>>,
|
||||
pub since: Option<ZonedDateTime>,
|
||||
/// Keep events at/before this time. Timestamp-less events always pass.
|
||||
pub until: Option<DateTime<FixedOffset>>,
|
||||
pub until: Option<ZonedDateTime>,
|
||||
/// Drop events below this level; level-less events always pass.
|
||||
pub min_level: Option<LogLevel>,
|
||||
pub max_samples: usize,
|
||||
@@ -64,7 +65,7 @@ pub struct Summary {
|
||||
pub events_after_filter: u64,
|
||||
/// "ERROR" -> n, …, "no_level" -> n.
|
||||
pub level_counts: BTreeMap<String, u64>,
|
||||
pub time_range: Option<(DateTime<FixedOffset>, DateTime<FixedOffset>)>,
|
||||
pub time_range: Option<(ZonedDateTime, ZonedDateTime)>,
|
||||
pub nodes: Vec<String>,
|
||||
pub parse: ParseStats,
|
||||
pub files_parsed: u64,
|
||||
@@ -88,7 +89,7 @@ pub struct UnmatchedCluster {
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct TimeBucket {
|
||||
/// Bucket start, normalized to UTC.
|
||||
pub start: DateTime<Utc>,
|
||||
pub start: ZonedDateTime,
|
||||
pub error: u64,
|
||||
pub warn: u64,
|
||||
pub other: u64,
|
||||
@@ -105,8 +106,8 @@ pub struct TimelineAnomaly {
|
||||
/// Involved node labels (node_ranges_disjoint); empty otherwise.
|
||||
pub nodes: Vec<String>,
|
||||
/// Gap boundaries in UTC (log_gap); null otherwise.
|
||||
pub gap_start: Option<DateTime<Utc>>,
|
||||
pub gap_end: Option<DateTime<Utc>>,
|
||||
pub gap_start: Option<ZonedDateTime>,
|
||||
pub gap_end: Option<ZonedDateTime>,
|
||||
/// log_gap only: a startup-class finding begins within 5min after the gap.
|
||||
pub restart_evidence: bool,
|
||||
}
|
||||
@@ -129,7 +130,7 @@ pub struct AnalysisReport {
|
||||
}
|
||||
|
||||
/// (min ts, max ts, timestamped event count) for one node label.
|
||||
type NodeRange = (DateTime<FixedOffset>, DateTime<FixedOffset>, u64);
|
||||
type NodeRange = (ZonedDateTime, ZonedDateTime, u64);
|
||||
|
||||
#[derive(Default)]
|
||||
struct BucketCounts {
|
||||
@@ -154,7 +155,7 @@ pub struct Analyzer {
|
||||
events_total: u64,
|
||||
events_after_filter: u64,
|
||||
level_counts: BTreeMap<String, u64>,
|
||||
time_range: Option<(DateTime<FixedOffset>, DateTime<FixedOffset>)>,
|
||||
time_range: Option<(ZonedDateTime, ZonedDateTime)>,
|
||||
nodes: BTreeSet<String>,
|
||||
offsets: BTreeSet<String>,
|
||||
/// node label -> (min ts, max ts, timestamped event count); feeds the
|
||||
@@ -189,8 +190,9 @@ impl Analyzer {
|
||||
pub fn observe(&mut self, mut event: LogEvent) {
|
||||
self.events_total += 1;
|
||||
|
||||
if let Some(ts) = event.timestamp
|
||||
&& (self.opts.since.is_some_and(|since| ts < since) || self.opts.until.is_some_and(|until| ts > until))
|
||||
if let Some(ts) = event.timestamp.as_ref()
|
||||
&& (self.opts.since.as_ref().is_some_and(|since| ts < since)
|
||||
|| self.opts.until.as_ref().is_some_and(|until| ts > until))
|
||||
{
|
||||
return;
|
||||
}
|
||||
@@ -217,23 +219,26 @@ impl Analyzer {
|
||||
self.nodes.insert(node.to_string());
|
||||
}
|
||||
|
||||
if let Some(ts) = event.timestamp {
|
||||
self.time_range = Some(match self.time_range {
|
||||
None => (ts, ts),
|
||||
Some((first, last)) => (first.min(ts), last.max(ts)),
|
||||
if let Some(ts) = event.timestamp.clone() {
|
||||
self.time_range = Some(match self.time_range.take() {
|
||||
None => (ts.clone(), ts.clone()),
|
||||
Some((first, last)) => (first.min(ts.clone()), last.max(ts.clone())),
|
||||
});
|
||||
self.offsets.insert(format_offset(&ts));
|
||||
if let Some(node) = event.node.as_deref() {
|
||||
self.node_ranges
|
||||
.entry(node.to_string())
|
||||
.and_modify(|(min, max, count)| {
|
||||
*min = (*min).min(ts);
|
||||
*max = (*max).max(ts);
|
||||
*min = min.clone().min(ts.clone());
|
||||
*max = max.clone().max(ts.clone());
|
||||
*count += 1;
|
||||
})
|
||||
.or_insert((ts, ts, 1));
|
||||
.or_insert((ts.clone(), ts.clone(), 1));
|
||||
}
|
||||
let bucket = self.minute_buckets.entry(ts.timestamp().div_euclid(60)).or_default();
|
||||
let bucket = self
|
||||
.minute_buckets
|
||||
.entry(ts.timestamp().as_second().div_euclid(60))
|
||||
.or_default();
|
||||
match event.level {
|
||||
Some(LogLevel::Error) => bucket.error += 1,
|
||||
Some(LogLevel::Warn) => bucket.warn += 1,
|
||||
@@ -356,8 +361,8 @@ impl Analyzer {
|
||||
}
|
||||
}
|
||||
|
||||
fn format_offset(ts: &DateTime<FixedOffset>) -> String {
|
||||
let seconds = ts.offset().local_minus_utc();
|
||||
fn format_offset(ts: &ZonedDateTime) -> String {
|
||||
let seconds = crate::time::offset_seconds(ts);
|
||||
if seconds == 0 {
|
||||
return "Z".to_string();
|
||||
}
|
||||
@@ -390,7 +395,7 @@ fn merge_timeline(minute_buckets: &BTreeMap<i64, BucketCounts>) -> (Vec<TimeBuck
|
||||
let buckets = merged
|
||||
.into_iter()
|
||||
.map(|(slot, counts)| TimeBucket {
|
||||
start: DateTime::<Utc>::from_timestamp(slot * width * 60, 0).expect("valid timestamp"),
|
||||
start: crate::time::utc_from_unix(slot * width * 60),
|
||||
error: counts.error,
|
||||
warn: counts.warn,
|
||||
other: counts.other,
|
||||
@@ -471,13 +476,13 @@ fn detect_timeline_anomalies(
|
||||
// span of at least max(15min, 3x display bucket width), then activity
|
||||
// again — the shape of a process restart or hang.
|
||||
let threshold = (3 * bucket_width_min).max(15);
|
||||
let startup_after_gap = |gap_end: DateTime<Utc>| {
|
||||
let startup_after_gap = |gap_end: &ZonedDateTime| {
|
||||
findings.iter().chain(low_confidence).find_map(|f| {
|
||||
if !STARTUP_RULE_IDS.contains(&f.rule_id.as_str()) {
|
||||
return None;
|
||||
}
|
||||
let first = f.first_seen?.with_timezone(&Utc);
|
||||
(first >= gap_end && first <= gap_end + Duration::minutes(5)).then(|| f.rule_id.clone())
|
||||
let first = f.first_seen.as_ref()?;
|
||||
(first >= gap_end && first <= gap_end + SignedDuration::from_mins(5)).then(|| f.rule_id.clone())
|
||||
})
|
||||
};
|
||||
let minutes: Vec<i64> = minute_buckets.keys().copied().collect();
|
||||
@@ -490,9 +495,9 @@ fn detect_timeline_anomalies(
|
||||
continue;
|
||||
}
|
||||
if run_len >= 3 && delta > threshold {
|
||||
let gap_start = DateTime::<Utc>::from_timestamp((prev + 1) * 60, 0).expect("valid timestamp");
|
||||
let gap_end = DateTime::<Utc>::from_timestamp(next * 60, 0).expect("valid timestamp");
|
||||
let restart = startup_after_gap(gap_end);
|
||||
let gap_start = crate::time::utc_from_unix((prev + 1) * 60);
|
||||
let gap_end = crate::time::utc_from_unix(next * 60);
|
||||
let restart = startup_after_gap(&gap_end);
|
||||
let span = human_duration_minutes(delta - 1);
|
||||
let message = match &restart {
|
||||
Some(rule_id) => format!(
|
||||
@@ -545,7 +550,7 @@ fn collapse_findings(findings: &mut [Finding]) {
|
||||
let index_of: HashMap<String, usize> = findings.iter().enumerate().map(|(i, f)| (f.rule_id.clone(), i)).collect();
|
||||
// Among several qualifying roots pick the earliest first_seen;
|
||||
// timestamp-less roots come last, rule id breaks remaining ties.
|
||||
let root_key = |f: &Finding| (f.first_seen.is_none(), f.first_seen, f.rule_id.clone());
|
||||
let root_key = |f: &Finding| (f.first_seen.is_none(), f.first_seen.clone(), f.rule_id.clone());
|
||||
|
||||
let mut chosen: Vec<Option<usize>> = vec![None; findings.len()];
|
||||
for (i, symptom) in findings.iter().enumerate() {
|
||||
@@ -555,10 +560,10 @@ fn collapse_findings(findings: &mut [Finding]) {
|
||||
continue;
|
||||
}
|
||||
let root = &findings[r];
|
||||
let qualifies = match (root.first_seen, symptom.first_seen) {
|
||||
let qualifies = match (root.first_seen.as_ref(), symptom.first_seen.as_ref()) {
|
||||
(Some(root_first), Some(symptom_first)) => {
|
||||
root_first <= symptom_first + Duration::minutes(5)
|
||||
&& root.last_seen.unwrap_or(root_first) >= symptom_first - Duration::minutes(30)
|
||||
root_first <= symptom_first + SignedDuration::from_mins(5)
|
||||
&& root.last_seen.as_ref().unwrap_or(root_first) >= symptom_first - SignedDuration::from_mins(30)
|
||||
}
|
||||
_ => true,
|
||||
};
|
||||
@@ -699,7 +704,7 @@ mod tests {
|
||||
|
||||
fn event(message: &str, level: Option<LogLevel>, ts: Option<&str>) -> LogEvent {
|
||||
LogEvent {
|
||||
timestamp: ts.map(|t| DateTime::parse_from_rfc3339(t).expect("ts")),
|
||||
timestamp: ts.map(|t| crate::time::parse_rfc3339(t).expect("ts")),
|
||||
level,
|
||||
target: Some("rustfs::server::http".to_string()),
|
||||
message: message.to_string(),
|
||||
@@ -815,8 +820,8 @@ mod tests {
|
||||
assert_eq!(gaps.len(), 1, "{:?}", report.timeline_anomalies);
|
||||
assert!(gaps[0].restart_evidence, "startup-fatal right after the gap: {:?}", gaps[0]);
|
||||
assert!(gaps[0].message.contains("重启证据"));
|
||||
assert_eq!(gaps[0].gap_start.expect("start").to_rfc3339(), "2026-07-15T03:04:00+00:00");
|
||||
assert_eq!(gaps[0].gap_end.expect("end").to_rfc3339(), "2026-07-15T03:44:00+00:00");
|
||||
assert_eq!(gaps[0].gap_start.as_ref().expect("start").to_rfc3339(), "2026-07-15T03:04:00+00:00");
|
||||
assert_eq!(gaps[0].gap_end.as_ref().expect("end").to_rfc3339(), "2026-07-15T03:44:00+00:00");
|
||||
|
||||
// Same hole but no startup finding afterwards: still a gap, no upgrade.
|
||||
let mut a = analyzer(AnalyzeOptions::default());
|
||||
@@ -1011,7 +1016,7 @@ mod tests {
|
||||
#[test]
|
||||
fn since_until_filters_timestamped_events_only() {
|
||||
let mut a = analyzer(AnalyzeOptions {
|
||||
since: Some(DateTime::parse_from_rfc3339("2026-07-15T03:00:00+08:00").expect("ts")),
|
||||
since: Some(crate::time::parse_rfc3339("2026-07-15T03:00:00+08:00").expect("ts")),
|
||||
..Default::default()
|
||||
});
|
||||
a.observe(event("Disk full", Some(LogLevel::Error), Some("2026-07-15T02:00:00+08:00")));
|
||||
|
||||
@@ -27,6 +27,7 @@ pub mod parse;
|
||||
mod redact;
|
||||
pub mod report;
|
||||
pub mod rules;
|
||||
pub(crate) mod time;
|
||||
|
||||
pub use analyze::{AnalysisReport, AnalyzeOptions, Analyzer, Summary, TimeBucket, TimelineAnomaly, UnmatchedCluster};
|
||||
pub use ingest::{IngestOptions, IngestReport, SkipReason, ingest_path, ingest_reader};
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
|
||||
//! Unified event model shared by every stage of the analyzer.
|
||||
|
||||
use chrono::{DateTime, FixedOffset};
|
||||
use jiff::ZonedDateTime;
|
||||
use serde::{Deserialize, Serialize, Serializer};
|
||||
use std::fmt;
|
||||
use std::sync::Arc;
|
||||
@@ -102,7 +102,7 @@ pub enum EventKind {
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct LogEvent {
|
||||
/// RFC-3339 timestamp with offset. None for panic/text lines that carry none.
|
||||
pub timestamp: Option<DateTime<FixedOffset>>,
|
||||
pub timestamp: Option<ZonedDateTime>,
|
||||
/// None when the line carries no recognizable level (text lines).
|
||||
pub level: Option<LogLevel>,
|
||||
/// tracing `target`, e.g. "rustfs::scanner::io". None for panic/text.
|
||||
|
||||
@@ -36,7 +36,7 @@ pub(crate) fn try_parse(line: &str, file: &Arc<str>, line_no: u64, node: &Option
|
||||
}
|
||||
|
||||
let timestamp = match map.remove("timestamp") {
|
||||
Some(Value::String(raw)) => match chrono::DateTime::parse_from_rfc3339(&raw) {
|
||||
Some(Value::String(raw)) => match crate::time::parse_rfc3339(&raw) {
|
||||
Ok(ts) => Some(ts),
|
||||
Err(_) => {
|
||||
map.insert("timestamp_raw".to_string(), Value::String(raw));
|
||||
|
||||
@@ -156,8 +156,8 @@ mod tests {
|
||||
assert_eq!(ev.level, Some(LogLevel::Error));
|
||||
assert_eq!(ev.target.as_deref(), Some("rustfs::scanner::io"));
|
||||
assert_eq!(ev.message, "Disk health check marked disk faulty");
|
||||
let ts = ev.timestamp.expect("timestamp");
|
||||
assert_eq!(ts.offset().local_minus_utc(), 8 * 3600);
|
||||
let ts = ev.timestamp.as_ref().expect("timestamp");
|
||||
assert_eq!(crate::time::offset_seconds(ts), 8 * 3600);
|
||||
assert_eq!(ev.field_str("reason"), Some("faulty_disk"));
|
||||
assert_eq!(ev.field_str("disk"), Some("/data/disk3"));
|
||||
assert!(ev.fields.contains_key("filename"));
|
||||
|
||||
@@ -17,6 +17,7 @@
|
||||
|
||||
use super::{human_bytes, parse_percentage};
|
||||
use crate::analyze::AnalysisReport;
|
||||
use crate::time::Rfc3339;
|
||||
use std::io;
|
||||
|
||||
/// GFM parses a table row into cells before inline code, so a literal `|`
|
||||
@@ -33,7 +34,7 @@ pub(super) fn render(report: &AnalysisReport, w: &mut dyn io::Write) -> io::Resu
|
||||
writeln!(w)?;
|
||||
writeln!(w, "| 项 | 值 |")?;
|
||||
writeln!(w, "|---|---|")?;
|
||||
match report.summary.time_range {
|
||||
match report.summary.time_range.as_ref() {
|
||||
Some((first, last)) => writeln!(w, "| 时间范围 | {} → {} |", first.to_rfc3339(), last.to_rfc3339())?,
|
||||
None => writeln!(w, "| 时间范围 | (无带时间戳的事件) |")?,
|
||||
}
|
||||
@@ -104,7 +105,7 @@ pub(super) fn render(report: &AnalysisReport, w: &mut dyn io::Write) -> io::Resu
|
||||
finding.count
|
||||
)?;
|
||||
writeln!(w)?;
|
||||
if let (Some(first), Some(last)) = (finding.first_seen, finding.last_seen) {
|
||||
if let (Some(first), Some(last)) = (finding.first_seen.as_ref(), finding.last_seen.as_ref()) {
|
||||
writeln!(w, "- 首次: {} · 末次: {}", first.to_rfc3339(), last.to_rfc3339())?;
|
||||
}
|
||||
if !finding.caused.is_empty() {
|
||||
|
||||
@@ -74,13 +74,13 @@ mod tests {
|
||||
use crate::ingest::IngestReport;
|
||||
use crate::model::{EventKind, LogEvent, LogLevel, SourceRef};
|
||||
use crate::rules::{RuleEngine, seed_rule_set};
|
||||
use chrono::DateTime;
|
||||
use crate::time::parse_rfc3339;
|
||||
use std::sync::Arc;
|
||||
|
||||
fn fixed_report() -> AnalysisReport {
|
||||
let mut analyzer = Analyzer::new(RuleEngine::new(seed_rule_set()), AnalyzeOptions::default());
|
||||
let mut ev = LogEvent {
|
||||
timestamp: Some(DateTime::parse_from_rfc3339("2026-07-15T03:12:44+08:00").expect("ts")),
|
||||
timestamp: Some(parse_rfc3339("2026-07-15T03:12:44+08:00").expect("ts")),
|
||||
level: Some(LogLevel::Error),
|
||||
target: Some("rustfs_ecstore".to_string() + "::erasure"),
|
||||
message: "erasure write quorum (required=8, achieved=5, failed=3)".to_string(),
|
||||
@@ -96,11 +96,11 @@ mod tests {
|
||||
.insert("required".to_string(), serde_json::Value::String("8".into()));
|
||||
analyzer.observe(ev.clone());
|
||||
analyzer.observe(LogEvent {
|
||||
timestamp: Some(DateTime::parse_from_rfc3339("2026-07-15T03:20:44+08:00").expect("ts")),
|
||||
timestamp: Some(parse_rfc3339("2026-07-15T03:20:44+08:00").expect("ts")),
|
||||
..ev
|
||||
});
|
||||
analyzer.observe(LogEvent {
|
||||
timestamp: Some(DateTime::parse_from_rfc3339("2026-07-15T03:25:00+08:00").expect("ts")),
|
||||
timestamp: Some(parse_rfc3339("2026-07-15T03:25:00+08:00").expect("ts")),
|
||||
level: Some(LogLevel::Error),
|
||||
target: None,
|
||||
message: "failed to sync /data/x9 after 5 retries".to_string(),
|
||||
|
||||
@@ -17,6 +17,7 @@
|
||||
use super::{human_bytes, human_duration, parse_percentage};
|
||||
use crate::analyze::AnalysisReport;
|
||||
use crate::rules::Finding;
|
||||
use crate::time::Rfc3339;
|
||||
use std::io;
|
||||
|
||||
const HEAVY_RULE: &str = "════════════════════════════════════════";
|
||||
@@ -26,9 +27,9 @@ pub(super) fn render(report: &AnalysisReport, w: &mut dyn io::Write) -> io::Resu
|
||||
writeln!(w, "RustFS 日志诊断报告")?;
|
||||
writeln!(w, "{HEAVY_RULE}")?;
|
||||
|
||||
match report.summary.time_range {
|
||||
match report.summary.time_range.as_ref() {
|
||||
Some((first, last)) => {
|
||||
let span = human_duration(last.signed_duration_since(first).num_seconds());
|
||||
let span = human_duration(last.duration_since(first).as_secs());
|
||||
writeln!(w, "时间范围 {} → {} ({span})", first.to_rfc3339(), last.to_rfc3339())?;
|
||||
}
|
||||
None => writeln!(w, "时间范围 (无带时间戳的事件)")?,
|
||||
@@ -157,7 +158,7 @@ fn render_finding(finding: &Finding, counts: &std::collections::HashMap<&str, u6
|
||||
finding.count,
|
||||
nodes
|
||||
)?;
|
||||
if let (Some(first), Some(last)) = (finding.first_seen, finding.last_seen) {
|
||||
if let (Some(first), Some(last)) = (finding.first_seen.as_ref(), finding.last_seen.as_ref()) {
|
||||
writeln!(w, " 首次 {} 末次 {}", first.to_rfc3339(), last.to_rfc3339())?;
|
||||
}
|
||||
if !finding.caused.is_empty() {
|
||||
@@ -180,7 +181,10 @@ fn render_finding(finding: &Finding, counts: &std::collections::HashMap<&str, u6
|
||||
writeln!(w, " 证据: {field} = {}{overflow}", shown.join(", "))?;
|
||||
}
|
||||
for sample in &finding.samples {
|
||||
let ts = sample.timestamp.map_or_else(|| "(无时间戳)".to_string(), |t| t.to_rfc3339());
|
||||
let ts = sample
|
||||
.timestamp
|
||||
.as_ref()
|
||||
.map_or_else(|| "(无时间戳)".to_string(), |t| t.to_rfc3339());
|
||||
let node = sample.node.as_deref().unwrap_or("-");
|
||||
writeln!(w, " 样本: {ts} {node} {}:{}", sample.source.file, sample.source.line)?;
|
||||
writeln!(w, " {}", sample.message)?;
|
||||
|
||||
@@ -18,7 +18,7 @@
|
||||
|
||||
use super::model::{Rule, Severity};
|
||||
use crate::model::LogEvent;
|
||||
use chrono::{DateTime, FixedOffset};
|
||||
use jiff::ZonedDateTime;
|
||||
use serde::Serialize;
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
|
||||
@@ -44,8 +44,8 @@ pub struct Finding {
|
||||
pub suggestion: String,
|
||||
pub count: u64,
|
||||
/// Earliest / latest timestamps among hits that carry one.
|
||||
pub first_seen: Option<DateTime<FixedOffset>>,
|
||||
pub last_seen: Option<DateTime<FixedOffset>>,
|
||||
pub first_seen: Option<ZonedDateTime>,
|
||||
pub last_seen: Option<ZonedDateTime>,
|
||||
/// Node labels of hits; label-less events are recorded as "-".
|
||||
pub nodes: BTreeSet<String>,
|
||||
/// The earliest `max_samples` hit events (timestamp-less ones sort last).
|
||||
@@ -79,28 +79,28 @@ pub struct FindingsCollector {
|
||||
/// Sort key for sample selection: earliest first, timestamp-less last,
|
||||
/// (file, line) as the deterministic tiebreaker — line numbers alone are only
|
||||
/// unique within one file, so ties across inputs would be order-dependent.
|
||||
fn sample_key(event: &LogEvent) -> (bool, Option<DateTime<FixedOffset>>, &str, u64) {
|
||||
(event.timestamp.is_none(), event.timestamp, &event.source.file, event.source.line)
|
||||
fn sample_key(event: &LogEvent) -> (bool, Option<ZonedDateTime>, &str, u64) {
|
||||
(event.timestamp.is_none(), event.timestamp.clone(), &event.source.file, event.source.line)
|
||||
}
|
||||
|
||||
/// `DateTime<FixedOffset>` orders by instant, so two equal instants carrying
|
||||
/// `ZonedDateTime` orders by instant, so two equal instants carrying
|
||||
/// different offsets compare equal and `min`/`max` keep whichever arrived
|
||||
/// first — an input-order dependency that then leaks into the serialized
|
||||
/// RFC3339 offset. Break the tie on the offset itself to stay deterministic.
|
||||
fn earliest(a: DateTime<FixedOffset>, b: DateTime<FixedOffset>) -> DateTime<FixedOffset> {
|
||||
fn earliest(a: ZonedDateTime, b: ZonedDateTime) -> ZonedDateTime {
|
||||
match a.cmp(&b) {
|
||||
std::cmp::Ordering::Less => a,
|
||||
std::cmp::Ordering::Greater => b,
|
||||
std::cmp::Ordering::Equal if a.offset().local_minus_utc() <= b.offset().local_minus_utc() => a,
|
||||
std::cmp::Ordering::Equal if crate::time::offset_seconds(&a) <= crate::time::offset_seconds(&b) => a,
|
||||
std::cmp::Ordering::Equal => b,
|
||||
}
|
||||
}
|
||||
|
||||
fn latest(a: DateTime<FixedOffset>, b: DateTime<FixedOffset>) -> DateTime<FixedOffset> {
|
||||
fn latest(a: ZonedDateTime, b: ZonedDateTime) -> ZonedDateTime {
|
||||
match a.cmp(&b) {
|
||||
std::cmp::Ordering::Greater => a,
|
||||
std::cmp::Ordering::Less => b,
|
||||
std::cmp::Ordering::Equal if a.offset().local_minus_utc() >= b.offset().local_minus_utc() => a,
|
||||
std::cmp::Ordering::Equal if crate::time::offset_seconds(&a) >= crate::time::offset_seconds(&b) => a,
|
||||
std::cmp::Ordering::Equal => b,
|
||||
}
|
||||
}
|
||||
@@ -140,9 +140,9 @@ impl FindingsCollector {
|
||||
let finding = &mut acc.finding;
|
||||
finding.count += 1;
|
||||
|
||||
if let Some(ts) = event.timestamp {
|
||||
finding.first_seen = Some(finding.first_seen.map_or(ts, |cur| earliest(cur, ts)));
|
||||
finding.last_seen = Some(finding.last_seen.map_or(ts, |cur| latest(cur, ts)));
|
||||
if let Some(ts) = event.timestamp.clone() {
|
||||
finding.first_seen = Some(finding.first_seen.take().map_or(ts.clone(), |cur| earliest(cur, ts.clone())));
|
||||
finding.last_seen = Some(finding.last_seen.take().map_or(ts.clone(), |cur| latest(cur, ts)));
|
||||
}
|
||||
|
||||
finding.nodes.insert(event.node.as_deref().unwrap_or("-").to_string());
|
||||
|
||||
@@ -40,6 +40,7 @@ pub fn seed_rule_set() -> RuleSet {
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::model::{EventKind, LogEvent, LogLevel, SourceRef};
|
||||
use crate::time::Rfc3339;
|
||||
use std::sync::Arc;
|
||||
|
||||
fn event(message: &str, level: Option<LogLevel>, line: u64) -> LogEvent {
|
||||
@@ -60,7 +61,7 @@ mod tests {
|
||||
|
||||
fn event_at(message: &str, ts: &str, line: u64) -> LogEvent {
|
||||
LogEvent {
|
||||
timestamp: Some(chrono::DateTime::parse_from_rfc3339(ts).expect("ts")),
|
||||
timestamp: Some(crate::time::parse_rfc3339(ts).expect("ts")),
|
||||
..event(message, Some(LogLevel::Error), line)
|
||||
}
|
||||
}
|
||||
@@ -224,8 +225,8 @@ mod tests {
|
||||
};
|
||||
let f = &findings[0];
|
||||
assert_eq!(f.count, 5);
|
||||
assert_eq!(f.first_seen.expect("first").to_rfc3339(), "2026-07-15T01:00:00+08:00");
|
||||
assert_eq!(f.last_seen.expect("last").to_rfc3339(), "2026-07-15T04:00:00+08:00");
|
||||
assert_eq!(f.first_seen.as_ref().expect("first").to_rfc3339(), "2026-07-15T01:00:00+08:00");
|
||||
assert_eq!(f.last_seen.as_ref().expect("last").to_rfc3339(), "2026-07-15T04:00:00+08:00");
|
||||
let sample_messages: Vec<_> = f.samples.iter().map(|s| s.message.as_str()).collect();
|
||||
assert_eq!(sample_messages, vec!["quorum e2", "quorum e3", "quorum e1"]);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
use jiff::{Timestamp, ZonedDateTime, tz::TimeZone};
|
||||
|
||||
pub(crate) trait Rfc3339 {
|
||||
fn to_rfc3339(&self) -> String;
|
||||
fn format(&self, pattern: &str) -> String;
|
||||
}
|
||||
|
||||
impl Rfc3339 for ZonedDateTime {
|
||||
fn to_rfc3339(&self) -> String {
|
||||
format_rfc3339(self)
|
||||
}
|
||||
|
||||
fn format(&self, pattern: &str) -> String {
|
||||
self.strftime(pattern).to_string()
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn parse_rfc3339(value: &str) -> Result<ZonedDateTime, jiff::Error> {
|
||||
if let Some(prefix) = value.strip_suffix('Z') {
|
||||
let normalized = format!("{prefix}+00:00");
|
||||
return ZonedDateTime::strptime("%Y-%m-%dT%H:%M:%S%.f%:z", normalized);
|
||||
}
|
||||
ZonedDateTime::strptime("%Y-%m-%dT%H:%M:%S%.f%:z", value)
|
||||
}
|
||||
|
||||
pub(crate) fn format_rfc3339(value: &ZonedDateTime) -> String {
|
||||
value.strftime("%Y-%m-%dT%H:%M:%S%.f%:z").to_string()
|
||||
}
|
||||
|
||||
pub(crate) fn utc_from_unix(seconds: i64) -> ZonedDateTime {
|
||||
Timestamp::new(seconds, 0)
|
||||
.expect("unix timestamp must fit jiff")
|
||||
.to_zoned_datetime(TimeZone::UTC)
|
||||
}
|
||||
|
||||
pub(crate) fn offset_seconds(value: &ZonedDateTime) -> i32 {
|
||||
value.offset().seconds()
|
||||
}
|
||||
@@ -30,12 +30,13 @@ workspace = true
|
||||
|
||||
[features]
|
||||
default = []
|
||||
hotpath = ["hotpath/hotpath"]
|
||||
hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc"]
|
||||
hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu"]
|
||||
# Kept as no-op compatibility features for dependents that forward the
|
||||
# workspace profiling feature set to every crate.
|
||||
hotpath = []
|
||||
hotpath-alloc = []
|
||||
hotpath-cpu = []
|
||||
|
||||
[dependencies]
|
||||
hotpath.workspace = true
|
||||
humantime.workspace = true
|
||||
http.workspace = true
|
||||
hyper = { workspace = true, features = ["http2", "http1", "server"] }
|
||||
|
||||
@@ -359,7 +359,7 @@ where
|
||||
}
|
||||
|
||||
fn format_timestamp_rfc3339_millis(value: &Timestamp) -> String {
|
||||
let utc = value.to_zoned(TimeZone::UTC);
|
||||
let utc = value.to_zoned_datetime(TimeZone::UTC);
|
||||
format!(
|
||||
"{:04}-{:02}-{:02}T{:02}:{:02}:{:02}.{:03}Z",
|
||||
utc.year(),
|
||||
|
||||
@@ -127,7 +127,6 @@ rustfs-security-governance = { workspace = true }
|
||||
rustfs-storage-api = { workspace = true }
|
||||
rustfs-utils = { workspace = true, features = ["ip"] }
|
||||
rustix = { workspace = true, features = ["fs", "stdio"] }
|
||||
chrono = { workspace = true, features = ["serde"] }
|
||||
flate2 = { workspace = true }
|
||||
glob = { workspace = true }
|
||||
jiff = { workspace = true, features = ["serde"] }
|
||||
|
||||
@@ -83,6 +83,11 @@ const GET_OBJECT_DURATION_HISTOGRAM_BUCKETS: &[f64] = &[
|
||||
0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0,
|
||||
];
|
||||
|
||||
// Keep millisecond resolution for fast requests and coarser buckets for long streams.
|
||||
const HTTP_SERVER_DURATION_HISTOGRAM_BUCKETS: &[f64] = &[
|
||||
0.0001, 0.001, 0.002, 0.003, 0.005, 0.01, 0.025, 0.05, 0.25, 1.0, 5.0, 25.0, 120.0, 600.0, 2500.0, 10000.0,
|
||||
];
|
||||
|
||||
const LOCAL_BATCH_DURATION_HISTOGRAM_METRICS: &[&str] = &[
|
||||
"rustfs_io_get_object_local_batch_prepare_seconds",
|
||||
"rustfs_io_get_object_local_batch_stage_seconds",
|
||||
@@ -522,16 +527,15 @@ fn duration_histogram_view(instrument: &Instrument) -> Option<Stream> {
|
||||
}
|
||||
|
||||
fn duration_histogram_buckets(name: &str) -> Option<&'static [f64]> {
|
||||
if LOCAL_BATCH_DURATION_HISTOGRAM_METRICS.contains(&name) {
|
||||
Some(LOCAL_BATCH_DURATION_HISTOGRAM_BUCKETS)
|
||||
} else if GET_OBJECT_DURATION_HISTOGRAM_METRICS.contains(&name) {
|
||||
Some(GET_OBJECT_DURATION_HISTOGRAM_BUCKETS)
|
||||
} else if OBJECT_LOCK_DURATION_HISTOGRAM_METRICS.contains(&name) {
|
||||
Some(OBJECT_LOCK_DURATION_HISTOGRAM_BUCKETS)
|
||||
} else if INTERNODE_DURATION_HISTOGRAM_METRICS.contains(&name) {
|
||||
Some(INTERNODE_DURATION_HISTOGRAM_BUCKETS)
|
||||
} else {
|
||||
None
|
||||
match name {
|
||||
"rustfs_http_server_request_duration_seconds"
|
||||
| "rustfs_http_server_response_body_chunk_latency_seconds"
|
||||
| "rustfs_http_server_response_body_stream_duration_seconds" => Some(HTTP_SERVER_DURATION_HISTOGRAM_BUCKETS),
|
||||
_ if LOCAL_BATCH_DURATION_HISTOGRAM_METRICS.contains(&name) => Some(LOCAL_BATCH_DURATION_HISTOGRAM_BUCKETS),
|
||||
_ if GET_OBJECT_DURATION_HISTOGRAM_METRICS.contains(&name) => Some(GET_OBJECT_DURATION_HISTOGRAM_BUCKETS),
|
||||
_ if OBJECT_LOCK_DURATION_HISTOGRAM_METRICS.contains(&name) => Some(OBJECT_LOCK_DURATION_HISTOGRAM_BUCKETS),
|
||||
_ if INTERNODE_DURATION_HISTOGRAM_METRICS.contains(&name) => Some(INTERNODE_DURATION_HISTOGRAM_BUCKETS),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1156,11 +1160,79 @@ mod tests {
|
||||
"rustfs_io_get_object_local_batch_shards",
|
||||
"rustfs_object_lock_diag_slow_acquire_total",
|
||||
"rustfs_object_lock_diag_acquire_duration_seconds_bucket",
|
||||
"rustfs_http_server_response_body_chunk_size_bytes",
|
||||
] {
|
||||
assert!(duration_histogram_buckets(name).is_none());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn http_server_duration_histograms_export_seconds_buckets() {
|
||||
use opentelemetry_sdk::metrics::{
|
||||
InMemoryMetricExporter,
|
||||
data::{AggregatedMetrics, MetricData},
|
||||
};
|
||||
|
||||
let exporter = InMemoryMetricExporter::default();
|
||||
let reader = PeriodicReader::builder(exporter.clone())
|
||||
.with_interval(Duration::from_secs(60))
|
||||
.build();
|
||||
let (provider, recorder) = Recorder::builder("http-server-duration-test")
|
||||
.with_meter_provider(|builder| builder.with_reader(reader).with_view(duration_histogram_view))
|
||||
.build();
|
||||
let instruments = [
|
||||
"rustfs_http_server_request_duration_seconds",
|
||||
"rustfs_http_server_response_body_chunk_latency_seconds",
|
||||
"rustfs_http_server_response_body_stream_duration_seconds",
|
||||
];
|
||||
let observations = [
|
||||
(0.00008, 0.0001),
|
||||
(0.0024, 0.003),
|
||||
(0.037, 0.05),
|
||||
(42.0, 120.0),
|
||||
(4000.0, 10000.0),
|
||||
(12000.0, f64::INFINITY),
|
||||
];
|
||||
metrics::with_local_recorder(&recorder, || {
|
||||
for name in instruments {
|
||||
let histogram = metrics::histogram!(name);
|
||||
for (seconds, _) in observations {
|
||||
histogram.record(seconds);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
provider.force_flush().expect("export HTTP duration observations");
|
||||
let exported = exporter.get_finished_metrics().expect("read exported HTTP histograms");
|
||||
let expected_sum = observations.iter().map(|(seconds, _)| seconds).sum::<f64>();
|
||||
let expected_buckets = observations.map(|(_, bound)| (bound, 1));
|
||||
for name in instruments {
|
||||
let metric = exported
|
||||
.iter()
|
||||
.flat_map(|resource| resource.scope_metrics())
|
||||
.flat_map(|scope| scope.metrics())
|
||||
.find(|metric| metric.name() == name)
|
||||
.unwrap_or_else(|| panic!("{name} must be exported"));
|
||||
let AggregatedMetrics::F64(MetricData::Histogram(histogram)) = metric.data() else {
|
||||
panic!("{name} must export as an f64 histogram");
|
||||
};
|
||||
let mut points = histogram.data_points();
|
||||
let point = points.next().unwrap_or_else(|| panic!("{name} must export a point"));
|
||||
assert!(points.next().is_none(), "{name} must export one point");
|
||||
assert_eq!(point.count(), 6, "{name} must retain every observation");
|
||||
assert!((point.sum() - expected_sum).abs() < 1e-9, "{name} must retain the sum in seconds");
|
||||
assert_eq!(point.bucket_counts().count(), 17, "{name} must have 16 finite buckets and overflow");
|
||||
let occupied: Vec<_> = point
|
||||
.bounds()
|
||||
.chain([f64::INFINITY])
|
||||
.zip(point.bucket_counts())
|
||||
.filter(|(_, count)| *count != 0)
|
||||
.collect();
|
||||
assert_eq!(occupied, expected_buckets, "{name}");
|
||||
}
|
||||
provider.shutdown().expect("stop HTTP histogram test provider");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn object_lock_histograms_export_millisecond_and_long_hold_buckets() {
|
||||
use metrics::Recorder as _;
|
||||
|
||||
@@ -23,8 +23,7 @@ use crate::global::{
|
||||
METRIC_LOG_CLEANER_ACTIVE_FILE_SIZE_BYTES, METRIC_LOG_CLEANER_ROTATION_DURATION_SECONDS,
|
||||
METRIC_LOG_CLEANER_ROTATION_FAILURES_TOTAL, METRIC_LOG_CLEANER_ROTATION_TOTAL,
|
||||
};
|
||||
use chrono::{Local, TimeZone as _};
|
||||
use jiff::Zoned;
|
||||
use jiff::{Timestamp, ZonedDateTime, tz::TimeZone};
|
||||
use metrics::{counter, gauge, histogram};
|
||||
use std::fs::{self, File};
|
||||
use std::io::{self, Write};
|
||||
@@ -58,8 +57,12 @@ impl Rotation {
|
||||
Rotation::Minutely => now / 60 != last / 60,
|
||||
Rotation::Hourly => now / 3600 != last / 3600,
|
||||
Rotation::Daily => {
|
||||
let last_day = Local.timestamp_opt(last, 0).single().map(|ts| ts.date_naive());
|
||||
let now_day = Local.timestamp_opt(now, 0).single().map(|ts| ts.date_naive());
|
||||
let last_day = Timestamp::new(last, 0)
|
||||
.ok()
|
||||
.map(|ts| ts.to_zoned_datetime(TimeZone::system()).date());
|
||||
let now_day = Timestamp::new(now, 0)
|
||||
.ok()
|
||||
.map(|ts| ts.to_zoned_datetime(TimeZone::system()).date());
|
||||
|
||||
match (last_day, now_day) {
|
||||
(Some(last_day), Some(now_day)) => last_day != now_day,
|
||||
@@ -131,7 +134,7 @@ impl RollingAppender {
|
||||
match_mode,
|
||||
file: None,
|
||||
size: 0,
|
||||
last_roll_ts: Zoned::now().timestamp().as_second(),
|
||||
last_roll_ts: ZonedDateTime::now().timestamp().as_second(),
|
||||
rotation_retry_cooldown_until: None,
|
||||
roll_signal: None,
|
||||
};
|
||||
@@ -225,7 +228,7 @@ impl RollingAppender {
|
||||
|
||||
// 2. Time-based check
|
||||
// We check this after size check to avoid unnecessary time calls if size forces a roll.
|
||||
let now = Zoned::now().timestamp().as_second();
|
||||
let now = ZonedDateTime::now().timestamp().as_second();
|
||||
|
||||
self.rotation.check_should_roll(self.last_roll_ts, now)
|
||||
}
|
||||
@@ -249,7 +252,7 @@ impl RollingAppender {
|
||||
// 2. Generate archive name.
|
||||
// Format: YYYYMMDDHHMMSS.uuuuuu (Microsecond/Nanosecond precision)
|
||||
// We use jiff's strftime. "%Y%m%d%H%M%S%.6f" gives microsecond precision.
|
||||
let now = Zoned::now();
|
||||
let now = ZonedDateTime::now();
|
||||
let timestamp_str = now.strftime("%Y%m%d%H%M%S%.6f").to_string();
|
||||
|
||||
// Add a unique counter to prevent collisions in high-concurrency/fast-rotation scenarios.
|
||||
@@ -667,11 +670,12 @@ mod tests {
|
||||
/// The fixtures below use mid-January dates, which sit on no DST or
|
||||
/// offset transition in any zone, so every wall-clock time exists exactly once.
|
||||
fn local_ts(year: i32, month: u32, day: u32, hour: u32, min: u32, sec: u32) -> i64 {
|
||||
Local
|
||||
.with_ymd_and_hms(year, month, day, hour, min, sec)
|
||||
.single()
|
||||
jiff::civil::DateTime::new(year as i16, month as i8, day as i8, hour as i8, min as i8, sec as i8, 0)
|
||||
.expect("fixture local time must be valid")
|
||||
.to_zoned_datetime(TimeZone::system())
|
||||
.expect("fixture local time must exist exactly once")
|
||||
.timestamp()
|
||||
.as_second()
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -31,21 +31,18 @@ workspace = true
|
||||
[features]
|
||||
default = []
|
||||
hotpath = [
|
||||
"hotpath/hotpath",
|
||||
"rustfs-config/hotpath",
|
||||
"rustfs-credentials/hotpath",
|
||||
"rustfs-crypto/hotpath",
|
||||
]
|
||||
hotpath-alloc = [
|
||||
"hotpath",
|
||||
"hotpath/hotpath-alloc",
|
||||
"rustfs-config/hotpath-alloc",
|
||||
"rustfs-credentials/hotpath-alloc",
|
||||
"rustfs-crypto/hotpath-alloc",
|
||||
]
|
||||
hotpath-cpu = [
|
||||
"hotpath",
|
||||
"hotpath/hotpath-cpu",
|
||||
"rustfs-config/hotpath-cpu",
|
||||
"rustfs-credentials/hotpath-cpu",
|
||||
"rustfs-crypto/hotpath-cpu",
|
||||
@@ -71,7 +68,6 @@ tracing.workspace = true
|
||||
moka = { workspace = true, features = ["future"] }
|
||||
async-trait.workspace = true
|
||||
futures.workspace = true
|
||||
hotpath.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
pollster.workspace = true
|
||||
|
||||
@@ -465,12 +465,18 @@ pub enum AdminAction {
|
||||
ServiceFreezeAdminAction,
|
||||
#[strum(serialize = "admin:ConfigUpdate")]
|
||||
ConfigUpdateAdminAction,
|
||||
#[strum(serialize = "admin:ManageOpenIDConfig")]
|
||||
ManageOpenIDConfigAdminAction,
|
||||
#[strum(serialize = "admin:TLSCertificatePublish")]
|
||||
TLSCertificatePublishAdminAction,
|
||||
#[strum(serialize = "admin:TLSCertificateStatus")]
|
||||
TLSCertificateStatusAdminAction,
|
||||
#[strum(serialize = "admin:CreateUser")]
|
||||
CreateUserAdminAction,
|
||||
#[strum(serialize = "admin:ChangeOwnPassword")]
|
||||
ChangeOwnPasswordAdminAction,
|
||||
#[strum(serialize = "admin:ManageOwnMFA")]
|
||||
ManageOwnMFAAdminAction,
|
||||
#[strum(serialize = "admin:DeleteUser")]
|
||||
DeleteUserAdminAction,
|
||||
#[strum(serialize = "admin:ListUsers")]
|
||||
@@ -692,9 +698,12 @@ impl AdminAction {
|
||||
| AdminAction::ServiceStopAdminAction
|
||||
| AdminAction::ServiceFreezeAdminAction
|
||||
| AdminAction::ConfigUpdateAdminAction
|
||||
| AdminAction::ManageOpenIDConfigAdminAction
|
||||
| AdminAction::TLSCertificatePublishAdminAction
|
||||
| AdminAction::TLSCertificateStatusAdminAction
|
||||
| AdminAction::CreateUserAdminAction
|
||||
| AdminAction::ChangeOwnPasswordAdminAction
|
||||
| AdminAction::ManageOwnMFAAdminAction
|
||||
| AdminAction::DeleteUserAdminAction
|
||||
| AdminAction::ListUsersAdminAction
|
||||
| AdminAction::EnableUserAdminAction
|
||||
@@ -1028,6 +1037,14 @@ mod tests {
|
||||
assert!(!Action::AdminAction(get_action).is_match(&Action::AdminAction(set_action)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn oidc_config_action_is_separate_from_general_config_writes() {
|
||||
let oidc = Action::try_from("admin:ManageOpenIDConfig").expect("parse OIDC management permission");
|
||||
assert!(!Action::AdminAction(AdminAction::ConfigUpdateAdminAction).is_match(&oidc));
|
||||
assert!(Action::AdminAction(AdminAction::AllAdminActions).is_match(&oidc));
|
||||
assert!(AdminAction::ManageOpenIDConfigAdminAction.is_valid());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tls_certificate_actions_preserve_least_privilege() {
|
||||
let publish = Action::try_from("admin:TLSCertificatePublish").expect("parse certificate publish action");
|
||||
|
||||
@@ -505,6 +505,45 @@ pub struct RenameDataResponse {
|
||||
#[prost(bytes = "bytes", tag = "4")]
|
||||
pub rename_data_resp_bin: ::prost::bytes::Bytes,
|
||||
}
|
||||
/// Ordinary rename transport batch. Protected rename variants intentionally use
|
||||
/// their dedicated fail-closed RPCs and are never admitted here.
|
||||
#[allow(clippy::all)]
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct RenameDataBatchItem {
|
||||
#[prost(uint32, tag = "1")]
|
||||
pub item_index: u32,
|
||||
#[prost(string, tag = "2")]
|
||||
pub disk: ::prost::alloc::string::String,
|
||||
#[prost(bytes = "bytes", tag = "3")]
|
||||
pub file_info_msgpack: ::prost::bytes::Bytes,
|
||||
}
|
||||
#[allow(clippy::all)]
|
||||
#[derive(Clone, PartialEq, ::prost::Message)]
|
||||
pub struct RenameDataBatchRequest {
|
||||
#[prost(string, tag = "1")]
|
||||
pub src_volume: ::prost::alloc::string::String,
|
||||
#[prost(string, tag = "2")]
|
||||
pub src_path: ::prost::alloc::string::String,
|
||||
#[prost(string, tag = "3")]
|
||||
pub dst_volume: ::prost::alloc::string::String,
|
||||
#[prost(string, tag = "4")]
|
||||
pub dst_path: ::prost::alloc::string::String,
|
||||
#[prost(message, repeated, tag = "5")]
|
||||
pub items: ::prost::alloc::vec::Vec<RenameDataBatchItem>,
|
||||
}
|
||||
#[allow(clippy::all)]
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
#[prost(skip_debug)]
|
||||
pub struct RenameDataBatchResult {
|
||||
#[prost(uint32, tag = "1")]
|
||||
pub item_index: u32,
|
||||
#[prost(bool, tag = "2")]
|
||||
pub success: bool,
|
||||
#[prost(message, optional, tag = "3")]
|
||||
pub error: ::core::option::Option<Error>,
|
||||
#[prost(bytes = "bytes", tag = "4")]
|
||||
pub rename_data_resp_msgpack: ::prost::bytes::Bytes,
|
||||
}
|
||||
/// Fixed-scope native pool.bin recovery. The typed MessagePack payload never
|
||||
/// accepts an arbitrary volume or object path.
|
||||
#[allow(clippy::all)]
|
||||
@@ -2894,6 +2933,21 @@ pub mod node_service_client {
|
||||
.insert(GrpcMethod::new("node_service.NodeService", "RenameData"));
|
||||
self.inner.unary(req, path, codec).await
|
||||
}
|
||||
pub async fn rename_data_batch(
|
||||
&mut self,
|
||||
request: impl tonic::IntoRequest<super::RenameDataBatchRequest>,
|
||||
) -> std::result::Result<tonic::Response<tonic::codec::Streaming<super::RenameDataBatchResult>>, tonic::Status> {
|
||||
self.inner
|
||||
.ready()
|
||||
.await
|
||||
.map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?;
|
||||
let codec = tonic_prost::ProstCodec::default();
|
||||
let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/RenameDataBatch");
|
||||
let mut req = request.into_request();
|
||||
req.extensions_mut()
|
||||
.insert(GrpcMethod::new("node_service.NodeService", "RenameDataBatch"));
|
||||
self.inner.server_streaming(req, path, codec).await
|
||||
}
|
||||
/// The method itself requires source-preserving multipart publication on every peer.
|
||||
pub async fn rename_multipart_data(
|
||||
&mut self,
|
||||
@@ -4252,6 +4306,14 @@ pub mod node_service_server {
|
||||
&self,
|
||||
request: tonic::Request<super::RenameDataRequest>,
|
||||
) -> std::result::Result<tonic::Response<super::RenameDataResponse>, tonic::Status>;
|
||||
/// Server streaming response type for the RenameDataBatch method.
|
||||
type RenameDataBatchStream: tonic::codegen::tokio_stream::Stream<Item = std::result::Result<super::RenameDataBatchResult, tonic::Status>>
|
||||
+ std::marker::Send
|
||||
+ 'static;
|
||||
async fn rename_data_batch(
|
||||
&self,
|
||||
request: tonic::Request<super::RenameDataBatchRequest>,
|
||||
) -> std::result::Result<tonic::Response<Self::RenameDataBatchStream>, tonic::Status>;
|
||||
/// The method itself requires source-preserving multipart publication on every peer.
|
||||
async fn rename_multipart_data(
|
||||
&self,
|
||||
@@ -5573,6 +5635,35 @@ pub mod node_service_server {
|
||||
};
|
||||
Box::pin(fut)
|
||||
}
|
||||
"/node_service.NodeService/RenameDataBatch" => {
|
||||
#[allow(non_camel_case_types)]
|
||||
struct RenameDataBatchSvc<T: NodeService>(pub Arc<T>);
|
||||
impl<T: NodeService> tonic::server::ServerStreamingService<super::RenameDataBatchRequest> for RenameDataBatchSvc<T> {
|
||||
type Response = super::RenameDataBatchResult;
|
||||
type ResponseStream = T::RenameDataBatchStream;
|
||||
type Future = BoxFuture<tonic::Response<Self::ResponseStream>, tonic::Status>;
|
||||
fn call(&mut self, request: tonic::Request<super::RenameDataBatchRequest>) -> Self::Future {
|
||||
let inner = Arc::clone(&self.0);
|
||||
let fut = async move { <T as NodeService>::rename_data_batch(&inner, request).await };
|
||||
Box::pin(fut)
|
||||
}
|
||||
}
|
||||
let accept_compression_encodings = self.accept_compression_encodings;
|
||||
let send_compression_encodings = self.send_compression_encodings;
|
||||
let max_decoding_message_size = self.max_decoding_message_size;
|
||||
let max_encoding_message_size = self.max_encoding_message_size;
|
||||
let inner = self.inner.clone();
|
||||
let fut = async move {
|
||||
let method = RenameDataBatchSvc(inner);
|
||||
let codec = tonic_prost::ProstCodec::default();
|
||||
let mut grpc = tonic::server::Grpc::new(codec)
|
||||
.apply_compression_config(accept_compression_encodings, send_compression_encodings)
|
||||
.apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size);
|
||||
let res = grpc.server_streaming(method, req).await;
|
||||
Ok(res)
|
||||
};
|
||||
Box::pin(fut)
|
||||
}
|
||||
"/node_service.NodeService/RenameMultipartData" => {
|
||||
#[allow(non_camel_case_types)]
|
||||
struct RenameMultipartDataSvc<T: NodeService>(pub Arc<T>);
|
||||
|
||||
@@ -67,6 +67,18 @@ impl std::fmt::Debug for proto_gen::node_service::RenameDataResponse {
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for proto_gen::node_service::RenameDataBatchResult {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
formatter
|
||||
.debug_struct("RenameDataBatchResult")
|
||||
.field("item_index", &self.item_index)
|
||||
.field("success", &self.success)
|
||||
.field("error", &self.error)
|
||||
.field("rename_data_resp_msgpack", &"[REDACTED]")
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for proto_gen::node_service::RecoverMultipartSourceRequest {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
formatter
|
||||
@@ -1171,6 +1183,23 @@ pub fn canonical_rename_data_request_body(
|
||||
Ok(body.finish())
|
||||
}
|
||||
|
||||
pub fn canonical_rename_data_batch_request_body(
|
||||
request: &proto_gen::node_service::RenameDataBatchRequest,
|
||||
) -> Result<Vec<u8>, std::num::TryFromIntError> {
|
||||
let mut body = CanonicalBodyBuilder::new(b"rustfs-rename-data-batch-request-v1\0");
|
||||
body.push_str(&request.src_volume)?;
|
||||
body.push_str(&request.src_path)?;
|
||||
body.push_str(&request.dst_volume)?;
|
||||
body.push_str(&request.dst_path)?;
|
||||
body.push_count(request.items.len())?;
|
||||
for item in &request.items {
|
||||
body.push_u32(item.item_index);
|
||||
body.push_str(&item.disk)?;
|
||||
body.push_bytes(&item.file_info_msgpack)?;
|
||||
}
|
||||
Ok(body.finish())
|
||||
}
|
||||
|
||||
/// The fixed-MPU recovery method authenticates every wire field in its own domain.
|
||||
pub fn canonical_recover_multipart_source_request_body(
|
||||
request: &proto_gen::node_service::RecoverMultipartSourceRequest,
|
||||
@@ -1690,9 +1719,10 @@ pub fn canonical_make_volumes_request_body(
|
||||
mod disk_mutation_canonical_tests {
|
||||
use super::proto_gen::node_service::{
|
||||
CompareAndUpdateFileRequest, DeletePathsRequest, DeleteRequest, DeleteVersionRequest, DeleteVersionsRequest,
|
||||
DeleteVolumeRequest, MakeVolumeRequest, MakeVolumesRequest, PreparePartTransactionRequest, RenameDataRequest,
|
||||
RenameFileRequest, RenamePartRequest, SettlePartTransactionRequest, SnapshotLeaseReleaseRequest,
|
||||
SnapshotLeaseRenewRequest, SnapshotLeaseRequest, UpdateMetadataRequest, WriteAllRequest, WriteMetadataRequest,
|
||||
DeleteVolumeRequest, MakeVolumeRequest, MakeVolumesRequest, PreparePartTransactionRequest, RenameDataBatchItem,
|
||||
RenameDataBatchRequest, RenameDataRequest, RenameFileRequest, RenamePartRequest, SettlePartTransactionRequest,
|
||||
SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest, SnapshotLeaseRequest, UpdateMetadataRequest, WriteAllRequest,
|
||||
WriteMetadataRequest,
|
||||
};
|
||||
use super::*;
|
||||
|
||||
@@ -1844,6 +1874,51 @@ mod disk_mutation_canonical_tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rename_data_batch_canonical_body_binds_every_field_and_item_order() {
|
||||
let baseline = RenameDataBatchRequest {
|
||||
src_volume: "src-vol".into(),
|
||||
src_path: "src-path".into(),
|
||||
dst_volume: "dst-vol".into(),
|
||||
dst_path: "dst-path".into(),
|
||||
items: vec![
|
||||
RenameDataBatchItem {
|
||||
item_index: 1,
|
||||
disk: "disk-a".into(),
|
||||
file_info_msgpack: vec![0x81, 0x01].into(),
|
||||
},
|
||||
RenameDataBatchItem {
|
||||
item_index: 2,
|
||||
disk: "disk-b".into(),
|
||||
file_info_msgpack: vec![0x81, 0x02].into(),
|
||||
},
|
||||
],
|
||||
};
|
||||
let mut bodies = vec![canonical_rename_data_batch_request_body(&baseline).expect("baseline batch body")];
|
||||
for mutate in [
|
||||
|r: &mut RenameDataBatchRequest| r.src_volume = "src-vol2".into(),
|
||||
|r: &mut RenameDataBatchRequest| r.src_path = "src-path2".into(),
|
||||
|r: &mut RenameDataBatchRequest| r.dst_volume = "dst-vol2".into(),
|
||||
|r: &mut RenameDataBatchRequest| r.dst_path = "dst-path2".into(),
|
||||
|r: &mut RenameDataBatchRequest| r.items[0].item_index = 3,
|
||||
|r: &mut RenameDataBatchRequest| r.items[0].disk = "disk-c".into(),
|
||||
|r: &mut RenameDataBatchRequest| r.items[0].file_info_msgpack = vec![0x81, 0x03].into(),
|
||||
|r: &mut RenameDataBatchRequest| r.items.swap(0, 1),
|
||||
|r: &mut RenameDataBatchRequest| {
|
||||
r.items.pop();
|
||||
},
|
||||
] {
|
||||
let mut changed = baseline.clone();
|
||||
mutate(&mut changed);
|
||||
bodies.push(canonical_rename_data_batch_request_body(&changed).expect("changed batch body"));
|
||||
}
|
||||
assert_all_distinct(&bodies);
|
||||
assert_ne!(
|
||||
canonical_rename_data_batch_request_body(&baseline).expect("batch body"),
|
||||
canonical_rename_data_request_body(&RenameDataRequest::default()).expect("unary body"),
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_version_canonical_body_binds_every_field() {
|
||||
let baseline = DeleteVersionRequest {
|
||||
@@ -3655,7 +3730,7 @@ fn complete_pending_rebuild_on_cache_hit(dial_addr: &str, cache_key: &str) -> bo
|
||||
}
|
||||
|
||||
// Limits transport response futures; client RPC metrics separately cover unary body decoding.
|
||||
const INTERNODE_RPC_CONCURRENCY_LIMIT: usize = 64;
|
||||
pub const INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT: usize = 64;
|
||||
|
||||
// Quorum reads cancel surplus metadata RPCs in bursts. Keep bounded reset state
|
||||
// for their in-flight replies without spending the connection's protocol-error
|
||||
@@ -4109,7 +4184,7 @@ async fn dial_channel(
|
||||
rustfs_config::DEFAULT_INTERNODE_BULK_CONCURRENCY_LIMIT,
|
||||
))?
|
||||
} else {
|
||||
INTERNODE_RPC_CONCURRENCY_LIMIT
|
||||
INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT
|
||||
};
|
||||
let stream_window = checked_rpc_window(stream_window)?;
|
||||
let conn_window = checked_rpc_window(conn_window)?;
|
||||
|
||||
@@ -69,6 +69,7 @@ fn main() -> Result<(), AnyError> {
|
||||
// Their hand-written Debug implementations redact the complete payload.
|
||||
.skip_debug([
|
||||
".node_service.RenameDataResponse",
|
||||
".node_service.RenameDataBatchResult",
|
||||
".node_service.RecoverMultipartSourceRequest",
|
||||
".node_service.RecoverMultipartSourceResponse",
|
||||
])
|
||||
|
||||
@@ -345,6 +345,29 @@ message RenameDataResponse {
|
||||
bytes rename_data_resp_bin = 4;
|
||||
}
|
||||
|
||||
// Ordinary rename transport batch. Protected rename variants intentionally use
|
||||
// their dedicated fail-closed RPCs and are never admitted here.
|
||||
message RenameDataBatchItem {
|
||||
uint32 item_index = 1;
|
||||
string disk = 2;
|
||||
bytes file_info_msgpack = 3;
|
||||
}
|
||||
|
||||
message RenameDataBatchRequest {
|
||||
string src_volume = 1;
|
||||
string src_path = 2;
|
||||
string dst_volume = 3;
|
||||
string dst_path = 4;
|
||||
repeated RenameDataBatchItem items = 5;
|
||||
}
|
||||
|
||||
message RenameDataBatchResult {
|
||||
uint32 item_index = 1;
|
||||
bool success = 2;
|
||||
optional Error error = 3;
|
||||
bytes rename_data_resp_msgpack = 4;
|
||||
}
|
||||
|
||||
// Fixed-scope native pool.bin recovery. The typed MessagePack payload never
|
||||
// accepts an arbitrary volume or object path.
|
||||
message PoolMetaNativeRecoveryRequest {
|
||||
@@ -1394,6 +1417,7 @@ service NodeService {
|
||||
rpc ListDir(ListDirRequest) returns (ListDirResponse) {}; // auth-policy: read-only
|
||||
rpc WalkDir(WalkDirRequest) returns (stream WalkDirResponse) {}; // auth-policy: streaming
|
||||
rpc RenameData(RenameDataRequest) returns (RenameDataResponse) {}; // auth-policy: body-bound
|
||||
rpc RenameDataBatch(RenameDataBatchRequest) returns (stream RenameDataBatchResult) {}; // auth-policy: body-bound
|
||||
// The method itself requires source-preserving multipart publication on every peer.
|
||||
rpc RenameMultipartData(RenameDataRequest) returns (RenameDataResponse) {}; // auth-policy: body-bound
|
||||
rpc RenameDataAtIncarnation(RenameDataRequest) returns (RenameDataResponse) {}; // auth-policy: body-bound
|
||||
|
||||
@@ -29,12 +29,11 @@ workspace = true
|
||||
|
||||
[features]
|
||||
default = []
|
||||
hotpath = ["hotpath/hotpath", "rustfs-s3-types/hotpath"]
|
||||
hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc", "rustfs-s3-types/hotpath-alloc"]
|
||||
hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu", "rustfs-s3-types/hotpath-cpu"]
|
||||
hotpath = ["rustfs-s3-types/hotpath"]
|
||||
hotpath-alloc = ["hotpath", "rustfs-s3-types/hotpath-alloc"]
|
||||
hotpath-cpu = ["hotpath", "rustfs-s3-types/hotpath-cpu"]
|
||||
|
||||
[dependencies]
|
||||
hotpath.workspace = true
|
||||
rustfs-s3-types = { workspace = true }
|
||||
|
||||
[lib]
|
||||
|
||||
@@ -67,6 +67,7 @@ bytes = { workspace = true, features = ["serde"] }
|
||||
base64-simd.workspace = true
|
||||
flatbuffers.workspace = true
|
||||
chrono = { workspace = true, features = ["serde"] }
|
||||
jiff = { workspace = true, features = ["serde"] }
|
||||
crc-fast.workspace = true
|
||||
rustfs-common.workspace = true
|
||||
datafusion = { workspace = true, default-features = false, features = ["parquet", "recursive_protection", "sql"] }
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user