mirror of
https://github.com/rustfs/rustfs.git
synced 2026-07-30 10:08:58 +00:00
Compare commits
2 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 830055ba9c | |||
| a8c2e3e253 |
@@ -10,16 +10,10 @@ never weaken a check to get green.
|
||||
|
||||
## `check_layer_dependencies.sh` — layer DAG in `rustfs/src`
|
||||
|
||||
Enforces `composition (server, startup/init) → interface (admin,
|
||||
storage/ecfs, storage/s3_api) → app → infra`; no upward imports. Server source
|
||||
files are composition roots, while imports of their exported HTTP contracts
|
||||
are classified as interface dependencies. Known legacy violations live in
|
||||
Enforces `interface (admin, storage/ecfs, storage/s3_api) → app → infra`; no
|
||||
upward imports. Known legacy violations live in
|
||||
`scripts/layer-dependency-baseline.txt`.
|
||||
|
||||
Dedicated `*_test.rs` and `tests/` modules are outside this production guard.
|
||||
Inline `#[cfg(test)]` imports remain checked under their source file's layer;
|
||||
move architecture-crossing test scaffolding into a dedicated test module.
|
||||
|
||||
- **New violation**: restructure your change so the dependency points
|
||||
downward (move the shared type/function to the lower layer).
|
||||
- **You legitimately removed a baseline entry**: run
|
||||
|
||||
@@ -173,10 +173,6 @@ jobs:
|
||||
run: cargo clippy --all-targets -- -D warnings
|
||||
|
||||
- name: Run nextest tests
|
||||
env:
|
||||
# Three concurrent workspace test links saturate the self-hosted
|
||||
# runner's overlay I/O and can wedge Cargo until the 75m timeout.
|
||||
CARGO_BUILD_JOBS: "2"
|
||||
run: |
|
||||
mkdir -p artifacts/test-and-lint
|
||||
# Evidence sampler for issue #5394: the post-mortem pgrep below runs
|
||||
|
||||
Generated
+130
-127
@@ -642,9 +642,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "async-compression"
|
||||
version = "0.4.43"
|
||||
version = "0.4.42"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3976abdc8fe7d1133d43d304afd42abdf5bc3e1319d263d223bde07b5efc4be8"
|
||||
checksum = "e79b3f8a79cccc2898f31920fc69f304859b3bd567490f75ebf51ae1c792a9ac"
|
||||
dependencies = [
|
||||
"compression-codecs",
|
||||
"compression-core",
|
||||
@@ -870,7 +870,7 @@ dependencies = [
|
||||
"bytes",
|
||||
"fastrand",
|
||||
"hex",
|
||||
"http 1.5.0",
|
||||
"http 1.4.2",
|
||||
"sha1 0.10.7",
|
||||
"time",
|
||||
"tokio",
|
||||
@@ -934,7 +934,7 @@ dependencies = [
|
||||
"bytes-utils",
|
||||
"fastrand",
|
||||
"http 0.2.12",
|
||||
"http 1.5.0",
|
||||
"http 1.4.2",
|
||||
"http-body 0.4.6",
|
||||
"http-body 1.1.0",
|
||||
"percent-encoding",
|
||||
@@ -970,7 +970,7 @@ dependencies = [
|
||||
"hex",
|
||||
"hmac 0.13.0",
|
||||
"http 0.2.12",
|
||||
"http 1.5.0",
|
||||
"http 1.4.2",
|
||||
"http-body 1.1.0",
|
||||
"lru 0.16.4",
|
||||
"percent-encoding",
|
||||
@@ -1001,7 +1001,7 @@ dependencies = [
|
||||
"bytes",
|
||||
"fastrand",
|
||||
"http 0.2.12",
|
||||
"http 1.5.0",
|
||||
"http 1.4.2",
|
||||
"regex-lite",
|
||||
"tracing",
|
||||
]
|
||||
@@ -1027,7 +1027,7 @@ dependencies = [
|
||||
"bytes",
|
||||
"fastrand",
|
||||
"http 0.2.12",
|
||||
"http 1.5.0",
|
||||
"http 1.4.2",
|
||||
"regex-lite",
|
||||
"tracing",
|
||||
]
|
||||
@@ -1054,7 +1054,7 @@ dependencies = [
|
||||
"aws-types",
|
||||
"fastrand",
|
||||
"http 0.2.12",
|
||||
"http 1.5.0",
|
||||
"http 1.4.2",
|
||||
"regex-lite",
|
||||
"tracing",
|
||||
]
|
||||
@@ -1076,7 +1076,7 @@ dependencies = [
|
||||
"hex",
|
||||
"hmac 0.13.0",
|
||||
"http 0.2.12",
|
||||
"http 1.5.0",
|
||||
"http 1.4.2",
|
||||
"p256 0.13.2",
|
||||
"percent-encoding",
|
||||
"sha2 0.11.0",
|
||||
@@ -1108,7 +1108,7 @@ dependencies = [
|
||||
"bytes",
|
||||
"crc-fast",
|
||||
"hex",
|
||||
"http 1.5.0",
|
||||
"http 1.4.2",
|
||||
"http-body 1.1.0",
|
||||
"http-body-util",
|
||||
"md-5 0.11.0",
|
||||
@@ -1142,7 +1142,7 @@ dependencies = [
|
||||
"bytes-utils",
|
||||
"futures-core",
|
||||
"futures-util",
|
||||
"http 1.5.0",
|
||||
"http 1.4.2",
|
||||
"http-body 1.1.0",
|
||||
"http-body-util",
|
||||
"percent-encoding",
|
||||
@@ -1161,7 +1161,7 @@ dependencies = [
|
||||
"aws-smithy-runtime-api",
|
||||
"aws-smithy-types",
|
||||
"h2",
|
||||
"http 1.5.0",
|
||||
"http 1.4.2",
|
||||
"hyper",
|
||||
"hyper-rustls",
|
||||
"hyper-util",
|
||||
@@ -1224,7 +1224,7 @@ dependencies = [
|
||||
"bytes",
|
||||
"fastrand",
|
||||
"http 0.2.12",
|
||||
"http 1.5.0",
|
||||
"http 1.4.2",
|
||||
"http-body 0.4.6",
|
||||
"http-body 1.1.0",
|
||||
"http-body-util",
|
||||
@@ -1245,7 +1245,7 @@ dependencies = [
|
||||
"aws-smithy-types",
|
||||
"bytes",
|
||||
"http 0.2.12",
|
||||
"http 1.5.0",
|
||||
"http 1.4.2",
|
||||
"pin-project-lite",
|
||||
"tokio",
|
||||
"tracing",
|
||||
@@ -1271,7 +1271,7 @@ checksum = "7d56e0a4e53127a632224e43633b0fe045fa9e1e3cfc68b9830f1115e103f910"
|
||||
dependencies = [
|
||||
"aws-smithy-runtime-api",
|
||||
"aws-smithy-types",
|
||||
"http 1.5.0",
|
||||
"http 1.4.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1285,7 +1285,7 @@ dependencies = [
|
||||
"bytes-utils",
|
||||
"futures-core",
|
||||
"http 0.2.12",
|
||||
"http 1.5.0",
|
||||
"http 1.4.2",
|
||||
"http-body 0.4.6",
|
||||
"http-body 1.1.0",
|
||||
"http-body-util",
|
||||
@@ -1337,7 +1337,7 @@ dependencies = [
|
||||
"bytes",
|
||||
"form_urlencoded",
|
||||
"futures-util",
|
||||
"http 1.5.0",
|
||||
"http 1.4.2",
|
||||
"http-body 1.1.0",
|
||||
"http-body-util",
|
||||
"hyper",
|
||||
@@ -1368,7 +1368,7 @@ checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"futures-core",
|
||||
"http 1.5.0",
|
||||
"http 1.4.2",
|
||||
"http-body 1.1.0",
|
||||
"http-body-util",
|
||||
"mime",
|
||||
@@ -3229,7 +3229,7 @@ dependencies = [
|
||||
"futures-util",
|
||||
"headers",
|
||||
"htmlescape",
|
||||
"http 1.5.0",
|
||||
"http 1.4.2",
|
||||
"http-body 1.1.0",
|
||||
"http-body-util",
|
||||
"libc",
|
||||
@@ -3654,7 +3654,7 @@ checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555"
|
||||
|
||||
[[package]]
|
||||
name = "e2e_test"
|
||||
version = "1.0.0-beta.12"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"astral-tokio-tar",
|
||||
@@ -3672,7 +3672,7 @@ dependencies = [
|
||||
"flate2",
|
||||
"futures",
|
||||
"hex",
|
||||
"http 1.5.0",
|
||||
"http 1.4.2",
|
||||
"http-body-util",
|
||||
"hyper",
|
||||
"hyper-util",
|
||||
@@ -4384,7 +4384,7 @@ dependencies = [
|
||||
"google-cloud-gax",
|
||||
"hex",
|
||||
"hmac 0.13.0",
|
||||
"http 1.5.0",
|
||||
"http 1.4.2",
|
||||
"jsonwebtoken 10.4.0",
|
||||
"reqwest",
|
||||
"rustc_version",
|
||||
@@ -4409,7 +4409,7 @@ dependencies = [
|
||||
"futures",
|
||||
"google-cloud-rpc",
|
||||
"google-cloud-wkt",
|
||||
"http 1.5.0",
|
||||
"http 1.4.2",
|
||||
"pin-project",
|
||||
"rand 0.10.2",
|
||||
"serde",
|
||||
@@ -4431,7 +4431,7 @@ dependencies = [
|
||||
"google-cloud-rpc",
|
||||
"google-cloud-wkt",
|
||||
"h2",
|
||||
"http 1.5.0",
|
||||
"http 1.4.2",
|
||||
"http-body 1.1.0",
|
||||
"http-body-util",
|
||||
"hyper",
|
||||
@@ -4544,7 +4544,7 @@ dependencies = [
|
||||
"google-cloud-type",
|
||||
"google-cloud-wkt",
|
||||
"hex",
|
||||
"http 1.5.0",
|
||||
"http 1.4.2",
|
||||
"http-body 1.1.0",
|
||||
"md5",
|
||||
"percent-encoding",
|
||||
@@ -4624,7 +4624,7 @@ dependencies = [
|
||||
"fnv",
|
||||
"futures-core",
|
||||
"futures-sink",
|
||||
"http 1.5.0",
|
||||
"http 1.4.2",
|
||||
"indexmap 2.14.0",
|
||||
"slab",
|
||||
"tokio",
|
||||
@@ -4727,7 +4727,7 @@ dependencies = [
|
||||
"base64 0.22.1",
|
||||
"bytes",
|
||||
"headers-core",
|
||||
"http 1.5.0",
|
||||
"http 1.4.2",
|
||||
"httpdate",
|
||||
"mime",
|
||||
"sha1 0.10.7",
|
||||
@@ -4739,7 +4739,7 @@ version = "0.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "54b4a22553d4242c49fddb9ba998a99962b5cc6f22cb5a3482bec22522403ce4"
|
||||
dependencies = [
|
||||
"http 1.5.0",
|
||||
"http 1.4.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -4961,9 +4961,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "http"
|
||||
version = "1.5.0"
|
||||
version = "1.4.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0"
|
||||
checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"itoa",
|
||||
@@ -4987,7 +4987,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"http 1.5.0",
|
||||
"http 1.4.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -4998,7 +4998,7 @@ checksum = "e9f41fd6a08e4d4ec69df65976da761afd5ad5e58a9d4acb46bd1c953a9e3ff2"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"futures-core",
|
||||
"http 1.5.0",
|
||||
"http 1.4.2",
|
||||
"http-body 1.1.0",
|
||||
"pin-project-lite",
|
||||
]
|
||||
@@ -5044,7 +5044,7 @@ dependencies = [
|
||||
"futures-channel",
|
||||
"futures-core",
|
||||
"h2",
|
||||
"http 1.5.0",
|
||||
"http 1.4.2",
|
||||
"http-body 1.1.0",
|
||||
"httparse",
|
||||
"httpdate",
|
||||
@@ -5061,7 +5061,7 @@ version = "0.27.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f"
|
||||
dependencies = [
|
||||
"http 1.5.0",
|
||||
"http 1.4.2",
|
||||
"hyper",
|
||||
"hyper-util",
|
||||
"log",
|
||||
@@ -5095,7 +5095,7 @@ dependencies = [
|
||||
"bytes",
|
||||
"futures-channel",
|
||||
"futures-util",
|
||||
"http 1.5.0",
|
||||
"http 1.4.2",
|
||||
"http-body 1.1.0",
|
||||
"hyper",
|
||||
"ipnet",
|
||||
@@ -6704,10 +6704,10 @@ version = "5.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "51e219e79014df21a225b1860a479e2dcd7cbd9130f4defd4bd0e191ea31d67d"
|
||||
dependencies = [
|
||||
"base64 0.21.7",
|
||||
"base64 0.22.1",
|
||||
"chrono",
|
||||
"getrandom 0.2.17",
|
||||
"http 1.5.0",
|
||||
"http 1.4.2",
|
||||
"rand 0.8.7",
|
||||
"serde",
|
||||
"serde_json",
|
||||
@@ -6815,7 +6815,7 @@ dependencies = [
|
||||
"futures-channel",
|
||||
"futures-core",
|
||||
"futures-util",
|
||||
"http 1.5.0",
|
||||
"http 1.4.2",
|
||||
"humantime",
|
||||
"itertools 0.14.0",
|
||||
"parking_lot",
|
||||
@@ -6871,7 +6871,7 @@ dependencies = [
|
||||
"dyn-clone",
|
||||
"ed25519-dalek 2.2.0",
|
||||
"hmac 0.12.1",
|
||||
"http 1.5.0",
|
||||
"http 1.4.2",
|
||||
"itertools 0.10.5",
|
||||
"log",
|
||||
"oauth2",
|
||||
@@ -6932,7 +6932,7 @@ checksum = "5683015d09e2df236ef005b17f6f196f0d5f6313c4fa43a7b6a53b52776e4331"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"bytes",
|
||||
"http 1.5.0",
|
||||
"http 1.4.2",
|
||||
"opentelemetry",
|
||||
"reqwest",
|
||||
]
|
||||
@@ -6944,7 +6944,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9966929966d17620d7c316c643ba62631826e10021409357772d5eea84f62c35"
|
||||
dependencies = [
|
||||
"flate2",
|
||||
"http 1.5.0",
|
||||
"http 1.4.2",
|
||||
"opentelemetry",
|
||||
"opentelemetry-http",
|
||||
"opentelemetry-proto",
|
||||
@@ -7237,6 +7237,12 @@ dependencies = [
|
||||
"path-dedot",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "path-clean"
|
||||
version = "1.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "17359afc20d7ab31fdb42bb844c8b3bb1dabd7dcf7e68428492da7f16966fcef"
|
||||
|
||||
[[package]]
|
||||
name = "path-dedot"
|
||||
version = "4.0.1"
|
||||
@@ -7820,7 +7826,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf"
|
||||
dependencies = [
|
||||
"heck",
|
||||
"itertools 0.10.5",
|
||||
"itertools 0.14.0",
|
||||
"log",
|
||||
"multimap",
|
||||
"once_cell",
|
||||
@@ -7840,7 +7846,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042"
|
||||
dependencies = [
|
||||
"heck",
|
||||
"itertools 0.10.5",
|
||||
"itertools 0.14.0",
|
||||
"log",
|
||||
"multimap",
|
||||
"petgraph 0.8.3",
|
||||
@@ -7861,7 +7867,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"itertools 0.10.5",
|
||||
"itertools 0.14.0",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.119",
|
||||
@@ -7874,7 +7880,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"itertools 0.10.5",
|
||||
"itertools 0.14.0",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.119",
|
||||
@@ -8329,9 +8335,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "redis"
|
||||
version = "1.5.0"
|
||||
version = "1.4.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3257df217f7eab0044627a268c9cc6cdb60c0c421c88f83ac41c4e31520b6b84"
|
||||
checksum = "b0b9503711b03773e43b31668c7b5bd279ee7cd9b7d18cff7c23a42cc1d08e5a"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"arcstr",
|
||||
@@ -8481,7 +8487,7 @@ dependencies = [
|
||||
"futures-core",
|
||||
"futures-util",
|
||||
"h2",
|
||||
"http 1.5.0",
|
||||
"http 1.4.2",
|
||||
"http-body 1.1.0",
|
||||
"http-body-util",
|
||||
"hyper",
|
||||
@@ -8627,7 +8633,7 @@ dependencies = [
|
||||
"async-tungstenite",
|
||||
"futures-io",
|
||||
"futures-util",
|
||||
"http 1.5.0",
|
||||
"http 1.4.2",
|
||||
"rustls-native-certs",
|
||||
"rustls-pki-types",
|
||||
"rustls-webpki",
|
||||
@@ -8657,7 +8663,7 @@ dependencies = [
|
||||
"flume",
|
||||
"futures-io",
|
||||
"futures-util",
|
||||
"http 1.5.0",
|
||||
"http 1.4.2",
|
||||
"log",
|
||||
"mqttbytes-core-next",
|
||||
"rumqttc-core-next",
|
||||
@@ -8845,7 +8851,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs"
|
||||
version = "1.0.0-beta.12"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"aes-gcm",
|
||||
"anyhow",
|
||||
@@ -8874,7 +8880,7 @@ dependencies = [
|
||||
"hex-simd",
|
||||
"hmac 0.13.0",
|
||||
"hotpath",
|
||||
"http 1.5.0",
|
||||
"http 1.4.2",
|
||||
"http-body 1.1.0",
|
||||
"http-body-util",
|
||||
"hyper",
|
||||
@@ -8981,7 +8987,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-audit"
|
||||
version = "1.0.0-beta.12"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"chrono",
|
||||
@@ -9003,12 +9009,12 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-checksums"
|
||||
version = "1.0.0-beta.12"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"base64-simd",
|
||||
"bytes",
|
||||
"crc-fast",
|
||||
"http 1.5.0",
|
||||
"http 1.4.2",
|
||||
"md-5 0.11.0",
|
||||
"pretty_assertions",
|
||||
"sha1 0.11.0",
|
||||
@@ -9018,7 +9024,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-common"
|
||||
version = "1.0.0-beta.12"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"chrono",
|
||||
"metrics",
|
||||
@@ -9033,7 +9039,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-concurrency"
|
||||
version = "1.0.0-beta.12"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"insta",
|
||||
"rustfs-io-core",
|
||||
@@ -9045,7 +9051,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-config"
|
||||
version = "1.0.0-beta.12"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"const-str",
|
||||
"serde",
|
||||
@@ -9054,7 +9060,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-credentials"
|
||||
version = "1.0.0-beta.12"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"base64-simd",
|
||||
"hmac 0.13.0",
|
||||
@@ -9067,7 +9073,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-crypto"
|
||||
version = "1.0.0-beta.12"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"aes-gcm",
|
||||
"argon2",
|
||||
@@ -9087,9 +9093,10 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-data-usage"
|
||||
version = "1.0.0-beta.12"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"path-clean",
|
||||
"rmp-serde",
|
||||
"rustfs-filemeta",
|
||||
"serde",
|
||||
@@ -9097,7 +9104,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-ecstore"
|
||||
version = "1.0.0-beta.12"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"aes-gcm",
|
||||
"arc-swap",
|
||||
@@ -9128,9 +9135,8 @@ dependencies = [
|
||||
"google-cloud-storage",
|
||||
"hex-simd",
|
||||
"hmac 0.13.0",
|
||||
"hostname",
|
||||
"hotpath",
|
||||
"http 1.5.0",
|
||||
"http 1.4.2",
|
||||
"http-body 1.1.0",
|
||||
"http-body-util",
|
||||
"hyper",
|
||||
@@ -9206,14 +9212,11 @@ dependencies = [
|
||||
"tonic",
|
||||
"tower",
|
||||
"tracing",
|
||||
"tracing-core",
|
||||
"tracing-opentelemetry",
|
||||
"tracing-subscriber",
|
||||
"url",
|
||||
"urlencoding",
|
||||
"uuid",
|
||||
"winapi-util",
|
||||
"windows-sys 0.61.2",
|
||||
"xxhash-rust",
|
||||
]
|
||||
|
||||
@@ -9236,7 +9239,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-extension-schema"
|
||||
version = "1.0.0-beta.12"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"serde",
|
||||
"serde_json",
|
||||
@@ -9245,7 +9248,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-filemeta"
|
||||
version = "1.0.0-beta.12"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"byteorder",
|
||||
@@ -9271,12 +9274,12 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-heal"
|
||||
version = "1.0.0-beta.12"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"base64 0.23.0",
|
||||
"futures",
|
||||
"http 1.5.0",
|
||||
"http 1.4.2",
|
||||
"metrics",
|
||||
"rustfs-common",
|
||||
"rustfs-concurrency",
|
||||
@@ -9301,13 +9304,13 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-iam"
|
||||
version = "1.0.0-beta.12"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-trait",
|
||||
"base64-simd",
|
||||
"futures",
|
||||
"http 1.5.0",
|
||||
"http 1.4.2",
|
||||
"jsonwebtoken 11.0.0",
|
||||
"moka",
|
||||
"openidconnect",
|
||||
@@ -9338,7 +9341,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-io-core"
|
||||
version = "1.0.0-beta.12"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"memmap2",
|
||||
@@ -9350,7 +9353,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-io-metrics"
|
||||
version = "1.0.0-beta.12"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"criterion",
|
||||
"metrics",
|
||||
@@ -9413,11 +9416,11 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-keystone"
|
||||
version = "1.0.0-beta.12"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"futures",
|
||||
"http 1.5.0",
|
||||
"http 1.4.2",
|
||||
"http-body 1.1.0",
|
||||
"http-body-util",
|
||||
"hyper",
|
||||
@@ -9439,7 +9442,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-kms"
|
||||
version = "1.0.0-beta.12"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"aes-gcm",
|
||||
"arc-swap",
|
||||
@@ -9473,7 +9476,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-lifecycle"
|
||||
version = "1.0.0-beta.12"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"metrics",
|
||||
@@ -9495,7 +9498,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-lock"
|
||||
version = "1.0.0-beta.12"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"crossbeam-queue",
|
||||
@@ -9517,7 +9520,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-log-analyzer"
|
||||
version = "1.0.0-beta.12"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"chrono",
|
||||
"flate2",
|
||||
@@ -9535,7 +9538,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-madmin"
|
||||
version = "1.0.0-beta.12"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"chrono",
|
||||
"humantime",
|
||||
@@ -9549,7 +9552,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-notify"
|
||||
version = "1.0.0-beta.12"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-trait",
|
||||
@@ -9583,7 +9586,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-object-capacity"
|
||||
version = "1.0.0-beta.12"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"criterion",
|
||||
"futures",
|
||||
@@ -9601,7 +9604,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-object-data-cache"
|
||||
version = "1.0.0-beta.12"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"criterion",
|
||||
@@ -9617,7 +9620,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-obs"
|
||||
version = "1.0.0-beta.12"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"chrono",
|
||||
"crossbeam-channel",
|
||||
@@ -9669,7 +9672,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-policy"
|
||||
version = "1.0.0-beta.12"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"base64-simd",
|
||||
@@ -9698,7 +9701,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-protocols"
|
||||
version = "1.0.0-beta.12"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"astral-tokio-tar",
|
||||
"async-compression",
|
||||
@@ -9711,7 +9714,7 @@ dependencies = [
|
||||
"futures-util",
|
||||
"hex",
|
||||
"hmac 0.13.0",
|
||||
"http 1.5.0",
|
||||
"http 1.4.2",
|
||||
"http-body-util",
|
||||
"hyper",
|
||||
"hyper-util",
|
||||
@@ -9760,7 +9763,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-protos"
|
||||
version = "1.0.0-beta.12"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"flatbuffers",
|
||||
"prost 0.14.4",
|
||||
@@ -9783,7 +9786,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-replication"
|
||||
version = "1.0.0-beta.12"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"byteorder",
|
||||
"bytes",
|
||||
@@ -9800,7 +9803,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-rio"
|
||||
version = "1.0.0-beta.12"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"aes-gcm",
|
||||
"arc-swap",
|
||||
@@ -9812,7 +9815,7 @@ dependencies = [
|
||||
"futures",
|
||||
"hex-simd",
|
||||
"hotpath",
|
||||
"http 1.5.0",
|
||||
"http 1.4.2",
|
||||
"http-body-util",
|
||||
"md-5 0.11.0",
|
||||
"pin-project-lite",
|
||||
@@ -9838,7 +9841,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-rio-v2"
|
||||
version = "1.0.0-beta.12"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"aes-gcm",
|
||||
"bytes",
|
||||
@@ -9860,14 +9863,14 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-s3-ops"
|
||||
version = "1.0.0-beta.12"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"rustfs-s3-types",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-s3-types"
|
||||
version = "1.0.0-beta.12"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"serde",
|
||||
"serde_json",
|
||||
@@ -9875,7 +9878,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-s3select-api"
|
||||
version = "1.0.0-beta.12"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"bytes",
|
||||
@@ -9883,7 +9886,7 @@ dependencies = [
|
||||
"datafusion",
|
||||
"futures",
|
||||
"futures-core",
|
||||
"http 1.5.0",
|
||||
"http 1.4.2",
|
||||
"metrics",
|
||||
"parking_lot",
|
||||
"rustfs-common",
|
||||
@@ -9904,7 +9907,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-s3select-query"
|
||||
version = "1.0.0-beta.12"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"async-recursion",
|
||||
"async-trait",
|
||||
@@ -9920,7 +9923,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-scanner"
|
||||
version = "1.0.0-beta.12"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"bytes",
|
||||
@@ -9928,7 +9931,7 @@ dependencies = [
|
||||
"futures",
|
||||
"hex-simd",
|
||||
"hmac 0.13.0",
|
||||
"http 1.5.0",
|
||||
"http 1.4.2",
|
||||
"metrics",
|
||||
"rand 0.10.2",
|
||||
"rmp-serde",
|
||||
@@ -9958,18 +9961,18 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-security-governance"
|
||||
version = "1.0.0-beta.12"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"thiserror 2.0.19",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-signer"
|
||||
version = "1.0.0-beta.12"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"base64-simd",
|
||||
"bytes",
|
||||
"http 1.5.0",
|
||||
"http 1.4.2",
|
||||
"hyper",
|
||||
"rustfs-utils",
|
||||
"s3s",
|
||||
@@ -9982,7 +9985,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-storage-api"
|
||||
version = "1.0.0-beta.12"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"insta",
|
||||
@@ -9996,7 +9999,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-targets"
|
||||
version = "1.0.0-beta.12"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-nats",
|
||||
@@ -10049,7 +10052,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-test-utils"
|
||||
version = "1.0.0-beta.12"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"rustfs-data-usage",
|
||||
"rustfs-ecstore",
|
||||
@@ -10064,7 +10067,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-tls-runtime"
|
||||
version = "1.0.0-beta.12"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"metrics",
|
||||
@@ -10084,11 +10087,11 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-trusted-proxies"
|
||||
version = "1.0.0-beta.12"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"axum",
|
||||
"http 1.5.0",
|
||||
"http 1.4.2",
|
||||
"ipnetwork",
|
||||
"metrics",
|
||||
"moka",
|
||||
@@ -10120,7 +10123,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-utils"
|
||||
version = "1.0.0-beta.12"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"base64-simd",
|
||||
"blake2",
|
||||
@@ -10134,7 +10137,7 @@ dependencies = [
|
||||
"hex-simd",
|
||||
"highway",
|
||||
"hmac 0.13.0",
|
||||
"http 1.5.0",
|
||||
"http 1.4.2",
|
||||
"hyper",
|
||||
"local-ip-address",
|
||||
"lz4",
|
||||
@@ -10161,7 +10164,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-zip"
|
||||
version = "1.0.0-beta.12"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"astral-tokio-tar",
|
||||
"async-compression",
|
||||
@@ -10191,7 +10194,7 @@ dependencies = [
|
||||
"anyhow",
|
||||
"async-trait",
|
||||
"bytes",
|
||||
"http 1.5.0",
|
||||
"http 1.4.2",
|
||||
"reqwest",
|
||||
"rustify_derive",
|
||||
"serde",
|
||||
@@ -10231,9 +10234,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustls"
|
||||
version = "0.23.43"
|
||||
version = "0.23.42"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0283386ce02abc0151e1761d08802dfe86c173b0b494af5cbc086574e453da06"
|
||||
checksum = "3c54fcab019b409d04215d3a17cb438fd7fbf192ee61461f20f4fe18704bc138"
|
||||
dependencies = [
|
||||
"aws-lc-rs",
|
||||
"log",
|
||||
@@ -10371,7 +10374,7 @@ dependencies = [
|
||||
"futures",
|
||||
"hex-simd",
|
||||
"hmac 0.13.0",
|
||||
"http 1.5.0",
|
||||
"http 1.4.2",
|
||||
"http-body 1.1.0",
|
||||
"http-body-util",
|
||||
"httparse",
|
||||
@@ -11499,7 +11502,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
|
||||
dependencies = [
|
||||
"fastrand",
|
||||
"getrandom 0.3.4",
|
||||
"getrandom 0.4.3",
|
||||
"once_cell",
|
||||
"rustix",
|
||||
"windows-sys 0.61.2",
|
||||
@@ -11735,13 +11738,13 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "tokio-macros"
|
||||
version = "2.7.2"
|
||||
version = "2.7.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e"
|
||||
checksum = "6328af13490e73a9b4694030fafd93f8c8c6a9dede33e821c3fc63eddf8042ba"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.3",
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -11845,7 +11848,7 @@ dependencies = [
|
||||
"bytes",
|
||||
"futures-core",
|
||||
"futures-sink",
|
||||
"http 1.5.0",
|
||||
"http 1.4.2",
|
||||
"httparse",
|
||||
"rand 0.8.7",
|
||||
"rustls-pki-types",
|
||||
@@ -11897,7 +11900,7 @@ dependencies = [
|
||||
"bytes",
|
||||
"flate2",
|
||||
"h2",
|
||||
"http 1.5.0",
|
||||
"http 1.4.2",
|
||||
"http-body 1.1.0",
|
||||
"http-body-util",
|
||||
"hyper",
|
||||
@@ -11984,7 +11987,7 @@ dependencies = [
|
||||
"bitflags 2.13.1",
|
||||
"bytes",
|
||||
"futures-util",
|
||||
"http 1.5.0",
|
||||
"http 1.4.2",
|
||||
"http-body 1.1.0",
|
||||
"pin-project-lite",
|
||||
"tower",
|
||||
@@ -12004,7 +12007,7 @@ dependencies = [
|
||||
"bytes",
|
||||
"futures-core",
|
||||
"futures-util",
|
||||
"http 1.5.0",
|
||||
"http 1.4.2",
|
||||
"http-body 1.1.0",
|
||||
"http-body-util",
|
||||
"percent-encoding",
|
||||
@@ -12177,7 +12180,7 @@ checksum = "6c01152af293afb9c7c2a57e4b559c5620b421f6d133261c60dd2d0cdb38e6b8"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"data-encoding",
|
||||
"http 1.5.0",
|
||||
"http 1.4.2",
|
||||
"httparse",
|
||||
"log",
|
||||
"rand 0.9.5",
|
||||
@@ -12364,7 +12367,7 @@ checksum = "30ffcc0e81025065dda612ec1e26a3d81bb16ef3062354873d17a35965d68522"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"derive_builder",
|
||||
"http 1.5.0",
|
||||
"http 1.4.2",
|
||||
"reqwest",
|
||||
"rustify",
|
||||
"rustify_derive",
|
||||
|
||||
+51
-55
@@ -69,7 +69,7 @@ edition = "2024"
|
||||
license = "Apache-2.0"
|
||||
repository = "https://github.com/rustfs/rustfs"
|
||||
rust-version = "1.97.1"
|
||||
version = "1.0.0-beta.12"
|
||||
version = "1.0.0-beta.11"
|
||||
homepage = "https://rustfs.com"
|
||||
description = "RustFS is a high-performance distributed object storage software built using Rust, one of the most popular languages worldwide. "
|
||||
keywords = ["RustFS", "Minio", "object-storage", "filesystem", "s3"]
|
||||
@@ -86,58 +86,58 @@ redundant_clone = "warn"
|
||||
|
||||
[workspace.dependencies]
|
||||
# RustFS Internal Crates
|
||||
rustfs = { path = "./rustfs", version = "1.0.0-beta.12" }
|
||||
rustfs-heal = { path = "crates/heal", version = "1.0.0-beta.12" }
|
||||
rustfs-audit = { path = "crates/audit", version = "1.0.0-beta.12" }
|
||||
rustfs-checksums = { path = "crates/checksums", version = "1.0.0-beta.12" }
|
||||
rustfs-common = { path = "crates/common", version = "1.0.0-beta.12" }
|
||||
rustfs-data-usage = { path = "crates/data-usage", version = "1.0.0-beta.12" }
|
||||
rustfs-config = { path = "./crates/config", version = "1.0.0-beta.12" }
|
||||
rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.0-beta.12" }
|
||||
rustfs-credentials = { path = "crates/credentials", version = "1.0.0-beta.12" }
|
||||
rustfs-crypto = { path = "crates/crypto", version = "1.0.0-beta.12" }
|
||||
rustfs-ecstore = { path = "crates/ecstore", version = "1.0.0-beta.12" }
|
||||
rustfs-filemeta = { path = "crates/filemeta", version = "1.0.0-beta.12" }
|
||||
rustfs-iam = { path = "crates/iam", version = "1.0.0-beta.12" }
|
||||
rustfs-keystone = { path = "crates/keystone", version = "1.0.0-beta.12" }
|
||||
rustfs-lifecycle = { path = "crates/lifecycle", version = "1.0.0-beta.12" }
|
||||
rustfs-kms = { path = "crates/kms", version = "1.0.0-beta.12" }
|
||||
rustfs-lock = { path = "crates/lock", version = "1.0.0-beta.12" }
|
||||
rustfs-madmin = { path = "crates/madmin", version = "1.0.0-beta.12" }
|
||||
rustfs-notify = { path = "crates/notify", version = "1.0.0-beta.12" }
|
||||
rustfs-io-metrics = { path = "crates/io-metrics", version = "1.0.0-beta.12" }
|
||||
rustfs-io-core = { path = "crates/io-core", version = "1.0.0-beta.12" }
|
||||
rustfs-object-capacity = { path = "crates/object-capacity", version = "1.0.0-beta.12" }
|
||||
rustfs-object-data-cache = { path = "crates/object-data-cache", version = "1.0.0-beta.12" }
|
||||
rustfs-log-analyzer = { path = "crates/log-analyzer", version = "1.0.0-beta.12" }
|
||||
rustfs-obs = { path = "crates/obs", version = "1.0.0-beta.12" }
|
||||
rustfs-policy = { path = "crates/policy", version = "1.0.0-beta.12" }
|
||||
rustfs-protos = { path = "crates/protos", version = "1.0.0-beta.12" }
|
||||
rustfs-protocols = { path = "crates/protocols", version = "1.0.0-beta.12" }
|
||||
rustfs-replication = { path = "crates/replication", version = "1.0.0-beta.12" }
|
||||
rustfs-rio = { path = "crates/rio", version = "1.0.0-beta.12" }
|
||||
rustfs-rio-v2 = { path = "crates/rio-v2", version = "1.0.0-beta.12" }
|
||||
rustfs-s3-types = { path = "crates/s3-types", version = "1.0.0-beta.12" }
|
||||
rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.0-beta.12" }
|
||||
rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.0-beta.12" }
|
||||
rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.0-beta.12" }
|
||||
rustfs-scanner = { path = "crates/scanner", version = "1.0.0-beta.12" }
|
||||
rustfs-security-governance = { path = "crates/security-governance", version = "1.0.0-beta.12" }
|
||||
rustfs-extension-schema = { path = "crates/extension-schema", version = "1.0.0-beta.12" }
|
||||
rustfs-signer = { path = "crates/signer", version = "1.0.0-beta.12" }
|
||||
rustfs-storage-api = { path = "crates/storage-api", version = "1.0.0-beta.12" }
|
||||
rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.0-beta.12" }
|
||||
rustfs-targets = { path = "crates/targets", version = "1.0.0-beta.12" }
|
||||
rustfs-test-utils = { path = "crates/test-utils", version = "1.0.0-beta.12" }
|
||||
rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.0-beta.12" }
|
||||
rustfs-utils = { path = "crates/utils", version = "1.0.0-beta.12" }
|
||||
rustfs-zip = { path = "./crates/zip", version = "1.0.0-beta.12" }
|
||||
rustfs = { path = "./rustfs", version = "1.0.0-beta.11" }
|
||||
rustfs-heal = { path = "crates/heal", version = "1.0.0-beta.11" }
|
||||
rustfs-audit = { path = "crates/audit", version = "1.0.0-beta.11" }
|
||||
rustfs-checksums = { path = "crates/checksums", version = "1.0.0-beta.11" }
|
||||
rustfs-common = { path = "crates/common", version = "1.0.0-beta.11" }
|
||||
rustfs-data-usage = { path = "crates/data-usage", version = "1.0.0-beta.11" }
|
||||
rustfs-config = { path = "./crates/config", version = "1.0.0-beta.11" }
|
||||
rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.0-beta.11" }
|
||||
rustfs-credentials = { path = "crates/credentials", version = "1.0.0-beta.11" }
|
||||
rustfs-crypto = { path = "crates/crypto", version = "1.0.0-beta.11" }
|
||||
rustfs-ecstore = { path = "crates/ecstore", version = "1.0.0-beta.11" }
|
||||
rustfs-filemeta = { path = "crates/filemeta", version = "1.0.0-beta.11" }
|
||||
rustfs-iam = { path = "crates/iam", version = "1.0.0-beta.11" }
|
||||
rustfs-keystone = { path = "crates/keystone", version = "1.0.0-beta.11" }
|
||||
rustfs-lifecycle = { path = "crates/lifecycle", version = "1.0.0-beta.11" }
|
||||
rustfs-kms = { path = "crates/kms", version = "1.0.0-beta.11" }
|
||||
rustfs-lock = { path = "crates/lock", version = "1.0.0-beta.11" }
|
||||
rustfs-madmin = { path = "crates/madmin", version = "1.0.0-beta.11" }
|
||||
rustfs-notify = { path = "crates/notify", version = "1.0.0-beta.11" }
|
||||
rustfs-io-metrics = { path = "crates/io-metrics", version = "1.0.0-beta.11" }
|
||||
rustfs-io-core = { path = "crates/io-core", version = "1.0.0-beta.11" }
|
||||
rustfs-object-capacity = { path = "crates/object-capacity", version = "1.0.0-beta.11" }
|
||||
rustfs-object-data-cache = { path = "crates/object-data-cache", version = "1.0.0-beta.11" }
|
||||
rustfs-log-analyzer = { path = "crates/log-analyzer", version = "1.0.0-beta.11" }
|
||||
rustfs-obs = { path = "crates/obs", version = "1.0.0-beta.11" }
|
||||
rustfs-policy = { path = "crates/policy", version = "1.0.0-beta.11" }
|
||||
rustfs-protos = { path = "crates/protos", version = "1.0.0-beta.11" }
|
||||
rustfs-protocols = { path = "crates/protocols", version = "1.0.0-beta.11" }
|
||||
rustfs-replication = { path = "crates/replication", version = "1.0.0-beta.11" }
|
||||
rustfs-rio = { path = "crates/rio", version = "1.0.0-beta.11" }
|
||||
rustfs-rio-v2 = { path = "crates/rio-v2", version = "1.0.0-beta.11" }
|
||||
rustfs-s3-types = { path = "crates/s3-types", version = "1.0.0-beta.11" }
|
||||
rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.0-beta.11" }
|
||||
rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.0-beta.11" }
|
||||
rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.0-beta.11" }
|
||||
rustfs-scanner = { path = "crates/scanner", version = "1.0.0-beta.11" }
|
||||
rustfs-security-governance = { path = "crates/security-governance", version = "1.0.0-beta.11" }
|
||||
rustfs-extension-schema = { path = "crates/extension-schema", version = "1.0.0-beta.11" }
|
||||
rustfs-signer = { path = "crates/signer", version = "1.0.0-beta.11" }
|
||||
rustfs-storage-api = { path = "crates/storage-api", version = "1.0.0-beta.11" }
|
||||
rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.0-beta.11" }
|
||||
rustfs-targets = { path = "crates/targets", version = "1.0.0-beta.11" }
|
||||
rustfs-test-utils = { path = "crates/test-utils", version = "1.0.0-beta.11" }
|
||||
rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.0-beta.11" }
|
||||
rustfs-utils = { path = "crates/utils", version = "1.0.0-beta.11" }
|
||||
rustfs-zip = { path = "./crates/zip", version = "1.0.0-beta.11" }
|
||||
|
||||
# Async Runtime and Networking
|
||||
async-channel = "2.5.0"
|
||||
async_zip = { default-features = false, version = "0.0.18" }
|
||||
mysql_async = { default-features = false, version = "0.37" }
|
||||
async-compression = { version = "0.4.43" }
|
||||
async-compression = { version = "0.4.42" }
|
||||
async-recursion = "1.1.1"
|
||||
async-trait = "0.1.91"
|
||||
async-nats = { version = "0.50.0", default-features = false }
|
||||
@@ -152,7 +152,7 @@ lapin = { default-features = false, version = "4.10.0" }
|
||||
hyper = { version = "1.11.0" }
|
||||
hyper-rustls = { default-features = false, version = "0.27.9" }
|
||||
hyper-util = { version = "0.1.20" }
|
||||
http = "1.5.0"
|
||||
http = "1.4.2"
|
||||
http-body = "1.1.0"
|
||||
http-body-util = "0.1.4"
|
||||
minlz = "1.2.3"
|
||||
@@ -200,7 +200,7 @@ jsonwebtoken = { version = "11.0.0" }
|
||||
openidconnect = { default-features = false, version = "4.0" }
|
||||
pbkdf2 = "0.13.0"
|
||||
rsa = { version = "=0.10.0-rc.18" }
|
||||
rustls = { default-features = false, version = "0.23.43" }
|
||||
rustls = { default-features = false, version = "0.23.42" }
|
||||
rustls-native-certs = "0.8"
|
||||
rustls-pki-types = "1.15.1"
|
||||
sha1 = "0.11.0"
|
||||
@@ -256,7 +256,6 @@ hashbrown = { version = "0.17.1" }
|
||||
hex = "0.4.3"
|
||||
hex-simd = "0.8.0"
|
||||
highway = { version = "1.3.0" }
|
||||
hostname = "0.4.2"
|
||||
ipnetwork = { version = "0.21.1" }
|
||||
lazy_static = "1.5.0"
|
||||
libc = "0.2.189"
|
||||
@@ -284,7 +283,7 @@ reed-solomon-erasure = { package = "rustfs-erasure-codec", version = "8.0.2" }
|
||||
reed-solomon-simd = "3.1.0"
|
||||
regex = { version = "1.13.1" }
|
||||
rumqttc = { package = "rumqttc-next", version = "0.33.3" }
|
||||
redis = { version = "1.5.0" }
|
||||
redis = { version = "1.4.1" }
|
||||
rustix = { version = "1.1.4" }
|
||||
rust-embed = { version = "8.12.0" }
|
||||
rustc-hash = { version = "2.1.3" }
|
||||
@@ -304,7 +303,6 @@ test-case = "3.3.1"
|
||||
thiserror = "2.0.19"
|
||||
tracing = { version = "0.1.44" }
|
||||
tracing-appender = "0.2.5"
|
||||
tracing-core = "0.1.36"
|
||||
tracing-error = "0.2.1"
|
||||
tracing-opentelemetry = { version = "0.33" }
|
||||
tracing-subscriber = { version = "0.3.23" }
|
||||
@@ -315,9 +313,7 @@ uuid = { version = "1.24.0" }
|
||||
vaultrs = { version = "0.8.0" }
|
||||
tar = "0.4.46"
|
||||
walkdir = "2.5.0"
|
||||
winapi-util = "0.1.11"
|
||||
windows = { version = "0.62.2" }
|
||||
windows-sys = "0.61.2"
|
||||
xxhash-rust = { version = "0.8.18" }
|
||||
zip = "8.6.0"
|
||||
zstd = "0.13.3"
|
||||
|
||||
@@ -116,7 +116,7 @@ chown -R 10001:10001 data logs
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
|
||||
|
||||
# Using specific version
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-beta.12
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-beta.11
|
||||
```
|
||||
|
||||
If you use [podman](https://github.com/containers/podman) instead of docker, you can install the RustFS with the below command
|
||||
|
||||
+1
-1
@@ -113,7 +113,7 @@ chown -R 10001:10001 data logs
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
|
||||
|
||||
# 使用指定版本运行
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-beta.12
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-beta.11
|
||||
```
|
||||
|
||||
如果您通过绑定挂载启用 TLS 证书目录,也请用同样方式准备该目录:
|
||||
|
||||
@@ -66,10 +66,6 @@ Current guidance:
|
||||
|
||||
- `RUSTFS_BROWSER_REDIRECT_URL` sets the externally reachable browser origin used for OIDC callback, console success redirect, and logout fallback URLs. Configure it to the public scheme and authority without a path, for example `https://console.example.com`. In load-balancer deployments, keep OIDC authorize and callback requests on the same backend node because the in-flight OIDC `state` is local to the RustFS node.
|
||||
|
||||
## Distributed endpoint locality
|
||||
|
||||
- `RUSTFS_LOCAL_ENDPOINT_HOST` identifies this server's host in a distributed `RUSTFS_VOLUMES` topology without resolving every peer during startup. Set it to exactly one host, without a scheme, port, or path. It is accepted only for orchestrated URL topologies and must match at least one endpoint on the RustFS server port; invalid or unmatched values fail startup. Leave it unset to retain DNS-based locality discovery.
|
||||
|
||||
## Scanner environment aliases
|
||||
|
||||
- `RUSTFS_SCANNER_SPEED` (canonical, also accepts `MINIO_SCANNER_SPEED`)
|
||||
|
||||
@@ -131,10 +131,6 @@ pub const ENV_RUSTFS_ADDRESS: &str = "RUSTFS_ADDRESS";
|
||||
/// Environment variable for server volumes.
|
||||
pub const ENV_RUSTFS_VOLUMES: &str = "RUSTFS_VOLUMES";
|
||||
|
||||
/// Environment variable identifying this server's host in distributed endpoint
|
||||
/// lists without relying on DNS locality discovery.
|
||||
pub const ENV_LOCAL_ENDPOINT_HOST: &str = "RUSTFS_LOCAL_ENDPOINT_HOST";
|
||||
|
||||
/// Environment variable to explicitly bypass local physical disk independence checks.
|
||||
pub const ENV_UNSAFE_BYPASS_DISK_CHECK: &str = "RUSTFS_UNSAFE_BYPASS_DISK_CHECK";
|
||||
|
||||
|
||||
@@ -158,33 +158,17 @@ pub const DEFAULT_INTERNODE_RPC_BODY_DIGEST_STRICT: bool = false;
|
||||
// rolling upgrades until the fleet-wide body-digest fallback counter reads zero.
|
||||
const _: () = assert!(!DEFAULT_INTERNODE_RPC_BODY_DIGEST_STRICT);
|
||||
|
||||
/// Require the replay-scoped internode RPC signature after the fleet has converged on it.
|
||||
///
|
||||
/// The default keeps v1/v2 peers available during a rolling upgrade. Operators may set this only
|
||||
/// after `rustfs_system_network_internode_replay_scope_fallback_total` remains zero for a full
|
||||
/// release window. The node still accepts a v2-authenticated `Ping` carrying an epoch challenge:
|
||||
/// that narrowly scoped bootstrap lets an upgraded client learn the receiving process epoch and
|
||||
/// immediately retry with the replay-scoped signature after a peer restart.
|
||||
pub const ENV_INTERNODE_RPC_REPLAY_SCOPE_STRICT: &str = "RUSTFS_INTERNODE_RPC_REPLAY_SCOPE_STRICT";
|
||||
pub const DEFAULT_INTERNODE_RPC_REPLAY_SCOPE_STRICT: bool = false;
|
||||
|
||||
// Compile-time invariant: mixed-version clusters must remain available until operators make the
|
||||
// observed fallback counter an explicit strictness decision.
|
||||
const _: () = assert!(!DEFAULT_INTERNODE_RPC_REPLAY_SCOPE_STRICT);
|
||||
|
||||
/// Capacity (distinct nonces) of the process-local internode RPC replay cache that enforces
|
||||
/// one-time consumption of authenticated RPC signatures.
|
||||
/// one-time consumption of body-bound v2 signatures.
|
||||
///
|
||||
/// The cache retains each nonce for the ~10-minute signature freshness envelope. Once peers use
|
||||
/// replay-scoped v3 authentication, every authenticated RPC consumes one entry, so the steady
|
||||
/// state holds roughly `authenticated RPC RPS x 601s` entries. The default sustains about 1,700
|
||||
/// authenticated RPCs per second (about 120 MiB worst case, allocated only under sustained load);
|
||||
/// operators must size it for the node's aggregate peak RPC rate before enabling strict replay
|
||||
/// scope. Overflow fails closed — legitimate signed traffic is the only thing that can fill the
|
||||
/// cache (replays are rejected before insertion, and an attacker cannot mint valid nonces without
|
||||
/// the shared secret) — and increments
|
||||
/// The cache retains each nonce for the ~10-minute signature freshness envelope, so the steady
|
||||
/// state holds roughly `mutating RPS x 601s` entries; the default sustains ~1,700 body-bound
|
||||
/// mutating RPCs per second (about 120 MiB worst case, allocated only under sustained load).
|
||||
/// Overflow fails closed — legitimate signed traffic is the only thing that can fill the cache
|
||||
/// (replays are rejected before insertion, and an attacker cannot mint valid nonces without the
|
||||
/// shared secret) — and increments
|
||||
/// `rustfs_system_network_internode_replay_cache_overflow_total`, so a sustained non-zero overflow
|
||||
/// counter means this capacity is undersized for the node's peak authenticated RPC rate.
|
||||
/// counter means this capacity is undersized for the node's peak mutation rate.
|
||||
pub const ENV_INTERNODE_RPC_REPLAY_CACHE_CAPACITY: &str = "RUSTFS_INTERNODE_RPC_REPLAY_CACHE_CAPACITY";
|
||||
pub const DEFAULT_INTERNODE_RPC_REPLAY_CACHE_CAPACITY: usize = 1_048_576;
|
||||
|
||||
@@ -370,12 +354,6 @@ mod tests {
|
||||
assert_eq!(ENV_INTERNODE_RPC_BODY_DIGEST_STRICT, "RUSTFS_INTERNODE_RPC_BODY_DIGEST_STRICT");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn internode_replay_scope_strict_env_name_is_stable() {
|
||||
// The fail-open default invariant is asserted at compile time next to the definition.
|
||||
assert_eq!(ENV_INTERNODE_RPC_REPLAY_SCOPE_STRICT, "RUSTFS_INTERNODE_RPC_REPLAY_SCOPE_STRICT");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn internode_replay_cache_capacity_defaults_and_env_name() {
|
||||
assert_eq!(ENV_INTERNODE_RPC_REPLAY_CACHE_CAPACITY, "RUSTFS_INTERNODE_RPC_REPLAY_CACHE_CAPACITY");
|
||||
|
||||
@@ -29,6 +29,7 @@ workspace = true
|
||||
|
||||
[dependencies]
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
path-clean = { workspace = true }
|
||||
rmp-serde = { workspace = true }
|
||||
async-trait = { workspace = true }
|
||||
rustfs-filemeta = { workspace = true }
|
||||
|
||||
@@ -12,10 +12,12 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use path_clean::PathClean;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::{
|
||||
collections::{HashMap, HashSet},
|
||||
hash::{DefaultHasher, Hash, Hasher},
|
||||
path::Path,
|
||||
time::{Duration, SystemTime},
|
||||
};
|
||||
|
||||
@@ -332,7 +334,7 @@ impl<'de> Deserialize<'de> for SizeHistogram {
|
||||
impl SizeHistogram {
|
||||
pub fn add(&mut self, size: u64) {
|
||||
let intervals = [
|
||||
(0, 1024 - 1), // LESS_THAN_1024_B
|
||||
(0, 1024), // LESS_THAN_1024_B
|
||||
(1024, 64 * 1024 - 1), // BETWEEN_1024_B_AND_64_KB
|
||||
(64 * 1024, 256 * 1024 - 1), // BETWEEN_64_KB_AND_256_KB
|
||||
(256 * 1024, 512 * 1024 - 1), // BETWEEN_256_KB_AND_512_KB
|
||||
@@ -360,7 +362,7 @@ impl SizeHistogram {
|
||||
// the sub-ranges in [1 KiB, 512 KiB).
|
||||
const ONE_MIB: u64 = 1024 * 1024;
|
||||
let intervals = [
|
||||
(0, 1024 - 1), // LESS_THAN_1024_B
|
||||
(0, 1024), // LESS_THAN_1024_B
|
||||
(1024, 64 * 1024 - 1), // BETWEEN_1024_B_AND_64_KB
|
||||
(64 * 1024, 256 * 1024 - 1), // BETWEEN_64_KB_AND_256_KB
|
||||
(256 * 1024, 512 * 1024 - 1), // BETWEEN_256_KB_AND_512_KB
|
||||
@@ -1108,39 +1110,9 @@ fn mark(duc: &DataUsageCache, entry: &DataUsageEntry, found: &mut HashSet<String
|
||||
}
|
||||
}
|
||||
|
||||
fn clean_data_usage_path(data: &str) -> String {
|
||||
let rooted = data.starts_with('/');
|
||||
let mut parts = Vec::new();
|
||||
|
||||
for part in data.split('/') {
|
||||
match part {
|
||||
"" | "." => {}
|
||||
".." => {
|
||||
if parts.last().is_some_and(|last| *last != "..") {
|
||||
parts.pop();
|
||||
} else if !rooted {
|
||||
parts.push(part);
|
||||
}
|
||||
}
|
||||
_ => parts.push(part),
|
||||
}
|
||||
}
|
||||
|
||||
let clean = parts.join("/");
|
||||
match (rooted, clean.is_empty()) {
|
||||
(true, true) => "/".to_string(),
|
||||
(true, false) => format!("/{clean}"),
|
||||
(false, true) => ".".to_string(),
|
||||
(false, false) => clean,
|
||||
}
|
||||
}
|
||||
|
||||
/// Hash a slash-separated path for data usage caching.
|
||||
///
|
||||
/// Cache identifiers are persisted and exchanged across nodes, so their
|
||||
/// normalization must not depend on the host operating system.
|
||||
/// Hash a path for data usage caching
|
||||
pub fn hash_path(data: &str) -> DataUsageHash {
|
||||
DataUsageHash(clean_data_usage_path(data))
|
||||
DataUsageHash(Path::new(&data).clean().to_string_lossy().to_string())
|
||||
}
|
||||
|
||||
impl DataUsageInfo {
|
||||
@@ -1525,23 +1497,6 @@ mod tests {
|
||||
buckets_count: u64,
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hash_path_uses_portable_slash_semantics() {
|
||||
for (input, expected) in [
|
||||
("", "."),
|
||||
(".", "."),
|
||||
("/", "/"),
|
||||
("//bucket///prefix/", "/bucket/prefix"),
|
||||
("bucket/./prefix//object", "bucket/prefix/object"),
|
||||
("bucket/a/../b", "bucket/b"),
|
||||
("../bucket/..", ".."),
|
||||
("/../../bucket", "/bucket"),
|
||||
("bucket\\prefix/object", "bucket\\prefix/object"),
|
||||
] {
|
||||
assert_eq!(hash_path(input).key(), expected, "unexpected portable cache key for {input:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn completeness_marker_is_additive_for_legacy_named_readers() {
|
||||
let current = DataUsageInfo {
|
||||
@@ -1646,49 +1601,6 @@ mod tests {
|
||||
assert_eq!(map["BETWEEN_512_KB_AND_1_MB"], 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_size_histogram_classifies_adjacent_boundaries_once() {
|
||||
let cases = [
|
||||
(1023, 0),
|
||||
(1024, 1),
|
||||
(64 * 1024 - 1, 1),
|
||||
(64 * 1024, 2),
|
||||
(256 * 1024 - 1, 2),
|
||||
(256 * 1024, 3),
|
||||
(512 * 1024 - 1, 3),
|
||||
(512 * 1024, 4),
|
||||
(1024 * 1024 - 1, 4),
|
||||
(1024 * 1024, 6),
|
||||
(10 * 1024 * 1024 - 1, 6),
|
||||
(10 * 1024 * 1024, 7),
|
||||
(64 * 1024 * 1024 - 1, 7),
|
||||
(64 * 1024 * 1024, 8),
|
||||
(128 * 1024 * 1024 - 1, 8),
|
||||
(128 * 1024 * 1024, 9),
|
||||
(512 * 1024 * 1024 - 1, 9),
|
||||
(512 * 1024 * 1024, 10),
|
||||
];
|
||||
|
||||
for (size, expected_bucket) in cases {
|
||||
let mut hist = SizeHistogram::default();
|
||||
hist.add(size);
|
||||
|
||||
assert_eq!(hist.0.iter().sum::<u64>(), 1, "size {size} must have exactly one physical bucket");
|
||||
assert_eq!(hist.0[expected_bucket], 1, "size {size} must select the expected bucket");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_size_histogram_1024_bytes_contributes_to_compat_rollup() {
|
||||
let mut hist = SizeHistogram::default();
|
||||
hist.add(1024);
|
||||
|
||||
let map = hist.to_map();
|
||||
assert_eq!(map["LESS_THAN_1024_B"], 0);
|
||||
assert_eq!(map["BETWEEN_1024_B_AND_64_KB"], 1);
|
||||
assert_eq!(map["BETWEEN_1024B_AND_1_MB"], 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_size_histogram_compat_rollup_saturates_on_corrupt_counts() {
|
||||
let mut hist = SizeHistogram::default();
|
||||
|
||||
@@ -13,9 +13,8 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Cross-process replay / tamper acceptance for internode NodeService RPC
|
||||
//! signatures (<https://github.com/rustfs/backlog/issues/1327>,
|
||||
//! <https://github.com/rustfs/backlog/issues/1542>).
|
||||
//! Cross-process replay / tamper acceptance for the internode NodeService v2 RPC
|
||||
//! signature (<https://github.com/rustfs/backlog/issues/1327>).
|
||||
//!
|
||||
//! # Why this exists on top of the in-process tests
|
||||
//!
|
||||
@@ -79,8 +78,6 @@
|
||||
//! | mixed version: legacy-only still served, not blocked | [`legacy_only_signature_is_accepted_in_default_posture`] |
|
||||
//! | strict flip closes the signature downgrade | [`signature_strict_rejects_legacy_only_downgrade`] |
|
||||
//! | strict flip closes the body-digest downgrade, incl. v1 | [`body_digest_strict_rejects_digestless_mutation`] |
|
||||
//! | replay scope binds every RPC and rejects restart replay | [`replay_scope_rejects_replay_path_transplant_and_stale_epoch_e2e`] |
|
||||
//! | strict replay scope allows only Ping bootstrap before v3 | [`replay_scope_strict_requires_v3_after_ping_bootstrap_e2e`] |
|
||||
//!
|
||||
//! Two acceptance items are deliberately left to the in-process tests. A stale
|
||||
//! timestamp cannot be forged from outside — it is inside the HMAC — so
|
||||
@@ -91,20 +88,18 @@
|
||||
|
||||
use crate::common::{RustFSTestEnvironment, init_logging};
|
||||
use crate::storage_api::internode_rpc_signature::{
|
||||
TONIC_RPC_PREFIX, gen_signature_headers, gen_tonic_replay_scope_headers, gen_tonic_signature_headers,
|
||||
node_service_time_out_client_no_auth, verify_tonic_boot_epoch_response,
|
||||
TONIC_RPC_PREFIX, gen_signature_headers, gen_tonic_signature_headers, node_service_time_out_client_no_auth,
|
||||
};
|
||||
use http::{HeaderMap, Method};
|
||||
use rustfs_config::{
|
||||
ENV_INTERNODE_RPC_BODY_DIGEST_STRICT, ENV_INTERNODE_RPC_REPLAY_CACHE_CAPACITY, ENV_INTERNODE_RPC_REPLAY_SCOPE_STRICT,
|
||||
ENV_INTERNODE_RPC_SIGNATURE_STRICT,
|
||||
ENV_INTERNODE_RPC_BODY_DIGEST_STRICT, ENV_INTERNODE_RPC_REPLAY_CACHE_CAPACITY, ENV_INTERNODE_RPC_SIGNATURE_STRICT,
|
||||
};
|
||||
use rustfs_protos::canonical_make_volume_request_body;
|
||||
use rustfs_protos::proto_gen::node_service::{MakeVolumeRequest, MakeVolumeResponse, PingRequest, PingResponse};
|
||||
use rustfs_protos::proto_gen::node_service::{MakeVolumeRequest, MakeVolumeResponse};
|
||||
use serial_test::serial;
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::error::Error;
|
||||
use tonic::{Code, Request, Response, Status};
|
||||
use tonic::{Code, Request, Status};
|
||||
use uuid::Uuid;
|
||||
|
||||
type TestResult = Result<(), Box<dyn Error + Send + Sync>>;
|
||||
@@ -120,14 +115,12 @@ const TEST_RPC_SECRET: &str = "rustfs-internode-signature-e2e-secret";
|
||||
/// clears authentication stops harmlessly at `find_disk`.
|
||||
const ABSENT_DISK: &str = "/nonexistent/rustfs-signature-e2e-disk";
|
||||
|
||||
/// Wire names of the v2 and replay-scope headers these black-box tests edit. They are
|
||||
/// `pub(crate)` in ecstore, so they are repeated here rather than imported.
|
||||
/// [`overwrite_header`] asserts the header it replaces was actually present, which turns a
|
||||
/// rename into a loud failure instead of silently reducing an attack to a no-op.
|
||||
/// Wire names of the two v2 headers these tests edit. They are `pub(crate)` in
|
||||
/// ecstore, so they are repeated here rather than imported — [`overwrite_header`]
|
||||
/// asserts the header it replaces was actually present, which turns a rename
|
||||
/// into a loud failure instead of silently reducing an attack to a no-op.
|
||||
const CONTENT_SHA256_HEADER: &str = "x-rustfs-content-sha256";
|
||||
const NONCE_HEADER: &str = "x-rustfs-rpc-nonce";
|
||||
const TIMESTAMP_HEADER: &str = "x-rustfs-timestamp";
|
||||
const BOOT_EPOCH_CHALLENGE_HEADER: &str = "x-rustfs-rpc-boot-epoch-challenge";
|
||||
|
||||
/// gRPC service name carried in the signed scope, i.e. `TONIC_RPC_PREFIX`
|
||||
/// without its leading `/`.
|
||||
@@ -162,28 +155,19 @@ fn align_rpc_secret_with_server() {
|
||||
///
|
||||
/// Uses the no-cleanup spawn so a `pkill` pattern cannot reap servers belonging
|
||||
/// to other tests running in the same binary.
|
||||
fn server_env(extra_env: &[(&'static str, &'static str)]) -> Vec<(&'static str, &'static str)> {
|
||||
async fn start_server(extra_env: &[(&str, &str)]) -> Result<RustFSTestEnvironment, Box<dyn Error + Send + Sync>> {
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
let mut child_env = vec![
|
||||
("RUSTFS_RPC_SECRET", TEST_RPC_SECRET),
|
||||
(ENV_INTERNODE_RPC_SIGNATURE_STRICT, "false"),
|
||||
(ENV_INTERNODE_RPC_BODY_DIGEST_STRICT, "false"),
|
||||
(ENV_INTERNODE_RPC_REPLAY_SCOPE_STRICT, "false"),
|
||||
(ENV_INTERNODE_RPC_REPLAY_CACHE_CAPACITY, "1048576"),
|
||||
];
|
||||
child_env.extend_from_slice(extra_env);
|
||||
child_env
|
||||
}
|
||||
|
||||
async fn start_server_with_env(child_env: &[(&str, &str)]) -> Result<RustFSTestEnvironment, Box<dyn Error + Send + Sync>> {
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server_without_cleanup_with_env(child_env).await?;
|
||||
env.start_rustfs_server_without_cleanup_with_env(&child_env).await?;
|
||||
Ok(env)
|
||||
}
|
||||
|
||||
async fn start_server(extra_env: &[(&'static str, &'static str)]) -> Result<RustFSTestEnvironment, Box<dyn Error + Send + Sync>> {
|
||||
start_server_with_env(&server_env(extra_env)).await
|
||||
}
|
||||
|
||||
/// Stop the child and drop the cached gRPC channel for its address.
|
||||
///
|
||||
/// `node_service_time_out_client_no_auth` memoises channels in a process-global
|
||||
@@ -254,83 +238,12 @@ fn overwrite_header(headers: &mut HeaderMap, name: &'static str, value: &str) {
|
||||
/// and nothing else — no interceptor adds or rewrites auth metadata, so the
|
||||
/// bytes on the wire are the ones the test chose.
|
||||
async fn call_make_volume(url: &str, request: MakeVolumeRequest, headers: HeaderMap) -> Result<MakeVolumeResponse, Status> {
|
||||
call_make_volume_response(url, request, headers)
|
||||
.await
|
||||
.map(Response::into_inner)
|
||||
}
|
||||
|
||||
async fn call_make_volume_response(
|
||||
url: &str,
|
||||
request: MakeVolumeRequest,
|
||||
headers: HeaderMap,
|
||||
) -> Result<Response<MakeVolumeResponse>, Status> {
|
||||
let mut client = node_service_time_out_client_no_auth(&url.to_string())
|
||||
.await
|
||||
.map_err(|err| Status::unavailable(format!("cannot reach the node service: {err}")))?;
|
||||
let mut rpc_request = Request::new(request);
|
||||
rpc_request.metadata_mut().as_mut().extend(headers);
|
||||
client.make_volume(rpc_request).await
|
||||
}
|
||||
|
||||
async fn call_ping_response(url: &str, headers: HeaderMap) -> Result<Response<PingResponse>, Status> {
|
||||
let mut client = node_service_time_out_client_no_auth(&url.to_string())
|
||||
.await
|
||||
.map_err(|err| Status::unavailable(format!("cannot reach the node service: {err}")))?;
|
||||
let mut rpc_request = Request::new(PingRequest {
|
||||
version: 1,
|
||||
body: bytes::Bytes::new(),
|
||||
});
|
||||
rpc_request.metadata_mut().as_mut().extend(headers);
|
||||
client.ping(rpc_request).await
|
||||
}
|
||||
|
||||
fn attach_boot_epoch_challenge(headers: &mut HeaderMap) -> Uuid {
|
||||
let challenge = Uuid::new_v4();
|
||||
headers.insert(
|
||||
BOOT_EPOCH_CHALLENGE_HEADER,
|
||||
challenge.to_string().parse().expect("UUID must be a valid header value"),
|
||||
);
|
||||
challenge
|
||||
}
|
||||
|
||||
fn mint_replay_scope_headers(audience: &str, path: &str, content_sha256: &str, boot_epoch: Uuid) -> HeaderMap {
|
||||
let mut headers = mint_v2_headers(audience, "MakeVolume", Some(content_sha256));
|
||||
let timestamp = headers
|
||||
.get(TIMESTAMP_HEADER)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.expect("v2 headers must carry a timestamp")
|
||||
.to_string();
|
||||
headers.extend(
|
||||
gen_tonic_replay_scope_headers(audience, path, ×tamp, content_sha256, boot_epoch)
|
||||
.expect("replay-scope headers must mint with the aligned RPC secret"),
|
||||
);
|
||||
headers
|
||||
}
|
||||
|
||||
async fn learn_boot_epoch_from_make_volume(url: &str, audience: &str) -> Uuid {
|
||||
let request = make_volume_request("signature-e2e-epoch-bootstrap");
|
||||
let mut headers = mint_v2_headers(audience, "MakeVolume", Some(&canonical_digest(&request)));
|
||||
let challenge = attach_boot_epoch_challenge(&mut headers);
|
||||
let response = call_make_volume_response(url, request, headers)
|
||||
.await
|
||||
.expect("v2 request with epoch challenge must clear default authentication");
|
||||
let boot_epoch = verify_tonic_boot_epoch_response(audience, challenge, response.metadata().as_ref())
|
||||
.expect("server must HMAC-authenticate the advertised boot epoch");
|
||||
assert_authenticated(
|
||||
Ok(response.into_inner()),
|
||||
"a v2 epoch-challenge request in the default replay-scope posture",
|
||||
);
|
||||
boot_epoch
|
||||
}
|
||||
|
||||
async fn learn_boot_epoch_from_ping(url: &str, audience: &str) -> Uuid {
|
||||
let mut headers = mint_v2_headers(audience, "Ping", None);
|
||||
let challenge = attach_boot_epoch_challenge(&mut headers);
|
||||
let response = call_ping_response(url, headers)
|
||||
.await
|
||||
.expect("v2 Ping with an epoch challenge must bootstrap strict replay scope");
|
||||
verify_tonic_boot_epoch_response(audience, challenge, response.metadata().as_ref())
|
||||
.expect("strict replay-scope Ping must return a valid boot epoch proof")
|
||||
client.make_volume(rpc_request).await.map(|response| response.into_inner())
|
||||
}
|
||||
|
||||
/// Assert a call cleared authentication.
|
||||
@@ -419,122 +332,6 @@ async fn internode_rpc_signature_default_posture_e2e() -> TestResult {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// A replay-scoped signature is usable exactly once against the exact gRPC path and the server
|
||||
/// process epoch that minted it. This crosses the child-process boundary twice: the HMAC-protected
|
||||
/// epoch is learned from a real response, then the same server is restarted in place to prove its
|
||||
/// replacement epoch rejects the captured request even though the nonce cache is necessarily new.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn replay_scope_rejects_replay_path_transplant_and_stale_epoch_e2e() -> TestResult {
|
||||
init_logging();
|
||||
align_rpc_secret_with_server();
|
||||
let child_env = server_env(&[]);
|
||||
let mut env = start_server_with_env(&child_env).await?;
|
||||
let url = env.url.clone();
|
||||
let audience = audience_of(&env);
|
||||
let boot_epoch = learn_boot_epoch_from_make_volume(&url, &audience).await;
|
||||
|
||||
let request = make_volume_request("replay-scope-e2e-once");
|
||||
let captured = mint_replay_scope_headers(
|
||||
&audience,
|
||||
&format!("{TONIC_RPC_PREFIX}/MakeVolume"),
|
||||
&canonical_digest(&request),
|
||||
boot_epoch,
|
||||
);
|
||||
assert_authenticated(
|
||||
call_make_volume(&url, request.clone(), captured.clone()).await,
|
||||
"the first replay-scoped mutation delivery",
|
||||
);
|
||||
assert_rejected(
|
||||
call_make_volume(&url, request.clone(), captured).await,
|
||||
Code::Unauthenticated,
|
||||
None,
|
||||
"the same replay-scoped mutation delivered twice",
|
||||
);
|
||||
|
||||
let transplanted =
|
||||
mint_replay_scope_headers(&audience, &format!("{TONIC_RPC_PREFIX}/Ping"), &canonical_digest(&request), boot_epoch);
|
||||
assert_rejected(
|
||||
call_make_volume(&url, request.clone(), transplanted).await,
|
||||
Code::Unauthenticated,
|
||||
None,
|
||||
"a replay-scoped Ping signature transplanted onto MakeVolume",
|
||||
);
|
||||
|
||||
let stale_epoch = mint_replay_scope_headers(
|
||||
&audience,
|
||||
&format!("{TONIC_RPC_PREFIX}/MakeVolume"),
|
||||
&canonical_digest(&request),
|
||||
boot_epoch,
|
||||
);
|
||||
env.restart_server_preserving_data(Vec::new(), &child_env).await?;
|
||||
rustfs_protos::evict_failed_connection(&url).await;
|
||||
assert_rejected(
|
||||
call_make_volume(&url, request.clone(), stale_epoch).await,
|
||||
Code::Unauthenticated,
|
||||
None,
|
||||
"a replay-scoped signature captured before the receiving process restart",
|
||||
);
|
||||
|
||||
let restarted_epoch = learn_boot_epoch_from_make_volume(&url, &audience).await;
|
||||
assert_ne!(boot_epoch, restarted_epoch, "a restarted child process must advertise a new boot epoch");
|
||||
let fresh_epoch = mint_replay_scope_headers(
|
||||
&audience,
|
||||
&format!("{TONIC_RPC_PREFIX}/MakeVolume"),
|
||||
&canonical_digest(&request),
|
||||
restarted_epoch,
|
||||
);
|
||||
assert_authenticated(
|
||||
call_make_volume(&url, request, fresh_epoch).await,
|
||||
"a replay-scoped mutation signed with the replacement process epoch",
|
||||
);
|
||||
|
||||
stop_server(env, &url).await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Strict replay scope leaves one authenticated v2 bootstrap: `Ping` carrying a fresh challenge.
|
||||
/// A mutating v2 request cannot use that lane; once the epoch proof is returned, the first v3
|
||||
/// mutation succeeds. This protects a server restart without reopening a general downgrade path.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn replay_scope_strict_requires_v3_after_ping_bootstrap_e2e() -> TestResult {
|
||||
init_logging();
|
||||
align_rpc_secret_with_server();
|
||||
let env = start_server(&[(ENV_INTERNODE_RPC_REPLAY_SCOPE_STRICT, "true")]).await?;
|
||||
let url = env.url.clone();
|
||||
let audience = audience_of(&env);
|
||||
|
||||
let v2_request = make_volume_request("replay-scope-e2e-strict-v2");
|
||||
assert_rejected(
|
||||
call_make_volume(
|
||||
&url,
|
||||
v2_request.clone(),
|
||||
mint_v2_headers(&audience, "MakeVolume", Some(&canonical_digest(&v2_request))),
|
||||
)
|
||||
.await,
|
||||
Code::Unauthenticated,
|
||||
None,
|
||||
"a v2 mutation after replay-scope strictness is enabled",
|
||||
);
|
||||
|
||||
let boot_epoch = learn_boot_epoch_from_ping(&url, &audience).await;
|
||||
let request = make_volume_request("replay-scope-e2e-strict-v3");
|
||||
let replay_scoped = mint_replay_scope_headers(
|
||||
&audience,
|
||||
&format!("{TONIC_RPC_PREFIX}/MakeVolume"),
|
||||
&canonical_digest(&request),
|
||||
boot_epoch,
|
||||
);
|
||||
assert_authenticated(
|
||||
call_make_volume(&url, request, replay_scoped).await,
|
||||
"a replay-scoped mutation after Ping bootstrap under strict replay scope",
|
||||
);
|
||||
|
||||
stop_server(env, &url).await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Baseline: correctly signed mutations are accepted, both with and without a
|
||||
/// body digest.
|
||||
///
|
||||
|
||||
@@ -21,22 +21,18 @@
|
||||
//! function, never as an S3 event sink.
|
||||
//!
|
||||
//! Coverage:
|
||||
//! * PUT / multipart-complete / DeleteObject / DeleteObjects each deliver one event with the correct
|
||||
//! * PUT / multipart-complete / DELETE each deliver one event with the correct
|
||||
//! eventName, bucket, key, versionId and eTag.
|
||||
//! * prefix/suffix filters drop non-matching keys (rule-engine gate).
|
||||
//! * an event queued while the target endpoint is unreachable is redelivered
|
||||
//! from the on-disk store once the endpoint recovers (store-and-forward).
|
||||
//! * responseElements and the S3 response use the canonical request ID while
|
||||
//! requestParameters preserve a conflicting client-supplied value.
|
||||
|
||||
use crate::common::{RustFSTestEnvironment, init_logging};
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::operation::RequestId;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{
|
||||
BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, Delete, Event, FilterRule, FilterRuleName,
|
||||
NotificationConfiguration, NotificationConfigurationFilter, ObjectIdentifier, QueueConfiguration, S3KeyFilter,
|
||||
VersioningConfiguration,
|
||||
BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, Event, FilterRule, FilterRuleName,
|
||||
NotificationConfiguration, NotificationConfigurationFilter, QueueConfiguration, S3KeyFilter, VersioningConfiguration,
|
||||
};
|
||||
use http::header::{CONTENT_TYPE, HOST};
|
||||
use local_ip_address::local_ip;
|
||||
@@ -44,12 +40,10 @@ use reqwest::StatusCode;
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
use rustfs_signer::sign_v4;
|
||||
use rustfs_utils::egress::ENV_OUTBOUND_ALLOW_ORIGINS;
|
||||
use rustfs_utils::http::headers::{AMZ_REQUEST_ID, REQUEST_ID_HEADER};
|
||||
use s3s::Body;
|
||||
use serde_json::Value;
|
||||
use serial_test::serial;
|
||||
use std::error::Error;
|
||||
use std::io::Cursor;
|
||||
use std::path::Path;
|
||||
use std::sync::{
|
||||
Arc, Once,
|
||||
@@ -69,8 +63,6 @@ type BoxError = Box<dyn Error + Send + Sync>;
|
||||
/// for target lookup (see `process_queue_configurations`), so the region is
|
||||
/// nominal, but it must be present for `ARN::parse` to succeed.
|
||||
const NOTIFY_REGION: &str = "us-east-1";
|
||||
const CLIENT_REQUEST_ID: &str = "client-supplied-request-id";
|
||||
const CLIENT_AMZ_REQUEST_ID: &str = "client-supplied-amz-request-id";
|
||||
|
||||
/// Webhook targets are registered as `TargetID { id: <name>, name: "webhook" }`,
|
||||
/// so the ARN a notification rule references is
|
||||
@@ -587,36 +579,6 @@ fn trimmed_etag(value: Option<&str>) -> Option<String> {
|
||||
value.map(|e| e.trim_matches('"').to_string())
|
||||
}
|
||||
|
||||
fn assert_conflicting_request_id_correlation(record: &Value, server_request_id: &str) {
|
||||
assert_eq!(
|
||||
record["requestParameters"][REQUEST_ID_HEADER].as_str(),
|
||||
Some(CLIENT_REQUEST_ID),
|
||||
"notification request parameters should retain the actual client header: {record}"
|
||||
);
|
||||
assert_eq!(
|
||||
record["requestParameters"][AMZ_REQUEST_ID].as_str(),
|
||||
Some(CLIENT_AMZ_REQUEST_ID),
|
||||
"notification request parameters should retain the actual client header: {record}"
|
||||
);
|
||||
assert_eq!(
|
||||
record["responseElements"][AMZ_REQUEST_ID].as_str(),
|
||||
Some(server_request_id),
|
||||
"notification response elements should use the canonical request ID: {record}"
|
||||
);
|
||||
}
|
||||
|
||||
fn assert_generated_request_id_correlation(record: &Value, request_id: &str) {
|
||||
assert!(
|
||||
record["requestParameters"][AMZ_REQUEST_ID].is_null(),
|
||||
"notification request parameters must not invent a client request header: {record}"
|
||||
);
|
||||
assert_eq!(
|
||||
record["responseElements"][AMZ_REQUEST_ID].as_str(),
|
||||
Some(request_id),
|
||||
"notification response elements should match the S3 response request ID: {record}"
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Tests
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -709,17 +671,8 @@ async fn test_webhook_event_delivery_and_filtering() -> TestResult {
|
||||
.bucket(bucket)
|
||||
.key(put_key)
|
||||
.body(ByteStream::from_static(b"peri-1 put body"))
|
||||
.customize()
|
||||
.mutate_request(|request| {
|
||||
request.headers_mut().insert(REQUEST_ID_HEADER, CLIENT_REQUEST_ID);
|
||||
request.headers_mut().insert(AMZ_REQUEST_ID, CLIENT_AMZ_REQUEST_ID);
|
||||
})
|
||||
.send()
|
||||
.await?;
|
||||
let put_request_id = put.request_id().ok_or("PUT response missing request ID")?.to_owned();
|
||||
assert!(uuid::Uuid::parse_str(&put_request_id).is_ok());
|
||||
assert_ne!(put_request_id, CLIENT_REQUEST_ID);
|
||||
assert_ne!(put_request_id, CLIENT_AMZ_REQUEST_ID);
|
||||
let put_version = put
|
||||
.version_id()
|
||||
.ok_or("PUT response missing versionId (versioning not enabled?)")?;
|
||||
@@ -739,7 +692,6 @@ async fn test_webhook_event_delivery_and_filtering() -> TestResult {
|
||||
"record eventName: {record}"
|
||||
);
|
||||
assert_eq!(record["s3"]["bucket"]["name"].as_str(), Some(bucket), "bucket in event: {record}");
|
||||
assert_conflicting_request_id_correlation(record, &put_request_id);
|
||||
assert_eq!(object["versionId"].as_str(), Some(put_version), "versionId in event: {object}");
|
||||
assert_eq!(
|
||||
trimmed_etag(object["eTag"].as_str()),
|
||||
@@ -792,35 +744,6 @@ async fn test_webhook_event_delivery_and_filtering() -> TestResult {
|
||||
"multipart eTag in event: {mp_record}"
|
||||
);
|
||||
|
||||
// --- Snowball extract: direct notification path keeps response correlation
|
||||
let snowball_key = "uploads/snowball.dat";
|
||||
let snowball_body = b"snowball notification body";
|
||||
let mut archive_builder = tokio_tar::Builder::new(Cursor::new(Vec::new()));
|
||||
let mut archive_header = tokio_tar::Header::new_gnu();
|
||||
archive_header.set_size(u64::try_from(snowball_body.len()).expect("snowball fixture length should fit in u64"));
|
||||
archive_header.set_mode(0o644);
|
||||
archive_header.set_cksum();
|
||||
archive_builder
|
||||
.append_data(&mut archive_header, snowball_key, Cursor::new(snowball_body))
|
||||
.await?;
|
||||
let archive = archive_builder.into_inner().await?.into_inner();
|
||||
let snowball = client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key("snowball-fixture.tar")
|
||||
.body(ByteStream::from(archive))
|
||||
.customize()
|
||||
.mutate_request(|request| {
|
||||
request.headers_mut().insert("x-amz-meta-snowball-auto-extract", "true");
|
||||
})
|
||||
.send()
|
||||
.await?;
|
||||
let snowball_request_id = snowball.request_id().ok_or("Snowball response missing request ID")?;
|
||||
|
||||
let snowball_event = wait_for_event(&mut rx, snowball_key, "s3:ObjectCreated:", Duration::from_secs(20)).await?;
|
||||
let snowball_record = &snowball_event["Records"][0];
|
||||
assert_generated_request_id_correlation(snowball_record, snowball_request_id);
|
||||
|
||||
// --- Filter: non-matching prefix and suffix must never be delivered ------
|
||||
let wrong_prefix = "logs/report.dat"; // right suffix, wrong prefix
|
||||
let wrong_suffix = "uploads/report.txt"; // right prefix, wrong suffix
|
||||
@@ -849,32 +772,7 @@ async fn test_webhook_event_delivery_and_filtering() -> TestResult {
|
||||
"wrong-suffix key {wrong_suffix} bypassed the filter; saw {seen:?}"
|
||||
);
|
||||
|
||||
// --- DeleteObjects: direct notification path keeps response correlation --
|
||||
let delete_many_key = "uploads/delete-many.dat";
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(delete_many_key)
|
||||
.body(ByteStream::from_static(b"delete objects notification body"))
|
||||
.send()
|
||||
.await?;
|
||||
wait_for_event(&mut rx, delete_many_key, "s3:ObjectCreated:", Duration::from_secs(20)).await?;
|
||||
|
||||
let delete_many = client
|
||||
.delete_objects()
|
||||
.bucket(bucket)
|
||||
.delete(
|
||||
Delete::builder()
|
||||
.objects(ObjectIdentifier::builder().key(delete_many_key).build()?)
|
||||
.build()?,
|
||||
)
|
||||
.send()
|
||||
.await?;
|
||||
let delete_many_request_id = delete_many.request_id().ok_or("DeleteObjects response missing request ID")?;
|
||||
let delete_many_event = wait_for_event(&mut rx, delete_many_key, "s3:ObjectRemoved:", Duration::from_secs(20)).await?;
|
||||
assert_generated_request_id_correlation(&delete_many_event["Records"][0], delete_many_request_id);
|
||||
|
||||
// --- DeleteObject on a versioned bucket: delete-marker version ----------
|
||||
// --- DELETE on a versioned bucket: ObjectRemoved:* with delete-marker version
|
||||
let delete = client.delete_object().bucket(bucket).key(put_key).send().await?;
|
||||
let removed = wait_for_event(&mut rx, put_key, "s3:ObjectRemoved:", Duration::from_secs(20)).await?;
|
||||
let removed_record = &removed["Records"][0];
|
||||
|
||||
@@ -24,7 +24,7 @@ use rustfs_protos::proto_gen::node_service::{BatchGenerallyLockRequest, Generall
|
||||
use tonic::Request;
|
||||
use tracing::{info, warn};
|
||||
|
||||
use crate::storage_api::grpc_lock::{AuthenticatedChannel, TonicInterceptor, node_service_time_out_client_no_auth};
|
||||
use crate::storage_api::grpc_lock::{TonicInterceptor, node_service_time_out_client_no_auth};
|
||||
|
||||
/// gRPC lock client without authentication for testing
|
||||
/// Similar to RemoteClient but uses no_auth client
|
||||
@@ -42,7 +42,7 @@ impl GrpcLockClient {
|
||||
&self,
|
||||
) -> Result<
|
||||
rustfs_protos::proto_gen::node_service::node_service_client::NodeServiceClient<
|
||||
tonic::service::interceptor::InterceptedService<AuthenticatedChannel, TonicInterceptor>,
|
||||
tonic::service::interceptor::InterceptedService<tonic::transport::Channel, TonicInterceptor>,
|
||||
>,
|
||||
> {
|
||||
node_service_time_out_client_no_auth(&self.addr)
|
||||
|
||||
@@ -3237,116 +3237,6 @@ async fn test_bucket_replication_converges_delete_marker_and_version_purge() ->
|
||||
.await?;
|
||||
assert_eq!(retained.body.collect().await?.into_bytes().as_ref(), b"versioned replication payload v2");
|
||||
|
||||
source_client
|
||||
.delete_object()
|
||||
.bucket(source_bucket)
|
||||
.key(object_key)
|
||||
.version_id(delete_marker_version_id)
|
||||
.send()
|
||||
.await?;
|
||||
assert_replication_converged(&source_client, source_bucket, &target_client, target_bucket).await?;
|
||||
let target_state = list_replication_state(&target_client, target_bucket).await?;
|
||||
assert!(
|
||||
target_state.iter().all(|entry| entry.version_id != delete_marker_version_id),
|
||||
"target retained the explicitly purged delete-marker version"
|
||||
);
|
||||
assert!(
|
||||
target_state
|
||||
.iter()
|
||||
.any(|entry| !entry.delete_marker && entry.version_id == retained_version_id),
|
||||
"target removed the retained object version while purging the delete marker: {target_state:?}"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_bucket_replication_disabled_version_delete_preserves_target_versions_issue_5442() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
let mut source_env = RustFSTestEnvironment::new().await?;
|
||||
let mut source_env_vars = replication_fast_env();
|
||||
source_env_vars.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
|
||||
source_env.start_rustfs_server_with_env(vec![], &source_env_vars).await?;
|
||||
|
||||
let mut target_env = RustFSTestEnvironment::new().await?;
|
||||
target_env.start_rustfs_server_without_cleanup(vec![]).await?;
|
||||
|
||||
let source_bucket = "replication-no-version-delete-src";
|
||||
let target_bucket = "replication-no-version-delete-dst";
|
||||
let object_key = "retained-versions.txt";
|
||||
let source_client = source_env.create_s3_client();
|
||||
let target_client = target_env.create_s3_client();
|
||||
|
||||
source_client.create_bucket().bucket(source_bucket).send().await?;
|
||||
target_client.create_bucket().bucket(target_bucket).send().await?;
|
||||
enable_bucket_versioning(&source_env, source_bucket).await?;
|
||||
enable_bucket_versioning(&target_env, target_bucket).await?;
|
||||
|
||||
let target_arn = set_replication_target(&source_env, source_bucket, &target_env, target_bucket).await?;
|
||||
put_bucket_replication_with_delete_statuses(&source_env, source_bucket, &target_arn, "Enabled", None).await?;
|
||||
|
||||
let put = source_client
|
||||
.put_object()
|
||||
.bucket(source_bucket)
|
||||
.key(object_key)
|
||||
.body(ByteStream::from_static(b"permanent delete replication disabled"))
|
||||
.send()
|
||||
.await?;
|
||||
let object_version_id = put.version_id().ok_or("source PUT omitted version ID")?.to_string();
|
||||
assert_replication_converged(&source_client, source_bucket, &target_client, target_bucket).await?;
|
||||
|
||||
let delete = source_client
|
||||
.delete_object()
|
||||
.bucket(source_bucket)
|
||||
.key(object_key)
|
||||
.send()
|
||||
.await?;
|
||||
let delete_marker_version_id = delete
|
||||
.version_id()
|
||||
.ok_or("source DELETE omitted marker version ID")?
|
||||
.to_string();
|
||||
assert_eq!(delete.delete_marker(), Some(true));
|
||||
assert_replication_converged(&source_client, source_bucket, &target_client, target_bucket).await?;
|
||||
|
||||
let expected_target_state = list_replication_state(&target_client, target_bucket).await?;
|
||||
assert_eq!(expected_target_state.len(), 2);
|
||||
assert!(
|
||||
expected_target_state
|
||||
.iter()
|
||||
.any(|entry| !entry.delete_marker && entry.version_id == object_version_id)
|
||||
);
|
||||
assert!(
|
||||
expected_target_state
|
||||
.iter()
|
||||
.any(|entry| entry.delete_marker && entry.version_id == delete_marker_version_id)
|
||||
);
|
||||
|
||||
for version_id in [&object_version_id, &delete_marker_version_id] {
|
||||
source_client
|
||||
.delete_object()
|
||||
.bucket(source_bucket)
|
||||
.key(object_key)
|
||||
.version_id(version_id)
|
||||
.send()
|
||||
.await?;
|
||||
}
|
||||
assert!(list_replication_state(&source_client, source_bucket).await?.is_empty());
|
||||
|
||||
let observation_deadline = tokio::time::Instant::now() + Duration::from_secs(10);
|
||||
loop {
|
||||
let target_state = list_replication_state(&target_client, target_bucket).await?;
|
||||
assert_eq!(
|
||||
target_state, expected_target_state,
|
||||
"disabled permanent-delete replication changed target versions"
|
||||
);
|
||||
if tokio::time::Instant::now() >= observation_deadline {
|
||||
break;
|
||||
}
|
||||
sleep(Duration::from_millis(100)).await;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
@@ -16,12 +16,9 @@
|
||||
pub(crate) use rustfs_ecstore::api::bucket::bucket_target_sys::BucketTargetSys;
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::api::disk::{VolumeInfo, WalkDirOptions};
|
||||
pub(crate) use rustfs_ecstore::api::rpc::{AuthenticatedChannel, TonicInterceptor, node_service_time_out_client_no_auth};
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::api::rpc::{
|
||||
TONIC_RPC_PREFIX, gen_signature_headers, gen_tonic_replay_scope_headers, gen_tonic_signature_headers,
|
||||
verify_tonic_boot_epoch_response,
|
||||
};
|
||||
pub(crate) use rustfs_ecstore::api::rpc::{TONIC_RPC_PREFIX, gen_signature_headers, gen_tonic_signature_headers};
|
||||
pub(crate) use rustfs_ecstore::api::rpc::{TonicInterceptor, node_service_time_out_client_no_auth};
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::api::rpc::{gen_tonic_signature_interceptor, node_service_time_out_client};
|
||||
|
||||
@@ -33,7 +30,7 @@ pub(crate) mod node_interact {
|
||||
}
|
||||
|
||||
pub(crate) mod grpc_lock {
|
||||
pub(crate) use super::{AuthenticatedChannel, TonicInterceptor, node_service_time_out_client_no_auth};
|
||||
pub(crate) use super::{TonicInterceptor, node_service_time_out_client_no_auth};
|
||||
}
|
||||
|
||||
/// Signing/transport surface used by the cross-process internode RPC signature
|
||||
@@ -43,8 +40,7 @@ pub(crate) mod grpc_lock {
|
||||
#[cfg(test)]
|
||||
pub(crate) mod internode_rpc_signature {
|
||||
pub(crate) use super::{
|
||||
TONIC_RPC_PREFIX, gen_signature_headers, gen_tonic_replay_scope_headers, gen_tonic_signature_headers,
|
||||
node_service_time_out_client_no_auth, verify_tonic_boot_epoch_response,
|
||||
TONIC_RPC_PREFIX, gen_signature_headers, gen_tonic_signature_headers, node_service_time_out_client_no_auth,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -105,7 +105,6 @@ tempfile.workspace = true
|
||||
hyper = { workspace = true, features = ["http2", "http1", "server"] }
|
||||
hyper-util = { workspace = true, features = ["tokio", "server-auto", "server-graceful", "tracing"] }
|
||||
hyper-rustls = { workspace = true, default-features = false, features = ["native-tokio", "http1", "tls12", "logging", "http2", "aws-lc-rs"] }
|
||||
hostname.workspace = true
|
||||
rustls = { workspace = true, default-features = false, features = ["aws-lc-rs", "logging", "tls12", "prefer-post-quantum", "std"] }
|
||||
rustls-pki-types.workspace = true
|
||||
tokio = { workspace = true, features = ["io-util", "sync", "signal", "fs", "rt-multi-thread"] }
|
||||
@@ -154,18 +153,11 @@ metrics = { workspace = true }
|
||||
[target.'cfg(target_os = "linux")'.dependencies]
|
||||
rustfs-uring = "0.2.1"
|
||||
|
||||
[target.'cfg(windows)'.dependencies]
|
||||
winapi-util.workspace = true
|
||||
windows-sys = { workspace = true, features = ["Win32_Foundation", "Win32_Storage_FileSystem"] }
|
||||
|
||||
[dev-dependencies]
|
||||
tokio = { workspace = true, features = ["rt-multi-thread", "macros", "test-util", "fs"] }
|
||||
criterion = { workspace = true, features = ["html_reports"] }
|
||||
temp-env = { workspace = true, features = ["async_closure"] }
|
||||
tracing-subscriber = { workspace = true, features = ["json", "env-filter", "time"] }
|
||||
# Only for `pin_callsite_interest_for_test`, which registers a `NoSubscriber`
|
||||
# dispatcher to keep tracing's process-global callsite-interest cache honest.
|
||||
tracing-core = { workspace = true }
|
||||
serial_test = { workspace = true }
|
||||
opentelemetry_sdk = { workspace = true, features = ["rt-tokio"] }
|
||||
proptest = "1"
|
||||
|
||||
@@ -67,14 +67,6 @@ pub mod bucket {
|
||||
};
|
||||
}
|
||||
|
||||
pub mod transition_transaction {
|
||||
pub use crate::bucket::lifecycle::transition_transaction::{
|
||||
TransitionOperatorDeleteResult, TransitionOperatorError, TransitionOperatorProbe, TransitionOperatorStatus,
|
||||
delete_transition_candidate_for_operator, finalize_missing_transition_transaction_for_operator,
|
||||
inspect_transition_transaction_for_operator,
|
||||
};
|
||||
}
|
||||
|
||||
pub mod evaluator {
|
||||
pub use crate::bucket::lifecycle::evaluator::Evaluator;
|
||||
}
|
||||
@@ -130,13 +122,13 @@ pub mod bucket {
|
||||
|
||||
pub mod metadata_sys {
|
||||
pub use crate::bucket::metadata_sys::{
|
||||
BucketMetadataSys, acquire_bucket_metadata_transaction_lock, delete, get, get_accelerate_config, get_bucket_policy,
|
||||
BucketMetadataSys, acquire_bucket_targets_transaction_lock, delete, get, get_accelerate_config, get_bucket_policy,
|
||||
get_bucket_policy_raw, get_bucket_targets_config, get_config_from_disk, get_cors_config, get_durability_config,
|
||||
get_global_bucket_metadata_sys, get_lifecycle_config, get_logging_config, get_notification_config,
|
||||
get_object_lock_config, get_public_access_block_config, get_quota_config, get_replication_config,
|
||||
get_request_payment_config, get_sse_config, get_tagging_config, get_versioning_config, get_website_config,
|
||||
init_bucket_metadata_sys, list_bucket_targets, reload_bucket_metadata, remove_bucket_metadata, set_bucket_metadata,
|
||||
update, update_bucket_targets_under_transaction_lock, update_config_with, update_under_transaction_lock,
|
||||
update, update_bucket_targets_under_transaction_lock, update_config_with,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -385,7 +377,6 @@ pub mod metrics {
|
||||
pub mod notification {
|
||||
pub use crate::services::notification_sys::{
|
||||
NotificationPeerErr, NotificationSys, get_global_notification_sys, new_global_notification_sys,
|
||||
start_remote_version_state_fleet_probe,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -416,15 +407,13 @@ pub mod rio {
|
||||
|
||||
pub mod rpc {
|
||||
pub use crate::cluster::rpc::{
|
||||
AuthenticatedChannel, LocalPeerS3Client, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient,
|
||||
PeerS3Client, S3PeerSys, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerBucketListing,
|
||||
ScannerPeerActivity, TONIC_RPC_PREFIX, TonicInterceptor, gen_signature_headers, gen_tonic_replay_scope_headers,
|
||||
gen_tonic_signature_headers, gen_tonic_signature_interceptor, node_service_time_out_client,
|
||||
node_service_time_out_client_no_auth, normalize_tonic_rpc_audience, set_tonic_canonical_body_digest,
|
||||
sign_ns_scanner_capability, sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers,
|
||||
verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest,
|
||||
verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof, verify_tonic_rpc_signature,
|
||||
verify_tonic_rpc_signature_with_bootstrap,
|
||||
LocalPeerS3Client, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, PeerS3Client, S3PeerSys,
|
||||
SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerBucketListing, ScannerPeerActivity,
|
||||
TONIC_RPC_PREFIX, TonicInterceptor, gen_signature_headers, gen_tonic_signature_headers, gen_tonic_signature_interceptor,
|
||||
node_service_time_out_client, node_service_time_out_client_no_auth, normalize_tonic_rpc_audience,
|
||||
set_tonic_canonical_body_digest, sign_ns_scanner_capability, sign_tonic_rpc_response_proof, verify_rpc_signature,
|
||||
verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof,
|
||||
verify_tonic_rpc_signature,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -1278,12 +1278,6 @@ fn build_remove_object_headers(version_id: Option<&str>, opts: &RemoveObjectOpti
|
||||
fn resolve_delete_api_version_id(version_id: Option<String>, opts: &RemoveObjectOptions) -> Option<String> {
|
||||
if opts.replication_request && opts.replication_delete_marker {
|
||||
None
|
||||
} else if opts.replication_request
|
||||
&& version_id
|
||||
.as_deref()
|
||||
.is_some_and(|version_id| Uuid::parse_str(version_id).is_ok_and(|version_id| version_id.is_nil()))
|
||||
{
|
||||
Some("null".to_string())
|
||||
} else {
|
||||
version_id
|
||||
}
|
||||
@@ -2265,12 +2259,6 @@ mod tests {
|
||||
assert_eq!(got.as_deref(), Some(vid.as_str()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn null_version_purge_sends_s3_null_version_id() {
|
||||
let got = resolve_delete_api_version_id(Some(Uuid::nil().to_string()), &remove_opts(true, false));
|
||||
assert_eq!(got.as_deref(), Some("null"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_marker_propagation_omits_versionid_query_param() {
|
||||
// Propagating a delete-marker CREATION (delete_marker=true): the target
|
||||
|
||||
@@ -23,7 +23,6 @@ use uuid::Uuid;
|
||||
use crate::bucket::lifecycle::config_boundary;
|
||||
use crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE;
|
||||
use crate::bucket::lifecycle::tier_sweeper::{
|
||||
delete_confirmed_transition_candidate_exact_with_lease_idempotent,
|
||||
delete_confirmed_transition_candidate_exact_with_manager_and_identity,
|
||||
delete_object_from_remote_tier_idempotent_with_manager_and_identity,
|
||||
};
|
||||
@@ -617,199 +616,6 @@ pub enum TransitionTransactionRecoveryOutcome {
|
||||
Retained,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum TransitionOperatorProbe {
|
||||
Missing,
|
||||
UnversionedPresent,
|
||||
VersionedPresent(String),
|
||||
Ambiguous,
|
||||
Unsupported,
|
||||
}
|
||||
|
||||
impl From<TransitionCandidateProbe> for TransitionOperatorProbe {
|
||||
fn from(value: TransitionCandidateProbe) -> Self {
|
||||
match value {
|
||||
TransitionCandidateProbe::Missing => Self::Missing,
|
||||
TransitionCandidateProbe::UnversionedPresent => Self::UnversionedPresent,
|
||||
TransitionCandidateProbe::VersionedPresent(version_id) => Self::VersionedPresent(version_id),
|
||||
TransitionCandidateProbe::Ambiguous => Self::Ambiguous,
|
||||
TransitionCandidateProbe::Unsupported => Self::Unsupported,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
pub struct TransitionOperatorStatus {
|
||||
pub transaction_id: Uuid,
|
||||
pub state: TransitionTransactionState,
|
||||
pub tier_name: String,
|
||||
pub remote_object: String,
|
||||
pub not_after_unix_nanos: i64,
|
||||
pub probe: TransitionOperatorProbe,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
pub struct TransitionOperatorDeleteResult {
|
||||
pub status: TransitionOperatorStatus,
|
||||
pub journal_observed_after_delete: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum TransitionOperatorError {
|
||||
#[error("transition transaction was not found")]
|
||||
NotFound,
|
||||
#[error("transition transaction is still inside its active ownership window")]
|
||||
NotExpired,
|
||||
#[error("transition transaction state is not eligible for operator reconciliation: {0:?}")]
|
||||
InvalidState(TransitionTransactionState),
|
||||
#[error("an exact non-empty remote version is required")]
|
||||
RemoteVersionRequired,
|
||||
#[error("remote candidate is not proven missing: {0:?}")]
|
||||
CandidateNotMissing(TransitionOperatorProbe),
|
||||
#[error("remote candidate version does not match requested exact version: expected {expected}, observed {actual:?}")]
|
||||
CandidateVersionMismatch {
|
||||
expected: String,
|
||||
actual: TransitionOperatorProbe,
|
||||
},
|
||||
#[error("transition transaction store failed: {0}")]
|
||||
Store(#[source] Error),
|
||||
#[error("remote tier reconciliation failed: {0}")]
|
||||
Remote(#[source] std::io::Error),
|
||||
}
|
||||
|
||||
type TransitionOperatorResult<T> = std::result::Result<T, TransitionOperatorError>;
|
||||
|
||||
fn validate_operator_reconcile_transaction(
|
||||
transaction: &TransitionTransaction,
|
||||
now_unix_nanos: i128,
|
||||
) -> TransitionOperatorResult<()> {
|
||||
transaction
|
||||
.validate()
|
||||
.map_err(|err| TransitionOperatorError::Store(Error::other(err)))?;
|
||||
if transaction.state != TransitionTransactionState::UploadOutcomeUnknown {
|
||||
return Err(TransitionOperatorError::InvalidState(transaction.state));
|
||||
}
|
||||
if now_unix_nanos < i128::from(transaction.not_after_unix_nanos) {
|
||||
return Err(TransitionOperatorError::NotExpired);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn load_operator_reconcile_transaction(
|
||||
api: Arc<ECStore>,
|
||||
transaction_id: Uuid,
|
||||
) -> TransitionOperatorResult<TransitionTransaction> {
|
||||
match load_transition_transaction_record(api, transaction_id).await {
|
||||
Ok(transaction) => Ok(transaction),
|
||||
Err(Error::ConfigNotFound) => Err(TransitionOperatorError::NotFound),
|
||||
Err(err) => Err(TransitionOperatorError::Store(err)),
|
||||
}
|
||||
}
|
||||
|
||||
async fn operator_probe_transition_candidate(
|
||||
api: Arc<ECStore>,
|
||||
transaction: &TransitionTransaction,
|
||||
) -> TransitionOperatorResult<TransitionOperatorProbe> {
|
||||
let lease = TierConfigMgr::acquire_operation_lease_for_backend_identity(
|
||||
&api.tier_config_mgr(),
|
||||
&transaction.tier_name,
|
||||
transaction.backend_fingerprint,
|
||||
)
|
||||
.await
|
||||
.map_err(|err| TransitionOperatorError::Remote(std::io::Error::other(err)))?;
|
||||
lease
|
||||
.probe_transition_candidate_for(&transaction.remote_object, transaction.transaction_id)
|
||||
.await
|
||||
.map(TransitionOperatorProbe::from)
|
||||
.map_err(TransitionOperatorError::Remote)
|
||||
}
|
||||
|
||||
pub async fn inspect_transition_transaction_for_operator(
|
||||
api: Arc<ECStore>,
|
||||
transaction_id: Uuid,
|
||||
) -> TransitionOperatorResult<TransitionOperatorStatus> {
|
||||
let transaction = load_operator_reconcile_transaction(api.clone(), transaction_id).await?;
|
||||
validate_operator_reconcile_transaction(&transaction, time::OffsetDateTime::now_utc().unix_timestamp_nanos())?;
|
||||
let probe = operator_probe_transition_candidate(api, &transaction).await?;
|
||||
Ok(TransitionOperatorStatus {
|
||||
transaction_id,
|
||||
state: transaction.state,
|
||||
tier_name: transaction.tier_name,
|
||||
remote_object: transaction.remote_object,
|
||||
not_after_unix_nanos: transaction.not_after_unix_nanos,
|
||||
probe,
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn delete_transition_candidate_for_operator(
|
||||
api: Arc<ECStore>,
|
||||
transaction_id: Uuid,
|
||||
remote_version_id: &str,
|
||||
) -> TransitionOperatorResult<TransitionOperatorDeleteResult> {
|
||||
if remote_version_id.is_empty() {
|
||||
return Err(TransitionOperatorError::RemoteVersionRequired);
|
||||
}
|
||||
let transaction = load_operator_reconcile_transaction(api.clone(), transaction_id).await?;
|
||||
validate_operator_reconcile_transaction(&transaction, time::OffsetDateTime::now_utc().unix_timestamp_nanos())?;
|
||||
let lease = TierConfigMgr::acquire_operation_lease_for_backend_identity(
|
||||
&api.tier_config_mgr(),
|
||||
&transaction.tier_name,
|
||||
transaction.backend_fingerprint,
|
||||
)
|
||||
.await
|
||||
.map_err(|err| TransitionOperatorError::Remote(std::io::Error::other(err)))?;
|
||||
lease
|
||||
.validate_remote_version_id(remote_version_id)
|
||||
.map_err(TransitionOperatorError::Remote)?;
|
||||
let before_delete_probe = lease
|
||||
.probe_transition_candidate_for(&transaction.remote_object, transaction.transaction_id)
|
||||
.await
|
||||
.map(TransitionOperatorProbe::from)
|
||||
.map_err(TransitionOperatorError::Remote)?;
|
||||
if !matches!(&before_delete_probe, TransitionOperatorProbe::VersionedPresent(version_id) if version_id == remote_version_id) {
|
||||
return Err(TransitionOperatorError::CandidateVersionMismatch {
|
||||
expected: remote_version_id.to_string(),
|
||||
actual: before_delete_probe,
|
||||
});
|
||||
}
|
||||
delete_confirmed_transition_candidate_exact_with_lease_idempotent(&transaction.remote_object, remote_version_id, &lease)
|
||||
.await
|
||||
.map_err(TransitionOperatorError::Remote)?;
|
||||
let probe = operator_probe_transition_candidate(api.clone(), &transaction).await?;
|
||||
let journal_observed_after_delete = match load_transition_transaction_record(api, transaction_id).await {
|
||||
Ok(_) => true,
|
||||
Err(Error::ConfigNotFound) => false,
|
||||
Err(err) => return Err(TransitionOperatorError::Store(err)),
|
||||
};
|
||||
Ok(TransitionOperatorDeleteResult {
|
||||
status: TransitionOperatorStatus {
|
||||
transaction_id,
|
||||
state: transaction.state,
|
||||
tier_name: transaction.tier_name,
|
||||
remote_object: transaction.remote_object,
|
||||
not_after_unix_nanos: transaction.not_after_unix_nanos,
|
||||
probe,
|
||||
},
|
||||
journal_observed_after_delete,
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn finalize_missing_transition_transaction_for_operator(
|
||||
api: Arc<ECStore>,
|
||||
transaction_id: Uuid,
|
||||
) -> TransitionOperatorResult<()> {
|
||||
let transaction = load_operator_reconcile_transaction(api.clone(), transaction_id).await?;
|
||||
validate_operator_reconcile_transaction(&transaction, time::OffsetDateTime::now_utc().unix_timestamp_nanos())?;
|
||||
let probe = operator_probe_transition_candidate(api.clone(), &transaction).await?;
|
||||
if probe != TransitionOperatorProbe::Missing {
|
||||
return Err(TransitionOperatorError::CandidateNotMissing(probe));
|
||||
}
|
||||
delete_transition_transaction_record(api, transaction_id)
|
||||
.await
|
||||
.map_err(TransitionOperatorError::Store)
|
||||
}
|
||||
|
||||
pub(crate) fn decode_transition_transaction_record(object: &str, data: &[u8]) -> Result<TransitionTransaction> {
|
||||
let transaction_id = transition_transaction_id_from_record_object_name(object)?;
|
||||
TransitionTransaction::decode(transaction_id, data)
|
||||
@@ -894,7 +700,7 @@ async fn recover_unknown_upload_outcome(
|
||||
.map_err(Error::other)?;
|
||||
|
||||
match lease
|
||||
.probe_transition_candidate_for(&transaction.remote_object, transaction.transaction_id)
|
||||
.probe_transition_candidate(&transaction.remote_object)
|
||||
.await
|
||||
.map_err(Error::other)?
|
||||
{
|
||||
@@ -1291,27 +1097,6 @@ mod tests {
|
||||
.expect("upload state change should succeed")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn operator_reconcile_requires_expired_unknown_upload_outcome() {
|
||||
let mut transaction = new_transaction();
|
||||
let active_deadline = transaction.not_after_unix_nanos;
|
||||
|
||||
assert!(matches!(
|
||||
validate_operator_reconcile_transaction(&transaction, i128::from(active_deadline) + 1),
|
||||
Err(TransitionOperatorError::InvalidState(TransitionTransactionState::UploadStarted))
|
||||
));
|
||||
|
||||
transaction
|
||||
.advance(transaction.fence(), TransitionTransactionState::UploadOutcomeUnknown, None)
|
||||
.expect("unknown upload outcome should be recorded");
|
||||
assert!(matches!(
|
||||
validate_operator_reconcile_transaction(&transaction, i128::from(active_deadline) - 1),
|
||||
Err(TransitionOperatorError::NotExpired)
|
||||
));
|
||||
validate_operator_reconcile_transaction(&transaction, i128::from(active_deadline))
|
||||
.expect("expired unknown upload outcome should be eligible");
|
||||
}
|
||||
|
||||
fn cleanup_proof(transaction: &TransitionTransaction, decision: TransitionCleanupDecision) -> TransitionCleanupProof {
|
||||
TransitionCleanupProof {
|
||||
transaction_id: transaction.transaction_id,
|
||||
|
||||
@@ -242,138 +242,73 @@ pub(crate) async fn remove_bucket_metadata_in(ctx: &crate::runtime::instance::In
|
||||
Ok(lock.remove(bucket).await)
|
||||
}
|
||||
|
||||
/// Rewrite one config file of a bucket's metadata, serialized cluster-wide.
|
||||
///
|
||||
/// See [`acquire_bucket_metadata_transaction_lock`] for why every config
|
||||
/// write — not just the replication-targets one — has to hold that lock.
|
||||
pub async fn update(bucket: &str, config_file: &str, data: Vec<u8>) -> Result<OffsetDateTime> {
|
||||
update_with_sys(get_bucket_metadata_sys()?, bucket, config_file, data).await
|
||||
}
|
||||
|
||||
pub async fn delete(bucket: &str, config_file: &str) -> Result<OffsetDateTime> {
|
||||
delete_with_sys(get_bucket_metadata_sys()?, bucket, config_file).await
|
||||
}
|
||||
|
||||
/// [`update`] against an explicitly supplied metadata system.
|
||||
///
|
||||
/// The free functions resolve the instance's own system; this variant takes
|
||||
/// it as an argument so a test can drive two independent systems over one
|
||||
/// backing store — the in-process stand-in for two nodes, which is the only
|
||||
/// configuration where the transaction lock is what does the serializing.
|
||||
async fn update_with_sys(
|
||||
sys: Arc<RwLock<BucketMetadataSys>>,
|
||||
bucket: &str,
|
||||
config_file: &str,
|
||||
data: Vec<u8>,
|
||||
) -> Result<OffsetDateTime> {
|
||||
let (_transaction_guard, mut sys) = acquire_config_write_guards(sys, bucket).await?;
|
||||
sys.update(bucket, config_file, data).await
|
||||
}
|
||||
|
||||
/// [`delete`] against an explicitly supplied metadata system. See
|
||||
/// [`update_with_sys`].
|
||||
async fn delete_with_sys(sys: Arc<RwLock<BucketMetadataSys>>, bucket: &str, config_file: &str) -> Result<OffsetDateTime> {
|
||||
let (_transaction_guard, mut sys) = acquire_config_write_guards(sys, bucket).await?;
|
||||
sys.delete(bucket, config_file).await
|
||||
}
|
||||
|
||||
/// Take, in the one order every config write uses, the two guards a
|
||||
/// read-modify-write of a bucket's metadata needs: the cluster-wide
|
||||
/// transaction lock first, then this process's metadata-system write guard.
|
||||
///
|
||||
/// The order is load-bearing. Acquiring the process-local guard first would
|
||||
/// park every local reader and writer of *every* bucket behind a lock whose
|
||||
/// holder may be another node, turning remote contention into a local stall.
|
||||
async fn acquire_config_write_guards(
|
||||
sys: Arc<RwLock<BucketMetadataSys>>,
|
||||
bucket: &str,
|
||||
) -> Result<(rustfs_lock::NamespaceLockGuard, tokio::sync::OwnedRwLockWriteGuard<BucketMetadataSys>)> {
|
||||
let transaction_guard = acquire_transaction_lock_with_sys(&sys, bucket).await?;
|
||||
let sys_guard = sys.write_owned().await;
|
||||
Ok((transaction_guard, sys_guard))
|
||||
}
|
||||
|
||||
/// Rewrite one config file while the caller already holds this bucket's
|
||||
/// transaction lock.
|
||||
///
|
||||
/// [`update`] would deadlock here: the lock is not reentrant, so a holder
|
||||
/// that called it would block until its own guard timed out.
|
||||
pub async fn update_under_transaction_lock(bucket: &str, config_file: &str, data: Vec<u8>) -> Result<OffsetDateTime> {
|
||||
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
|
||||
let _targets_guard = if config_file == BUCKET_TARGETS_FILE {
|
||||
Some(acquire_bucket_targets_transaction_lock(bucket).await?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let mut bucket_meta_sys = bucket_meta_sys_lock.write().await;
|
||||
|
||||
bucket_meta_sys.update(bucket, config_file, data).await
|
||||
}
|
||||
|
||||
pub async fn update_bucket_targets_under_transaction_lock(bucket: &str, data: Vec<u8>) -> Result<OffsetDateTime> {
|
||||
update_under_transaction_lock(bucket, BUCKET_TARGETS_FILE, data).await
|
||||
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
|
||||
let mut bucket_meta_sys = bucket_meta_sys_lock.write().await;
|
||||
bucket_meta_sys.update(bucket, BUCKET_TARGETS_FILE, data).await
|
||||
}
|
||||
|
||||
/// Read-modify-write one bucket config file under both guards a config
|
||||
/// write takes.
|
||||
/// Read-modify-write one bucket config file under the metadata system's
|
||||
/// outer write guard.
|
||||
///
|
||||
/// `mutate` sees the freshly loaded on-disk metadata and returns the
|
||||
/// replacement payload for `config_file` (empty clears it, like
|
||||
/// [`delete`]). Both the read and the persisted write happen inside the
|
||||
/// same guards [`update`] takes, so the rewrite can neither clobber a
|
||||
/// concurrent update to another config file nor lose a concurrent write to
|
||||
/// the same one — unlike caching a mutated clone of previously read
|
||||
/// metadata.
|
||||
/// same guard that [`update`] uses, so within this process the rewrite can
|
||||
/// neither clobber a concurrent update to another config file nor lose a
|
||||
/// concurrent write to the same one — unlike caching a mutated clone of
|
||||
/// previously read metadata.
|
||||
///
|
||||
/// That exclusion is cluster-wide, not merely process-local: the transaction
|
||||
/// lock is now taken for every config file rather than only the replication
|
||||
/// targets one, so a writer on another node cannot land a whole-file save in
|
||||
/// the middle of this read-modify-write.
|
||||
/// This guard is process-local. Writers on other nodes still race, exactly
|
||||
/// as they do for [`update`]: each rewrites the whole metadata file, so the
|
||||
/// later save wins. What this narrows is the window — from "as stale as the
|
||||
/// local cache" down to a single metadata read plus write.
|
||||
pub async fn update_config_with<F>(bucket: &str, config_file: &str, mutate: F) -> Result<OffsetDateTime>
|
||||
where
|
||||
F: FnOnce(&BucketMetadata) -> Result<Vec<u8>> + Send,
|
||||
{
|
||||
let (_transaction_guard, mut sys) = acquire_config_write_guards(get_bucket_metadata_sys()?, bucket).await?;
|
||||
sys.update_config_with(bucket, config_file, mutate).await
|
||||
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
|
||||
let _targets_guard = if config_file == BUCKET_TARGETS_FILE {
|
||||
Some(acquire_bucket_targets_transaction_lock(bucket).await?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let mut bucket_meta_sys = bucket_meta_sys_lock.write().await;
|
||||
bucket_meta_sys.update_config_with(bucket, config_file, mutate).await
|
||||
}
|
||||
|
||||
/// Acquire a bucket's metadata transaction lock, held across a whole
|
||||
/// read-modify-write of its metadata file.
|
||||
///
|
||||
/// Every config write loads the entire [`BucketMetadata`] blob, replaces one
|
||||
/// field, and saves the whole thing back. The namespace locks inside
|
||||
/// `read_config`/`save_config` are taken and released separately, so they do
|
||||
/// not span that cycle: two nodes updating *different* config files of one
|
||||
/// bucket both load the same blob, each set their own field, and the later
|
||||
/// save drops the other's — with both clients already told 2xx. This is not
|
||||
/// last-writer-wins on one document; an orthogonal config silently vanishes.
|
||||
///
|
||||
/// So the lock is per bucket, not per config file: a per-file key would let
|
||||
/// exactly that pair run concurrently.
|
||||
///
|
||||
/// Callers that hold this guard must use [`update_under_transaction_lock`]
|
||||
/// rather than [`update`] — see that function.
|
||||
pub async fn acquire_bucket_metadata_transaction_lock(bucket: &str) -> Result<rustfs_lock::NamespaceLockGuard> {
|
||||
acquire_transaction_lock_with_sys(&get_bucket_metadata_sys()?, bucket).await
|
||||
}
|
||||
|
||||
async fn acquire_transaction_lock_with_sys(
|
||||
sys: &Arc<RwLock<BucketMetadataSys>>,
|
||||
bucket: &str,
|
||||
) -> Result<rustfs_lock::NamespaceLockGuard> {
|
||||
// Resolve the store under a short-lived read guard: this runs before the
|
||||
// write guard in `acquire_config_write_guards`, and must not still hold a
|
||||
// read guard when the namespace lock is awaited.
|
||||
let api = sys.read().await.object_store();
|
||||
pub async fn acquire_bucket_targets_transaction_lock(bucket: &str) -> Result<rustfs_lock::NamespaceLockGuard> {
|
||||
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
|
||||
let api = bucket_meta_sys_lock.read().await.object_store();
|
||||
let lock = api
|
||||
.new_ns_lock(RUSTFS_META_BUCKET, &bucket_metadata_transaction_lock_key(bucket))
|
||||
.new_ns_lock(RUSTFS_META_BUCKET, &bucket_targets_transaction_lock_key(bucket))
|
||||
.await?;
|
||||
Ok(lock.get_write_lock(crate::set_disk::get_lock_acquire_timeout()).await?)
|
||||
}
|
||||
|
||||
/// The lock resource name is deliberately still the `bucket-targets` one it
|
||||
/// had when only replication-target writes took it. The key is what nodes
|
||||
/// agree on, so renaming it would leave a mixed-version cluster with two
|
||||
/// disjoint keys — and old and new nodes would stop excluding each other on
|
||||
/// the very writes that are serialized today.
|
||||
fn bucket_metadata_transaction_lock_key(bucket: &str) -> String {
|
||||
fn bucket_targets_transaction_lock_key(bucket: &str) -> String {
|
||||
format!("bucket-targets/{bucket}/transaction.lock")
|
||||
}
|
||||
|
||||
pub async fn delete(bucket: &str, config_file: &str) -> Result<OffsetDateTime> {
|
||||
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
|
||||
let mut bucket_meta_sys = bucket_meta_sys_lock.write().await;
|
||||
|
||||
bucket_meta_sys.delete(bucket, config_file).await
|
||||
}
|
||||
|
||||
pub async fn get_bucket_policy(bucket: &str) -> Result<(BucketPolicy, OffsetDateTime)> {
|
||||
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
|
||||
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
|
||||
@@ -929,11 +864,11 @@ impl BucketMetadataSys {
|
||||
}
|
||||
|
||||
async fn update_and_parse(&mut self, bucket: &str, config_file: &str, data: Vec<u8>, parse: bool) -> Result<OffsetDateTime> {
|
||||
// Load through this system's own store, the one `save` persists to
|
||||
// (backlog#1052 S7). Reading from the ambient handle instead made the
|
||||
// read and the write of a single read-modify-write able to target
|
||||
// different instances.
|
||||
let mut bm = Self::load_bucket_metadata_for_update(self.api.clone(), bucket, parse).await?;
|
||||
let Some(store) = runtime_sources::object_store_handle() else {
|
||||
return Err(Error::other("errServerNotInitialized"));
|
||||
};
|
||||
|
||||
let mut bm = Self::load_bucket_metadata_for_update(store, bucket, parse).await?;
|
||||
|
||||
let updated = bm.update_config(config_file, data)?;
|
||||
|
||||
@@ -1880,160 +1815,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
/// Two metadata systems over one backing store: the in-process stand-in
|
||||
/// for two nodes. They share no `RwLock`, so nothing but the transaction
|
||||
/// lock can serialize them — exactly the cross-node case.
|
||||
async fn two_nodes_over_one_store() -> (Vec<tempfile::TempDir>, Arc<RwLock<BucketMetadataSys>>, Arc<RwLock<BucketMetadataSys>>)
|
||||
{
|
||||
let (dirs, ecstore) = isolated_store_over_temp_disks().await;
|
||||
let node_a = Arc::new(RwLock::new(BucketMetadataSys::new(ecstore.clone())));
|
||||
let node_b = Arc::new(RwLock::new(BucketMetadataSys::new(ecstore)));
|
||||
(dirs, node_a, node_b)
|
||||
}
|
||||
|
||||
/// Writers on different nodes updating *different* config files of one
|
||||
/// bucket must both survive. Each rewrites the whole metadata blob, so
|
||||
/// without a lock spanning the read-modify-write the later save carries
|
||||
/// the earlier writer's field back to its pre-update value — losing an
|
||||
/// orthogonal config while both clients were told the write succeeded.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[serial]
|
||||
async fn concurrent_config_writes_from_separate_nodes_do_not_lose_writes() {
|
||||
use crate::bucket::metadata::{BUCKET_POLICY_CONFIG, BUCKET_TAGGING_CONFIG};
|
||||
|
||||
let (_dirs, node_a, node_b) = two_nodes_over_one_store().await;
|
||||
let bucket = "cross-node-config-writes";
|
||||
node_a
|
||||
.read()
|
||||
.await
|
||||
.persist_and_set(BucketMetadata::new(bucket))
|
||||
.await
|
||||
.expect("initial metadata should persist");
|
||||
|
||||
// Several rounds: a single pass can serialize by luck, but a lost
|
||||
// update only needs one interleaving to show up.
|
||||
const ROUNDS: usize = 8;
|
||||
for round in 0..ROUNDS {
|
||||
let tagging = format!("<Tagging><Round>{round}</Round></Tagging>").into_bytes();
|
||||
let policy = format!(r#"{{"Version":"2012-10-17","Round":{round}}}"#).into_bytes();
|
||||
|
||||
let start = Arc::new(tokio::sync::Barrier::new(2));
|
||||
let tagging_writer = {
|
||||
let (node, start, tagging) = (node_a.clone(), start.clone(), tagging.clone());
|
||||
tokio::spawn(async move {
|
||||
start.wait().await;
|
||||
update_with_sys(node, bucket, BUCKET_TAGGING_CONFIG, tagging).await
|
||||
})
|
||||
};
|
||||
let policy_writer = {
|
||||
let (node, start, policy) = (node_b.clone(), start.clone(), policy.clone());
|
||||
tokio::spawn(async move {
|
||||
start.wait().await;
|
||||
update_with_sys(node, bucket, BUCKET_POLICY_CONFIG, policy).await
|
||||
})
|
||||
};
|
||||
|
||||
tagging_writer
|
||||
.await
|
||||
.expect("tagging writer should join")
|
||||
.expect("tagging update should succeed");
|
||||
policy_writer
|
||||
.await
|
||||
.expect("policy writer should join")
|
||||
.expect("policy update should succeed");
|
||||
|
||||
// Disk truth, not either node's cache: the losing write is the one
|
||||
// that never reached the metadata file.
|
||||
let persisted = node_a
|
||||
.read()
|
||||
.await
|
||||
.get_config_from_disk(bucket)
|
||||
.await
|
||||
.expect("metadata should load from disk");
|
||||
assert_eq!(
|
||||
persisted.tagging_config_xml, tagging,
|
||||
"round {round}: the policy write clobbered the concurrent tagging write"
|
||||
);
|
||||
assert_eq!(
|
||||
persisted.policy_config_json, policy,
|
||||
"round {round}: the tagging write clobbered the concurrent policy write"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// The guard has to cover the load as well as the save. If it were taken
|
||||
/// only around the save, a second node could load between the two and
|
||||
/// still overwrite with pre-update state.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[serial]
|
||||
async fn bucket_metadata_transaction_lock_blocks_a_concurrent_config_write() {
|
||||
use crate::bucket::metadata::BUCKET_TAGGING_CONFIG;
|
||||
|
||||
let (_dirs, node_a, node_b) = two_nodes_over_one_store().await;
|
||||
let bucket = "cross-node-transaction-lock";
|
||||
node_a
|
||||
.read()
|
||||
.await
|
||||
.persist_and_set(BucketMetadata::new(bucket))
|
||||
.await
|
||||
.expect("initial metadata should persist");
|
||||
|
||||
let held = acquire_transaction_lock_with_sys(&node_a, bucket)
|
||||
.await
|
||||
.expect("transaction lock should be acquirable");
|
||||
|
||||
let blocked = tokio::spawn({
|
||||
let node_b = node_b.clone();
|
||||
async move { update_with_sys(node_b, bucket, BUCKET_TAGGING_CONFIG, b"<Tagging/>".to_vec()).await }
|
||||
});
|
||||
|
||||
// Long enough for the write to have finished had it not waited: the
|
||||
// whole read-modify-write against temp disks is far quicker than this.
|
||||
tokio::time::sleep(Duration::from_millis(500)).await;
|
||||
assert!(
|
||||
!blocked.is_finished(),
|
||||
"a config write must not proceed while another node holds the bucket's transaction lock"
|
||||
);
|
||||
|
||||
drop(held);
|
||||
|
||||
let updated = timeout(Duration::from_secs(10), blocked)
|
||||
.await
|
||||
.expect("the blocked write should proceed once the lock is released")
|
||||
.expect("blocked writer should join");
|
||||
updated.expect("the write should succeed after acquiring the lock");
|
||||
}
|
||||
|
||||
/// A holder of the transaction lock must not call the locking entry
|
||||
/// point: the namespace lock is not reentrant, so it would block on
|
||||
/// itself until the acquire timeout.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[serial]
|
||||
async fn transaction_lock_is_not_reentrant() {
|
||||
let (_dirs, node_a, node_b) = two_nodes_over_one_store().await;
|
||||
let bucket = "transaction-lock-reentrancy";
|
||||
|
||||
let held = acquire_transaction_lock_with_sys(&node_a, bucket)
|
||||
.await
|
||||
.expect("first acquisition should succeed");
|
||||
|
||||
// Same store, hence the same locker owner: exclusion must not depend
|
||||
// on the two acquisitions coming from different owners. Either
|
||||
// outcome is acceptable — still waiting, or refused — as long as no
|
||||
// second guard is handed out.
|
||||
let reacquired = timeout(Duration::from_millis(500), acquire_transaction_lock_with_sys(&node_b, bucket)).await;
|
||||
assert!(
|
||||
!matches!(reacquired, Ok(Ok(_))),
|
||||
"the transaction lock must exclude a second holder even under the same owner"
|
||||
);
|
||||
|
||||
drop(held);
|
||||
timeout(Duration::from_secs(10), acquire_transaction_lock_with_sys(&node_b, bucket))
|
||||
.await
|
||||
.expect("re-acquisition should not time out once released")
|
||||
.expect("the lock should be acquirable after release");
|
||||
}
|
||||
|
||||
fn target(bucket: &str, id: &str) -> BucketTarget {
|
||||
BucketTarget {
|
||||
source_bucket: bucket.to_string(),
|
||||
|
||||
@@ -13,6 +13,6 @@
|
||||
// limitations under the License.
|
||||
|
||||
pub use rustfs_replication::{
|
||||
ObjectOpts, ReplicationConfigurationExt, ReplicationTargetValidationError, delete_replication_target_arns,
|
||||
replication_target_arns, should_remove_replication_target, validate_replication_config_target_arns,
|
||||
ObjectOpts, ReplicationConfigurationExt, ReplicationTargetValidationError, replication_target_arns,
|
||||
should_remove_replication_target, validate_replication_config_target_arns,
|
||||
};
|
||||
|
||||
@@ -28,7 +28,7 @@ use super::replication_logging::{EVENT_RESYNC_CONFIG_LOOKUP_SKIPPED, LOG_COMPONE
|
||||
use super::replication_metadata_boundary::ReplicationMetadataStore;
|
||||
use super::replication_object_decision_boundary::{
|
||||
MustReplicateOptions, ReplicationDeleteSource, ReplicationResyncTargetObject, delete_replication_missing_source_decision,
|
||||
delete_replication_object_opts, delete_replication_version_id, resync_target_for_object,
|
||||
delete_replication_object_opts, resync_target_for_object,
|
||||
};
|
||||
use super::replication_storage_boundary::{ObjectInfo, ObjectOptions, ObjectToDelete, object_to_delete_for_replication};
|
||||
use super::replication_target_boundary::{BucketTargets, ReplicationTargetStore};
|
||||
@@ -73,7 +73,7 @@ impl ReplicationConfig {
|
||||
if oi.delete_marker {
|
||||
let opts = ObjectOpts {
|
||||
name: oi.name.clone(),
|
||||
version_id: delete_replication_version_id(oi.delete_marker, oi.version_id, !oi.version_purge_status.is_empty()),
|
||||
version_id: oi.version_id,
|
||||
delete_marker: true,
|
||||
op_type: ReplicationType::Delete,
|
||||
existing_object: true,
|
||||
@@ -299,14 +299,8 @@ pub(crate) async fn must_replicate(bucket: &str, object: &str, mopts: MustReplic
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use s3s::dto::{
|
||||
DeleteMarkerReplication, DeleteMarkerReplicationStatus, DeleteReplication, DeleteReplicationStatus, Destination,
|
||||
ReplicationRule, ReplicationRuleStatus,
|
||||
};
|
||||
use uuid::Uuid;
|
||||
use s3s::dto::{Destination, ReplicationRule, ReplicationRuleStatus};
|
||||
|
||||
use super::super::replication_filemeta_boundary::VersionPurgeStatusType;
|
||||
use super::super::replication_target_boundary::BucketTarget;
|
||||
use super::*;
|
||||
|
||||
fn replication_rule() -> ReplicationRule {
|
||||
@@ -366,56 +360,4 @@ mod tests {
|
||||
assert!(options.is_replication_request());
|
||||
assert_eq!(options.user_tags(), "env=prod");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn resync_distinguishes_delete_marker_creation_from_version_purge() {
|
||||
let arn = "arn:rustfs:replication:us-east-1:target:bucket";
|
||||
let mut rule = replication_rule();
|
||||
rule.destination.bucket = arn.to_string();
|
||||
rule.delete_marker_replication = Some(DeleteMarkerReplication {
|
||||
status: Some(DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::ENABLED)),
|
||||
});
|
||||
rule.delete_replication = Some(DeleteReplication {
|
||||
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::DISABLED),
|
||||
});
|
||||
let config = ReplicationConfig::new(
|
||||
Some(ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![rule],
|
||||
}),
|
||||
Some(BucketTargets {
|
||||
targets: vec![BucketTarget {
|
||||
arn: arn.to_string(),
|
||||
endpoint: "target.example".to_string(),
|
||||
..Default::default()
|
||||
}],
|
||||
}),
|
||||
);
|
||||
let marker_version_id = Uuid::new_v4();
|
||||
let marker = ObjectInfo {
|
||||
bucket: "source".to_string(),
|
||||
name: "object".to_string(),
|
||||
delete_marker: true,
|
||||
version_id: Some(marker_version_id),
|
||||
replication_status: ReplicationStatusType::Pending,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let creation = config
|
||||
.resync(marker.clone(), ReplicateDecision::default(), &HashMap::new())
|
||||
.await;
|
||||
assert!(creation.targets.contains_key(arn));
|
||||
|
||||
let purge = config
|
||||
.resync(
|
||||
ObjectInfo {
|
||||
version_purge_status: VersionPurgeStatusType::Pending,
|
||||
..marker
|
||||
},
|
||||
ReplicateDecision::default(),
|
||||
&HashMap::new(),
|
||||
)
|
||||
.await;
|
||||
assert!(purge.targets.is_empty());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,14 +13,14 @@
|
||||
// limitations under the License.
|
||||
|
||||
pub use rustfs_replication::{
|
||||
MustReplicateOptions, ReplicationDeleteScheduleInput, ReplicationDeleteStateSource, delete_replication_parts,
|
||||
delete_replication_state_from_config, delete_replication_version_id, should_schedule_delete_replication,
|
||||
should_use_existing_delete_replication_info, should_use_existing_delete_replication_source,
|
||||
MustReplicateOptions, ReplicationDeleteScheduleInput, ReplicationDeleteStateSource, delete_replication_state_from_config,
|
||||
delete_replication_version_id, should_schedule_delete_replication, should_use_existing_delete_replication_info,
|
||||
should_use_existing_delete_replication_source,
|
||||
};
|
||||
pub(crate) use rustfs_replication::{
|
||||
ReplicationDeleteSource, ReplicationMultipartPartInput, ReplicationResyncTargetObject,
|
||||
delete_replication_missing_source_decision, delete_replication_object_opts, heal_uses_delete_replication_path,
|
||||
is_retryable_delete_replication_head_error, is_version_delete_replication, replication_etags_match,
|
||||
replication_multipart_complete_actual_size, replication_multipart_part_plan, resync_target_for_object,
|
||||
should_retry_delete_marker_purge, version_purge_target_missing,
|
||||
should_retry_delete_marker_purge,
|
||||
};
|
||||
|
||||
@@ -67,7 +67,6 @@ const EVENT_REPLICATION_BACKPRESSURE: &str = "replication_backpressure";
|
||||
const EVENT_REPLICATION_RESYNC_LOAD_SKIPPED: &str = "replication_resync_load_skipped";
|
||||
const EVENT_REPLICATION_RESYNC_RECOVERED: &str = "replication_resync_recovered";
|
||||
const EVENT_REPLICATION_MRF_QUEUE_UNAVAILABLE: &str = "replication_mrf_queue_unavailable";
|
||||
const EVENT_REPLICATION_MRF_ENTRY_SKIPPED: &str = "replication_mrf_entry_skipped";
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub struct DurableMrfBacklog {
|
||||
@@ -661,21 +660,6 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
for entry in entries.iter() {
|
||||
match entry.op {
|
||||
MrfOpKind::Delete => {
|
||||
let Some(delete_parts) = entry.delete_parts_for_replay() else {
|
||||
debug!(
|
||||
event = EVENT_REPLICATION_MRF_ENTRY_SKIPPED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||
bucket = %entry.bucket,
|
||||
object = %entry.object,
|
||||
reason = "invalid_delete_version_ids",
|
||||
"Skipped invalid persisted replication delete"
|
||||
);
|
||||
continue;
|
||||
};
|
||||
let version_purge_id = delete_parts
|
||||
.version_id
|
||||
.or_else(|| delete_parts.delete_marker_version_id.filter(|_| !delete_parts.delete_marker));
|
||||
// Reconstruct a heal delete and re-queue it. We do NOT call
|
||||
// get_object_info here because the delete-marker or version may
|
||||
// already be absent from the local store — that is expected.
|
||||
@@ -690,20 +674,15 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
let oi = ObjectInfo {
|
||||
bucket: entry.bucket.clone(),
|
||||
name: entry.object.clone(),
|
||||
version_id: version_purge_id,
|
||||
delete_marker: delete_parts.delete_marker,
|
||||
replication_status: if entry.replica {
|
||||
ReplicationStatusType::Replica
|
||||
} else {
|
||||
ReplicationStatusType::Empty
|
||||
},
|
||||
version_id: entry.version_id,
|
||||
delete_marker: entry.delete_marker,
|
||||
..Default::default()
|
||||
};
|
||||
let dsc = check_replicate_delete(
|
||||
&entry.bucket,
|
||||
&ObjectToDelete {
|
||||
object_name: entry.object.clone(),
|
||||
version_id: version_purge_id,
|
||||
version_id: entry.version_id,
|
||||
..Default::default()
|
||||
},
|
||||
&oi,
|
||||
@@ -729,9 +708,9 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
let dv = DeletedObjectReplicationInfo {
|
||||
delete_object: ReplicationDeletedObject {
|
||||
object_name: entry.object.clone(),
|
||||
version_id: delete_parts.version_id,
|
||||
delete_marker_version_id: delete_parts.delete_marker_version_id,
|
||||
delete_marker: delete_parts.delete_marker,
|
||||
version_id: entry.version_id,
|
||||
delete_marker_version_id: entry.delete_marker_version_id,
|
||||
delete_marker: entry.delete_marker,
|
||||
delete_marker_mtime,
|
||||
replication_state: Some(rstate),
|
||||
..Default::default()
|
||||
@@ -2354,7 +2333,6 @@ mod tests {
|
||||
op: MrfOpKind::Object,
|
||||
delete_marker_version_id: None,
|
||||
delete_marker: false,
|
||||
replica: false,
|
||||
delete_marker_mtime: None,
|
||||
};
|
||||
let second = MrfReplicateEntry {
|
||||
@@ -2468,7 +2446,6 @@ mod tests {
|
||||
op: MrfOpKind::Object,
|
||||
delete_marker_version_id: None,
|
||||
delete_marker: false,
|
||||
replica: false,
|
||||
delete_marker_mtime: None,
|
||||
};
|
||||
|
||||
@@ -2502,7 +2479,6 @@ mod tests {
|
||||
op: MrfOpKind::Delete,
|
||||
delete_marker_version_id: Some(dm_vid),
|
||||
delete_marker: true,
|
||||
replica: false,
|
||||
delete_marker_mtime: Some(mtime_nanos),
|
||||
};
|
||||
|
||||
@@ -2536,7 +2512,6 @@ mod tests {
|
||||
op: MrfOpKind::Delete,
|
||||
delete_marker_version_id: None,
|
||||
delete_marker: false,
|
||||
replica: false,
|
||||
delete_marker_mtime: None,
|
||||
};
|
||||
|
||||
@@ -2565,7 +2540,6 @@ mod tests {
|
||||
op: MrfOpKind::Object,
|
||||
delete_marker_version_id: None,
|
||||
delete_marker: false,
|
||||
replica: false,
|
||||
delete_marker_mtime: None,
|
||||
},
|
||||
MrfReplicateEntry {
|
||||
@@ -2577,7 +2551,6 @@ mod tests {
|
||||
op: MrfOpKind::Delete,
|
||||
delete_marker_version_id: Some(del_dm_vid),
|
||||
delete_marker: true,
|
||||
replica: false,
|
||||
delete_marker_mtime: None,
|
||||
},
|
||||
];
|
||||
@@ -2607,7 +2580,6 @@ mod tests {
|
||||
op: MrfOpKind::Object,
|
||||
delete_marker_version_id: None,
|
||||
delete_marker: false,
|
||||
replica: false,
|
||||
delete_marker_mtime: None,
|
||||
};
|
||||
assert_eq!(obj_entry.op, MrfOpKind::Object);
|
||||
@@ -2622,7 +2594,6 @@ mod tests {
|
||||
op: MrfOpKind::Delete,
|
||||
delete_marker_version_id: Some(Uuid::new_v4()),
|
||||
delete_marker: true,
|
||||
replica: false,
|
||||
delete_marker_mtime: None,
|
||||
};
|
||||
assert_eq!(del_entry.op, MrfOpKind::Delete);
|
||||
@@ -2638,7 +2609,6 @@ mod tests {
|
||||
op: MrfOpKind::default(),
|
||||
delete_marker_version_id: None,
|
||||
delete_marker: false,
|
||||
replica: false,
|
||||
delete_marker_mtime: None,
|
||||
};
|
||||
assert_eq!(legacy_entry.op, MrfOpKind::Object, "legacy default must be Object");
|
||||
@@ -2702,7 +2672,6 @@ mod tests {
|
||||
op: MrfOpKind::Object,
|
||||
delete_marker_version_id: None,
|
||||
delete_marker: false,
|
||||
replica: false,
|
||||
delete_marker_mtime: None,
|
||||
}];
|
||||
let encoded = encode_mrf_file(&entries).expect("durable MRF backlog should encode");
|
||||
@@ -2733,7 +2702,6 @@ mod tests {
|
||||
op: MrfOpKind::Object,
|
||||
delete_marker_version_id: None,
|
||||
delete_marker: false,
|
||||
replica: false,
|
||||
delete_marker_mtime: None,
|
||||
}])
|
||||
.expect("invalid persisted entry should still encode for boundary testing");
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
// limitations under the License.
|
||||
|
||||
use super::replication_bandwidth_boundary;
|
||||
use super::replication_config_boundary::{ObjectOpts, ReplicationConfigurationExt as _, delete_replication_target_arns};
|
||||
use super::replication_config_boundary::{ObjectOpts, ReplicationConfigurationExt as _};
|
||||
use super::replication_config_store::ReplicationConfigStore;
|
||||
use super::replication_error_boundary::{Result, is_err_object_not_found, is_err_version_not_found};
|
||||
use super::replication_event_sink::{EventArgs, send_event, send_local_event};
|
||||
@@ -29,10 +29,9 @@ use super::replication_metadata_boundary::ReplicationMetadataStore;
|
||||
use super::replication_msgp_boundary::ReplicationMsgpCodec;
|
||||
use super::replication_object_config::{ReplicationConfig, check_replicate_delete, get_replication_config, must_replicate};
|
||||
use super::replication_object_decision_boundary::{
|
||||
MustReplicateOptions, ReplicationMultipartPartInput, delete_replication_parts, heal_uses_delete_replication_path,
|
||||
MustReplicateOptions, ReplicationMultipartPartInput, heal_uses_delete_replication_path,
|
||||
is_retryable_delete_replication_head_error, is_version_delete_replication, replication_etags_match,
|
||||
replication_multipart_complete_actual_size, replication_multipart_part_plan, should_retry_delete_marker_purge,
|
||||
version_purge_target_missing,
|
||||
};
|
||||
use super::replication_queue_boundary::DeletedObjectReplicationInfo;
|
||||
use super::replication_resync_boundary::ResyncStatusType;
|
||||
@@ -50,7 +49,7 @@ use super::replication_target_boundary::{
|
||||
PutObjectOptions, PutObjectPartOptions, ReplicationTargetStore, TargetClient, replication_action_for_target_head,
|
||||
replication_complete_multipart_options, replication_delete_marker_purge_remove_options, replication_delete_remove_options,
|
||||
replication_force_delete_remove_options, replication_object_is_ssec_encrypted, replication_put_object_header_size,
|
||||
replication_put_object_options, replication_target_head_is_newer_null_version, replication_target_version_id,
|
||||
replication_put_object_options, replication_target_head_is_newer_null_version,
|
||||
};
|
||||
use super::replication_versioning_boundary::ReplicationVersioningStore;
|
||||
use super::runtime_boundary as runtime_sources;
|
||||
@@ -71,8 +70,9 @@ use rustfs_utils::http::{
|
||||
AMZ_TAGGING_DIRECTIVE, SUFFIX_REPLICATION_RESET, SUFFIX_REPLICATION_STATUS, has_internal_suffix, insert_str,
|
||||
};
|
||||
use rustfs_utils::{DEFAULT_SIP_HASH_KEY, sip_hash};
|
||||
#[cfg(test)]
|
||||
use s3s::dto::ReplicationConfiguration;
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::collections::HashMap;
|
||||
use std::fmt::Display;
|
||||
use std::sync::Arc;
|
||||
use time::OffsetDateTime;
|
||||
@@ -97,8 +97,6 @@ const EVENT_RESYNC_TASK_FAILED: &str = "replication_resync_task_failed";
|
||||
const EVENT_RESYNC_TARGET_OPERATION_FAILED: &str = "replication_resync_target_operation_failed";
|
||||
const EVENT_RESYNC_RUNTIME_CHANNEL_FAILED: &str = "replication_resync_runtime_channel_failed";
|
||||
const ERR_REPLICATION_METADATA_COPY_UNSUPPORTED: &str = "metadata-only replication is not implemented";
|
||||
const ERR_VERSION_PURGE_TARGET_STILL_EXISTS: &str = "target version still exists after replication purge";
|
||||
const ERR_VERSION_PURGE_MISSING_VERSION_ID: &str = "version purge record is missing version ID";
|
||||
const REPLICATION_TARGET_OFFLINE_ERROR_MARKERS: &[&str] = &[
|
||||
"dispatch failure",
|
||||
"timeouterror",
|
||||
@@ -788,46 +786,19 @@ impl ReplicationResyncer {
|
||||
}
|
||||
|
||||
if roi.delete_marker || !roi.version_purge_status.is_empty() {
|
||||
let Some(parts) =
|
||||
delete_replication_parts(roi.delete_marker, roi.version_id, !roi.version_purge_status.is_empty())
|
||||
else {
|
||||
debug!(
|
||||
event = EVENT_RESYNC_OBJECT_PROCESSED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||
bucket = %bucket_name,
|
||||
object = %roi.name,
|
||||
reason = "version_purge_missing_version_id",
|
||||
"Failed to process resync version purge"
|
||||
);
|
||||
let status = TargetReplicationResyncStatus {
|
||||
bucket: roi.bucket.clone(),
|
||||
object: roi.name.clone(),
|
||||
failed_count: 1,
|
||||
error: Some(ERR_VERSION_PURGE_MISSING_VERSION_ID.to_string()),
|
||||
..Default::default()
|
||||
};
|
||||
if let Err(err) = results_tx.send(status).await {
|
||||
error!(
|
||||
event = EVENT_RESYNC_RUNTIME_CHANNEL_FAILED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||
bucket = %bucket_name,
|
||||
reason = "status_channel_send_failed",
|
||||
error = %err,
|
||||
"Failed to send resync status"
|
||||
);
|
||||
}
|
||||
continue;
|
||||
let (version_id, dm_version_id) = if roi.version_purge_status.is_empty() {
|
||||
(None, roi.version_id)
|
||||
} else {
|
||||
(roi.version_id, None)
|
||||
};
|
||||
|
||||
let doi = DeletedObjectReplicationInfo {
|
||||
delete_object: ReplicationDeletedObject {
|
||||
object_name: roi.name.clone(),
|
||||
delete_marker_version_id: parts.delete_marker_version_id,
|
||||
version_id: parts.version_id,
|
||||
delete_marker_version_id: dm_version_id,
|
||||
version_id,
|
||||
replication_state: roi.replication_state.clone(),
|
||||
delete_marker: parts.delete_marker,
|
||||
delete_marker: roi.delete_marker,
|
||||
delete_marker_mtime: roi.mod_time,
|
||||
..Default::default()
|
||||
},
|
||||
@@ -850,41 +821,21 @@ impl ReplicationResyncer {
|
||||
};
|
||||
|
||||
let reset_id = target_client.reset_id.clone();
|
||||
let is_version_purge = !roi.version_purge_status.is_empty();
|
||||
|
||||
let head_result = head_object_with_proxy_stats(
|
||||
&bucket_name,
|
||||
target_client.as_ref(),
|
||||
&target_client.bucket,
|
||||
&roi.name,
|
||||
replication_target_version_id(roi.version_id, is_version_purge),
|
||||
roi.version_id.map(|v| v.to_string()),
|
||||
)
|
||||
.await;
|
||||
let (size, err) = match head_result {
|
||||
Ok(_) if is_version_purge => {
|
||||
st.failed_count += 1;
|
||||
(0, Some(ERR_VERSION_PURGE_TARGET_STILL_EXISTS.to_string()))
|
||||
}
|
||||
Ok(_) => {
|
||||
st.replicated_count += 1;
|
||||
st.replicated_size += roi.size;
|
||||
(roi.size, None)
|
||||
}
|
||||
Err(err) if is_version_purge => {
|
||||
let (is_not_found, code) = err
|
||||
.as_service_error()
|
||||
.map(|service_err| (service_err.is_not_found(), service_err.code()))
|
||||
.unwrap_or((false, None));
|
||||
let raw_status = err.raw_response().map(|response| response.status().as_u16());
|
||||
let missing = version_purge_target_missing(is_not_found, code, raw_status);
|
||||
if missing {
|
||||
st.replicated_count += 1;
|
||||
(0, None)
|
||||
} else {
|
||||
st.failed_count += 1;
|
||||
(0, resync_target_error_detail(&err))
|
||||
}
|
||||
}
|
||||
Err(err) if roi.delete_marker => {
|
||||
// Verifying a replicated delete marker: only a
|
||||
// definitive 404/NoSuchKey or 405/MethodNotAllowed
|
||||
@@ -901,7 +852,7 @@ impl ReplicationResyncer {
|
||||
};
|
||||
if retryable {
|
||||
st.failed_count += 1;
|
||||
(0, resync_target_error_detail(&err))
|
||||
(0, Some(err))
|
||||
} else {
|
||||
st.replicated_count += 1;
|
||||
(0, None)
|
||||
@@ -920,17 +871,17 @@ impl ReplicationResyncer {
|
||||
}
|
||||
Ok(None) => {
|
||||
st.failed_count += 1;
|
||||
(0, resync_target_error_detail(&err))
|
||||
(0, Some(err))
|
||||
}
|
||||
Err(e2) => {
|
||||
st.failed_count += 1;
|
||||
(0, resync_target_error_detail(&e2))
|
||||
(0, Some(e2))
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(err) => {
|
||||
st.failed_count += 1;
|
||||
(0, resync_target_error_detail(&err))
|
||||
(0, Some(err))
|
||||
}
|
||||
};
|
||||
|
||||
@@ -960,7 +911,7 @@ impl ReplicationResyncer {
|
||||
"Processed resync object"
|
||||
);
|
||||
}
|
||||
st.error = err;
|
||||
st.error = err.as_ref().and_then(resync_target_error_detail);
|
||||
|
||||
if cancel_token.is_cancelled() {
|
||||
return;
|
||||
@@ -1114,27 +1065,22 @@ pub async fn get_heal_replicate_object_info(oi: &ObjectInfo, rcfg: &ReplicationC
|
||||
}
|
||||
|
||||
let dsc = if heal_uses_delete_replication_path(oi.delete_marker, &oi.version_purge_status) {
|
||||
match delete_replication_parts(oi.delete_marker, oi.version_id, !oi.version_purge_status.is_empty()) {
|
||||
Some(parts) => {
|
||||
check_replicate_delete(
|
||||
oi.bucket.as_str(),
|
||||
&ObjectToDelete {
|
||||
object_name: oi.name.clone(),
|
||||
version_id: parts.version_id,
|
||||
..Default::default()
|
||||
},
|
||||
&oi,
|
||||
&ObjectOptions {
|
||||
versioned: ReplicationVersioningStore::prefix_enabled(&oi.bucket, &oi.name).await,
|
||||
version_suspended: ReplicationVersioningStore::prefix_suspended(&oi.bucket, &oi.name).await,
|
||||
..Default::default()
|
||||
},
|
||||
None,
|
||||
)
|
||||
.await
|
||||
}
|
||||
None => ReplicateDecision::default(),
|
||||
}
|
||||
check_replicate_delete(
|
||||
oi.bucket.as_str(),
|
||||
&ObjectToDelete {
|
||||
object_name: oi.name.clone(),
|
||||
version_id: oi.version_id,
|
||||
..Default::default()
|
||||
},
|
||||
&oi,
|
||||
&ObjectOptions {
|
||||
versioned: ReplicationVersioningStore::prefix_enabled(&oi.bucket, &oi.name).await,
|
||||
version_suspended: ReplicationVersioningStore::prefix_suspended(&oi.bucket, &oi.name).await,
|
||||
..Default::default()
|
||||
},
|
||||
None,
|
||||
)
|
||||
.await
|
||||
} else {
|
||||
must_replicate(
|
||||
oi.bucket.as_str(),
|
||||
@@ -1204,6 +1150,7 @@ pub async fn replicate_delete<S: ReplicationStorage>(dobj: DeletedObjectReplicat
|
||||
} else {
|
||||
dobj.delete_object.version_id
|
||||
};
|
||||
|
||||
let _rcfg = match get_replication_config(&bucket).await {
|
||||
Ok(Some(config)) => config,
|
||||
Ok(None) => {
|
||||
@@ -1522,8 +1469,8 @@ pub async fn replicate_delete<S: ReplicationStorage>(dobj: DeletedObjectReplicat
|
||||
delete_marker_version_id,
|
||||
)
|
||||
.await
|
||||
&& replicate_delete_marker_purge_to_targets(&bucket_clone, &dobj_clone, &dsc_clone).await
|
||||
{
|
||||
replicate_delete_marker_purge_to_targets(&bucket_clone, &dobj_clone, &dsc_clone).await;
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(TokioDuration::from_secs(1)).await;
|
||||
@@ -1653,50 +1600,9 @@ async fn source_delete_marker_missing<S: EcstoreObjectOperations>(
|
||||
}
|
||||
}
|
||||
|
||||
fn delete_marker_purge_target_arns(config: &ReplicationConfiguration, dobj: &DeletedObjectReplicationInfo) -> HashSet<String> {
|
||||
let replica = dobj
|
||||
.delete_object
|
||||
.replication_state
|
||||
.as_ref()
|
||||
.is_some_and(|state| state.replica_status == ReplicationStatusType::Replica);
|
||||
|
||||
delete_replication_target_arns(config, &dobj.delete_object.object_name, replica)
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
enum DeleteMarkerPurgeConfig {
|
||||
Apply(HashSet<String>),
|
||||
Stop,
|
||||
Retry,
|
||||
}
|
||||
|
||||
fn delete_marker_purge_config<E>(
|
||||
result: std::result::Result<Option<ReplicationConfiguration>, E>,
|
||||
dobj: &DeletedObjectReplicationInfo,
|
||||
) -> DeleteMarkerPurgeConfig {
|
||||
match result {
|
||||
Ok(Some(config)) => DeleteMarkerPurgeConfig::Apply(delete_marker_purge_target_arns(&config, dobj)),
|
||||
Ok(None) => DeleteMarkerPurgeConfig::Stop,
|
||||
Err(_) => DeleteMarkerPurgeConfig::Retry,
|
||||
}
|
||||
}
|
||||
|
||||
async fn replicate_delete_marker_purge_to_targets(
|
||||
bucket: &str,
|
||||
dobj: &DeletedObjectReplicationInfo,
|
||||
dsc: &ReplicateDecision,
|
||||
) -> bool {
|
||||
async fn replicate_delete_marker_purge_to_targets(bucket: &str, dobj: &DeletedObjectReplicationInfo, dsc: &ReplicateDecision) {
|
||||
let Some(delete_marker_version_id) = dobj.delete_object.delete_marker_version_id else {
|
||||
return true;
|
||||
};
|
||||
let marker_creation_purge_targets = if dobj.delete_object.delete_marker {
|
||||
match delete_marker_purge_config(get_replication_config(bucket).await, dobj) {
|
||||
DeleteMarkerPurgeConfig::Apply(targets) => Some(targets),
|
||||
DeleteMarkerPurgeConfig::Stop => return true,
|
||||
DeleteMarkerPurgeConfig::Retry => return false,
|
||||
}
|
||||
} else {
|
||||
None
|
||||
return;
|
||||
};
|
||||
|
||||
for tgt_entry in dsc.targets_map.values() {
|
||||
@@ -1706,12 +1612,6 @@ async fn replicate_delete_marker_purge_to_targets(
|
||||
if !dobj.target_arn.is_empty() && dobj.target_arn != tgt_entry.arn {
|
||||
continue;
|
||||
}
|
||||
if marker_creation_purge_targets
|
||||
.as_ref()
|
||||
.is_some_and(|targets| !targets.contains(&tgt_entry.arn))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
let Some(tgt_client) = ReplicationTargetStore::remote_target_client(bucket, &tgt_entry.arn).await else {
|
||||
continue;
|
||||
};
|
||||
@@ -1725,7 +1625,6 @@ async fn replicate_delete_marker_purge_to_targets(
|
||||
)
|
||||
.await;
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &DeletedObjectReplicationInfo, storage: Arc<S>) {
|
||||
@@ -1952,10 +1851,10 @@ async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &Deleted
|
||||
}
|
||||
|
||||
async fn replicate_delete_to_target(dobj: &DeletedObjectReplicationInfo, tgt_client: Arc<TargetClient>) -> ReplicatedTargetInfo {
|
||||
let version_id = if let Some(version_id) = dobj.delete_object.delete_marker_version_id {
|
||||
Some(version_id)
|
||||
let version_id = if let Some(version_id) = &dobj.delete_object.delete_marker_version_id {
|
||||
version_id.to_owned()
|
||||
} else {
|
||||
dobj.delete_object.version_id
|
||||
dobj.delete_object.version_id.unwrap_or_default()
|
||||
};
|
||||
|
||||
let mut rinfo = dobj
|
||||
@@ -1990,7 +1889,11 @@ async fn replicate_delete_to_target(dobj: &DeletedObjectReplicationInfo, tgt_cli
|
||||
return rinfo;
|
||||
}
|
||||
|
||||
let version_id = replication_target_version_id(version_id, is_version_purge);
|
||||
let version_id = if version_id.is_nil() {
|
||||
None
|
||||
} else {
|
||||
Some(version_id.to_string())
|
||||
};
|
||||
|
||||
if dobj.delete_object.delete_marker && dobj.delete_object.delete_marker_version_id.is_some() {
|
||||
match head_object_with_proxy_stats(
|
||||
@@ -3240,10 +3143,6 @@ async fn replicate_object_with_multipart<S: ReplicationObjectIO>(ctx: MultipartR
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use s3s::dto::{
|
||||
DeleteReplication, DeleteReplicationStatus, Destination, ReplicaModifications, ReplicaModificationsStatus,
|
||||
ReplicationRule, ReplicationRuleAndOperator, ReplicationRuleFilter, ReplicationRuleStatus, SourceSelectionCriteria, Tag,
|
||||
};
|
||||
use std::collections::HashMap;
|
||||
use time::OffsetDateTime;
|
||||
use uuid::Uuid;
|
||||
@@ -3553,110 +3452,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_delete_marker_purge_targets_follow_delete_and_replica_modification_rules() {
|
||||
fn rule(arn: &str, delete_status: &'static str) -> ReplicationRule {
|
||||
ReplicationRule {
|
||||
delete_marker_replication: None,
|
||||
delete_replication: Some(DeleteReplication {
|
||||
status: DeleteReplicationStatus::from_static(delete_status),
|
||||
}),
|
||||
destination: Destination {
|
||||
bucket: arn.to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
existing_object_replication: None,
|
||||
filter: None,
|
||||
id: Some(arn.to_string()),
|
||||
prefix: Some("logs/".to_string()),
|
||||
priority: Some(1),
|
||||
source_selection_criteria: None,
|
||||
status: ReplicationRuleStatus::from_static(ReplicationRuleStatus::ENABLED),
|
||||
}
|
||||
}
|
||||
|
||||
let enabled_arn = "arn:rustfs:replication:us-east-1:target:enabled";
|
||||
let disabled_arn = "arn:rustfs:replication:us-east-1:target:disabled";
|
||||
let delete_marker_version_id = Uuid::new_v4();
|
||||
let mut config = ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![
|
||||
rule(enabled_arn, DeleteReplicationStatus::ENABLED),
|
||||
rule(disabled_arn, DeleteReplicationStatus::DISABLED),
|
||||
],
|
||||
};
|
||||
let mut dobj = DeletedObjectReplicationInfo {
|
||||
delete_object: ReplicationDeletedObject {
|
||||
object_name: "logs/object.txt".to_string(),
|
||||
delete_marker: true,
|
||||
delete_marker_version_id: Some(delete_marker_version_id),
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert_eq!(delete_marker_purge_target_arns(&config, &dobj), HashSet::from([enabled_arn.to_string()]));
|
||||
assert_eq!(
|
||||
delete_marker_purge_config::<()>(Ok(Some(config.clone())), &dobj),
|
||||
DeleteMarkerPurgeConfig::Apply(HashSet::from([enabled_arn.to_string()]))
|
||||
);
|
||||
|
||||
dobj.delete_object.replication_state = Some(Default::default());
|
||||
dobj.delete_object
|
||||
.replication_state
|
||||
.as_mut()
|
||||
.expect("test replication state")
|
||||
.replica_status = ReplicationStatusType::Replica;
|
||||
assert!(delete_marker_purge_target_arns(&config, &dobj).is_empty());
|
||||
|
||||
config.rules[0].source_selection_criteria = Some(SourceSelectionCriteria {
|
||||
replica_modifications: Some(ReplicaModifications {
|
||||
status: ReplicaModificationsStatus::from_static(ReplicaModificationsStatus::ENABLED),
|
||||
}),
|
||||
sse_kms_encrypted_objects: None,
|
||||
});
|
||||
assert_eq!(delete_marker_purge_target_arns(&config, &dobj), HashSet::from([enabled_arn.to_string()]));
|
||||
|
||||
config.rules[0].prefix = None;
|
||||
config.rules[0].filter = Some(ReplicationRuleFilter {
|
||||
tag: Some(Tag {
|
||||
key: Some("env".to_string()),
|
||||
value: Some("prod".to_string()),
|
||||
}),
|
||||
..Default::default()
|
||||
});
|
||||
assert!(delete_marker_purge_target_arns(&config, &dobj).is_empty());
|
||||
|
||||
config.rules[0].filter = Some(ReplicationRuleFilter {
|
||||
and: Some(ReplicationRuleAndOperator {
|
||||
prefix: Some("logs/".to_string()),
|
||||
tags: Some(vec![Tag {
|
||||
key: Some("env".to_string()),
|
||||
value: Some("prod".to_string()),
|
||||
}]),
|
||||
}),
|
||||
..Default::default()
|
||||
});
|
||||
assert!(delete_marker_purge_target_arns(&config, &dobj).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_delete_marker_purge_config_errors_are_retryable() {
|
||||
let delete_marker_version_id = Uuid::new_v4();
|
||||
let dobj = DeletedObjectReplicationInfo {
|
||||
delete_object: ReplicationDeletedObject {
|
||||
object_name: "object.txt".to_string(),
|
||||
delete_marker: true,
|
||||
delete_marker_version_id: Some(delete_marker_version_id),
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert_eq!(delete_marker_purge_config::<()>(Ok(None), &dobj), DeleteMarkerPurgeConfig::Stop);
|
||||
assert_eq!(delete_marker_purge_config::<()>(Err(()), &dobj), DeleteMarkerPurgeConfig::Retry);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_is_retryable_delete_replication_head_error_allows_delete_marker_head_responses() {
|
||||
assert!(
|
||||
|
||||
@@ -31,13 +31,10 @@ use rustfs_utils::http::{
|
||||
};
|
||||
use time::OffsetDateTime;
|
||||
use time::format_description::well_known::Rfc3339;
|
||||
use uuid::Uuid;
|
||||
|
||||
pub(crate) use crate::bucket::bucket_target_sys::{
|
||||
AdvancedPutOptions, PutObjectOptions, PutObjectPartOptions, RemoveObjectOptions, TargetClient,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use crate::bucket::target::BucketTarget;
|
||||
pub(crate) use crate::bucket::target::BucketTargets;
|
||||
|
||||
use super::replication_config_store::ReplicationConfigStore;
|
||||
@@ -308,14 +305,6 @@ pub(crate) fn replication_target_head_is_newer_null_version(object_info: &Object
|
||||
target_is_newer_than_source_null_version(&replication_source_object(object_info), &replication_target_object(target))
|
||||
}
|
||||
|
||||
pub(crate) fn replication_target_version_id(version_id: Option<Uuid>, version_purge: bool) -> Option<String> {
|
||||
match version_id {
|
||||
Some(version_id) if version_id.is_nil() && version_purge => Some("null".to_string()),
|
||||
Some(version_id) if !version_id.is_nil() => Some(version_id.to_string()),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn replication_delete_remove_options(
|
||||
delete_marker: bool,
|
||||
replication_mtime: Option<OffsetDateTime>,
|
||||
@@ -530,18 +519,6 @@ mod tests {
|
||||
assert!(force.replication_request);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replication_target_version_id_preserves_null_purges() {
|
||||
assert_eq!(replication_target_version_id(Some(Uuid::nil()), true).as_deref(), Some("null"));
|
||||
assert_eq!(replication_target_version_id(Some(Uuid::nil()), false), None);
|
||||
|
||||
let version_id = Uuid::new_v4();
|
||||
assert_eq!(
|
||||
replication_target_version_id(Some(version_id), true).as_deref(),
|
||||
Some(version_id.to_string().as_str())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replication_complete_multipart_options_sets_actual_size() {
|
||||
let options = replication_complete_multipart_options("1024".to_string());
|
||||
|
||||
@@ -172,7 +172,7 @@ impl ProviderVersionCapabilities {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn validate_remote_version_id(version_id: &str) -> Result<(), Error> {
|
||||
fn validate_remote_version_id(version_id: &str) -> Result<(), Error> {
|
||||
if version_id.is_empty() {
|
||||
return Err(Error::new(
|
||||
ErrorKind::InvalidData,
|
||||
|
||||
@@ -12,20 +12,11 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#[cfg(test)]
|
||||
use crate::cluster::rpc::http_auth::RPC_REPLAY_SCOPE_VERSION_HEADER;
|
||||
use crate::cluster::rpc::http_auth::{
|
||||
RPC_AUTH_VERSION_HEADER, RPC_AUTH_VERSION_V2, RPC_BOOT_EPOCH_CHALLENGE_HEADER, RPC_BOOT_EPOCH_HEADER,
|
||||
RPC_BOOT_EPOCH_PROOF_HEADER, RPC_CONTENT_SHA256_HEADER, TIMESTAMP_HEADER,
|
||||
};
|
||||
use crate::cluster::rpc::{
|
||||
gen_tonic_replay_scope_headers, gen_tonic_signature_headers, normalize_tonic_rpc_audience, verify_tonic_boot_epoch_response,
|
||||
};
|
||||
#[cfg(test)]
|
||||
use crate::cluster::rpc::{tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers};
|
||||
use crate::cluster::rpc::http_auth::RPC_CONTENT_SHA256_HEADER;
|
||||
use crate::cluster::rpc::{gen_tonic_signature_headers, normalize_tonic_rpc_audience};
|
||||
use crate::disk::error::{DiskError, Error as DiskErrorType, RpcStatusError};
|
||||
use crate::runtime::sources as runtime_sources;
|
||||
use http::{Request as HttpRequest, Response as HttpResponse, Uri};
|
||||
use http::Uri;
|
||||
use rustfs_protos::{
|
||||
ChannelClass, create_new_channel, get_channel_for_class,
|
||||
proto_gen::node_service::{
|
||||
@@ -33,19 +24,9 @@ use rustfs_protos::{
|
||||
tier_mutation_control_service_client::TierMutationControlServiceClient,
|
||||
},
|
||||
};
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
error::Error,
|
||||
future::Future,
|
||||
io::ErrorKind,
|
||||
pin::Pin,
|
||||
sync::{LazyLock, Mutex},
|
||||
task::{Context, Poll},
|
||||
};
|
||||
use std::{error::Error, io::ErrorKind};
|
||||
use tonic::{service::interceptor::InterceptedService, transport::Channel};
|
||||
use tower::Service;
|
||||
use tracing::debug;
|
||||
use uuid::Uuid;
|
||||
|
||||
use super::context_propagation::{inject_request_id_into_metadata, inject_trace_context_into_metadata};
|
||||
|
||||
@@ -54,7 +35,7 @@ use super::context_propagation::{inject_request_id_into_metadata, inject_trace_c
|
||||
pub async fn node_service_time_out_client(
|
||||
addr: &String,
|
||||
interceptor: TonicInterceptor,
|
||||
) -> Result<NodeServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>, Box<dyn Error>> {
|
||||
) -> Result<NodeServiceClient<InterceptedService<Channel, TonicInterceptor>>, Box<dyn Error>> {
|
||||
// Default to the latency-sensitive control channel; bulk `bytes` RPCs opt in via the
|
||||
// `_for_class` variant below (grpc-optimization P1).
|
||||
node_service_time_out_client_for_class(addr, interceptor, ChannelClass::Control).await
|
||||
@@ -63,14 +44,13 @@ pub async fn node_service_time_out_client(
|
||||
pub async fn heal_control_time_out_client(
|
||||
addr: &str,
|
||||
interceptor: TonicInterceptor,
|
||||
) -> Result<HealControlServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>, Box<dyn Error>> {
|
||||
) -> Result<HealControlServiceClient<InterceptedService<Channel, TonicInterceptor>>, Box<dyn Error>> {
|
||||
let interceptor = interceptor.with_rpc_audience(addr)?;
|
||||
let channel = match runtime_sources::cached_node_channel(addr).await {
|
||||
Some(channel) => channel,
|
||||
None => create_new_channel(addr).await?,
|
||||
};
|
||||
let max_message_size = rustfs_protos::HEAL_CONTROL_RPC_MAX_MESSAGE_SIZE;
|
||||
let channel = ReplayScopeChannel::new(channel, interceptor.replay_scope_audience());
|
||||
Ok(HealControlServiceClient::with_interceptor(channel, interceptor)
|
||||
.max_decoding_message_size(max_message_size)
|
||||
.max_encoding_message_size(max_message_size))
|
||||
@@ -79,14 +59,13 @@ pub async fn heal_control_time_out_client(
|
||||
pub async fn tier_mutation_control_time_out_client(
|
||||
addr: &str,
|
||||
interceptor: TonicInterceptor,
|
||||
) -> Result<TierMutationControlServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>, Box<dyn Error>> {
|
||||
) -> Result<TierMutationControlServiceClient<InterceptedService<Channel, TonicInterceptor>>, Box<dyn Error>> {
|
||||
let interceptor = interceptor.with_rpc_audience(addr)?;
|
||||
let channel = match runtime_sources::cached_node_channel(addr).await {
|
||||
Some(channel) => channel,
|
||||
None => create_new_channel(addr).await?,
|
||||
};
|
||||
let max_message_size = rustfs_protos::TIER_MUTATION_RPC_MAX_MESSAGE_SIZE;
|
||||
let channel = ReplayScopeChannel::new(channel, interceptor.replay_scope_audience());
|
||||
Ok(TierMutationControlServiceClient::with_interceptor(channel, interceptor)
|
||||
.max_decoding_message_size(max_message_size)
|
||||
.max_encoding_message_size(max_message_size))
|
||||
@@ -102,7 +81,7 @@ pub async fn node_service_time_out_client_for_class(
|
||||
addr: &String,
|
||||
interceptor: TonicInterceptor,
|
||||
class: ChannelClass,
|
||||
) -> Result<NodeServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>, Box<dyn Error>> {
|
||||
) -> Result<NodeServiceClient<InterceptedService<Channel, TonicInterceptor>>, Box<dyn Error>> {
|
||||
let interceptor = interceptor.with_rpc_audience(addr)?;
|
||||
let channel = match class {
|
||||
ChannelClass::Control => match runtime_sources::cached_node_channel(addr).await {
|
||||
@@ -117,7 +96,6 @@ pub async fn node_service_time_out_client_for_class(
|
||||
};
|
||||
|
||||
let max_message_size = rustfs_protos::internode_rpc_max_message_size();
|
||||
let channel = ReplayScopeChannel::new(channel, interceptor.replay_scope_audience());
|
||||
Ok(NodeServiceClient::with_interceptor(channel, interceptor)
|
||||
.max_decoding_message_size(max_message_size)
|
||||
.max_encoding_message_size(max_message_size))
|
||||
@@ -125,7 +103,7 @@ pub async fn node_service_time_out_client_for_class(
|
||||
|
||||
pub async fn node_service_time_out_client_no_auth(
|
||||
addr: &String,
|
||||
) -> Result<NodeServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>, Box<dyn Error>> {
|
||||
) -> Result<NodeServiceClient<InterceptedService<Channel, TonicInterceptor>>, Box<dyn Error>> {
|
||||
node_service_time_out_client(addr, TonicInterceptor::NoOp(NoOpInterceptor)).await
|
||||
}
|
||||
|
||||
@@ -221,104 +199,6 @@ pub(crate) fn is_network_like_disk_error(err: &DiskErrorType) -> bool {
|
||||
}
|
||||
}
|
||||
|
||||
/// The transport service that learns an authenticated peer boot epoch and adds the replay-scoped
|
||||
/// signature only after one has been observed. The v1/v2 interceptor stays inside this wrapper so
|
||||
/// old servers continue receiving precisely the metadata they understand.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct ReplayScopeChannel<S> {
|
||||
inner: S,
|
||||
audience: Option<String>,
|
||||
}
|
||||
|
||||
/// The channel type used by internode clients after v2 authentication and replay-scope handling.
|
||||
pub type AuthenticatedChannel = ReplayScopeChannel<Channel>;
|
||||
|
||||
static PEER_BOOT_EPOCHS: LazyLock<Mutex<HashMap<String, Uuid>>> = LazyLock::new(|| Mutex::new(HashMap::new()));
|
||||
|
||||
impl<S> ReplayScopeChannel<S> {
|
||||
fn new(inner: S, audience: Option<String>) -> Self {
|
||||
Self { inner, audience }
|
||||
}
|
||||
}
|
||||
|
||||
fn cached_peer_boot_epoch(audience: &str) -> Option<Uuid> {
|
||||
PEER_BOOT_EPOCHS.lock().ok().and_then(|epochs| epochs.get(audience).copied())
|
||||
}
|
||||
|
||||
fn remember_peer_boot_epoch(audience: String, epoch: Uuid) {
|
||||
if let Ok(mut epochs) = PEER_BOOT_EPOCHS.lock() {
|
||||
epochs.insert(audience, epoch);
|
||||
}
|
||||
}
|
||||
|
||||
impl<S, ReqBody, ResBody> Service<HttpRequest<ReqBody>> for ReplayScopeChannel<S>
|
||||
where
|
||||
S: Service<HttpRequest<ReqBody>, Response = HttpResponse<ResBody>>,
|
||||
S::Error: Send + 'static,
|
||||
S::Future: Send + 'static,
|
||||
ReqBody: Send + 'static,
|
||||
ResBody: Send + 'static,
|
||||
{
|
||||
type Response = HttpResponse<ResBody>;
|
||||
type Error = S::Error;
|
||||
type Future = Pin<Box<dyn Future<Output = Result<Self::Response, Self::Error>> + Send>>;
|
||||
|
||||
fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
|
||||
self.inner.poll_ready(cx)
|
||||
}
|
||||
|
||||
fn call(&mut self, mut request: HttpRequest<ReqBody>) -> Self::Future {
|
||||
let authenticated = self.audience.as_ref().is_some_and(|_| {
|
||||
request
|
||||
.headers()
|
||||
.get(RPC_AUTH_VERSION_HEADER)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
== Some(RPC_AUTH_VERSION_V2)
|
||||
});
|
||||
let challenge = authenticated.then(Uuid::new_v4);
|
||||
if let (Some(audience), Some(challenge)) = (self.audience.as_deref(), challenge) {
|
||||
// The challenge is independently HMAC-authenticated by the response proof. It is not
|
||||
// part of v2 so old peers ignore it, while a new peer can safely advertise its epoch.
|
||||
request.headers_mut().insert(
|
||||
RPC_BOOT_EPOCH_CHALLENGE_HEADER,
|
||||
challenge.to_string().parse().expect("UUID must be a valid header value"),
|
||||
);
|
||||
if let (Some(boot_epoch), Some(timestamp), Some(content_sha256)) = (
|
||||
cached_peer_boot_epoch(audience),
|
||||
request.headers().get(TIMESTAMP_HEADER).and_then(|value| value.to_str().ok()),
|
||||
request
|
||||
.headers()
|
||||
.get(RPC_CONTENT_SHA256_HEADER)
|
||||
.and_then(|value| value.to_str().ok()),
|
||||
) {
|
||||
match gen_tonic_replay_scope_headers(audience, request.uri().path(), timestamp, content_sha256, boot_epoch) {
|
||||
Ok(headers) => request.headers_mut().extend(headers),
|
||||
Err(error) => debug!(error = %error, "could not attach replay-scoped RPC signature"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let audience = self.audience.clone();
|
||||
let future = self.inner.call(request);
|
||||
Box::pin(async move {
|
||||
let response = future.await?;
|
||||
if let (Some(audience), Some(challenge)) = (audience, challenge) {
|
||||
match verify_tonic_boot_epoch_response(&audience, challenge, response.headers()) {
|
||||
Ok(epoch) => remember_peer_boot_epoch(audience, epoch),
|
||||
Err(error)
|
||||
if response.headers().contains_key(RPC_BOOT_EPOCH_HEADER)
|
||||
|| response.headers().contains_key(RPC_BOOT_EPOCH_PROOF_HEADER) =>
|
||||
{
|
||||
debug!(error = %error, "peer boot epoch response proof was rejected")
|
||||
}
|
||||
Err(_) => {}
|
||||
}
|
||||
}
|
||||
Ok(response)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
pub struct TonicSignatureInterceptor {
|
||||
audience: Option<String>,
|
||||
}
|
||||
@@ -377,13 +257,6 @@ impl TonicInterceptor {
|
||||
}
|
||||
Ok(self)
|
||||
}
|
||||
|
||||
fn replay_scope_audience(&self) -> Option<String> {
|
||||
match self {
|
||||
Self::Signature(interceptor) => interceptor.audience.clone(),
|
||||
Self::NoOp(_) => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl tonic::service::Interceptor for TonicInterceptor {
|
||||
@@ -406,38 +279,6 @@ mod tests {
|
||||
use tracing_opentelemetry::OpenTelemetrySpanExt;
|
||||
use tracing_subscriber::{Registry, layer::SubscriberExt};
|
||||
|
||||
#[derive(Clone)]
|
||||
struct EpochProofService {
|
||||
audience: String,
|
||||
seen_headers: std::sync::Arc<Mutex<Vec<http::HeaderMap>>>,
|
||||
}
|
||||
|
||||
impl Service<HttpRequest<()>> for EpochProofService {
|
||||
type Response = HttpResponse<()>;
|
||||
type Error = std::convert::Infallible;
|
||||
type Future = std::future::Ready<Result<Self::Response, Self::Error>>;
|
||||
|
||||
fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
|
||||
fn call(&mut self, request: HttpRequest<()>) -> Self::Future {
|
||||
self.seen_headers
|
||||
.lock()
|
||||
.expect("test header capture lock must not be poisoned")
|
||||
.push(request.headers().clone());
|
||||
let challenge = tonic_boot_epoch_challenge(request.headers())
|
||||
.expect("client challenge must be syntactically valid")
|
||||
.expect("authenticated client request must carry a boot epoch challenge");
|
||||
let mut response = HttpResponse::new(());
|
||||
response.headers_mut().extend(
|
||||
tonic_boot_epoch_response_headers(&self.audience, challenge)
|
||||
.expect("test server must be able to sign an epoch proof"),
|
||||
);
|
||||
std::future::ready(Ok(response))
|
||||
}
|
||||
}
|
||||
|
||||
fn ensure_test_rpc_secret() {
|
||||
runtime_sources::ensure_test_rpc_secret();
|
||||
}
|
||||
@@ -579,52 +420,6 @@ mod tests {
|
||||
assert_eq!(interceptor.audience.as_deref(), Some("node-a:9000"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replay_scope_channel_uses_epoch_proof_before_sending_v3() {
|
||||
ensure_test_rpc_secret();
|
||||
let audience = "replay-scope-client-test:9000";
|
||||
PEER_BOOT_EPOCHS
|
||||
.lock()
|
||||
.expect("peer epoch cache lock must not be poisoned")
|
||||
.remove(audience);
|
||||
let seen_headers = std::sync::Arc::new(Mutex::new(Vec::new()));
|
||||
let service = EpochProofService {
|
||||
audience: audience.to_string(),
|
||||
seen_headers: seen_headers.clone(),
|
||||
};
|
||||
let mut channel = ReplayScopeChannel::new(service, Some(audience.to_string()));
|
||||
let make_request = || {
|
||||
let mut request = HttpRequest::builder()
|
||||
.uri("/node_service.NodeService/Ping")
|
||||
.body(())
|
||||
.expect("test RPC request must build");
|
||||
request.headers_mut().extend(
|
||||
gen_tonic_signature_headers(audience, "node_service.NodeService", "Ping", None)
|
||||
.expect("v2 test headers must mint"),
|
||||
);
|
||||
request
|
||||
};
|
||||
|
||||
futures::executor::block_on(channel.call(make_request())).expect("first request must complete");
|
||||
futures::executor::block_on(channel.call(make_request())).expect("second request must complete");
|
||||
|
||||
let headers = seen_headers.lock().expect("test header capture lock must not be poisoned");
|
||||
assert_eq!(headers.len(), 2);
|
||||
assert!(headers[0].contains_key(RPC_BOOT_EPOCH_CHALLENGE_HEADER));
|
||||
assert!(
|
||||
!headers[0].contains_key(RPC_REPLAY_SCOPE_VERSION_HEADER),
|
||||
"the first request must remain v2-compatible until the peer proves its epoch"
|
||||
);
|
||||
assert!(
|
||||
headers[1].contains_key(RPC_REPLAY_SCOPE_VERSION_HEADER),
|
||||
"the second request must carry the replay-scoped v3 signature"
|
||||
);
|
||||
PEER_BOOT_EPOCHS
|
||||
.lock()
|
||||
.expect("peer epoch cache lock must not be poisoned")
|
||||
.remove(audience);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_signature_interceptor_requires_generated_method_metadata() {
|
||||
ensure_test_rpc_secret();
|
||||
|
||||
@@ -20,8 +20,8 @@
|
||||
//! rustfs/rustfs#4402) is anchored by the `ghsa_r5qv_*` tests in the module
|
||||
//! below, plus the broader negative-signature suite. The advisory class is: a
|
||||
//! node must never accept an RPC whose auth is missing, malformed, or signed
|
||||
//! with the default/empty shared secret. Body-bound v2 requests and all replay-scoped v3
|
||||
//! requests additionally receive process-local replay protection. See
|
||||
//! with the default/empty shared secret. Body-bound v2 requests additionally
|
||||
//! receive process-local replay protection. See
|
||||
//! `docs/testing/security-regressions.md` for the full advisory -> test map.
|
||||
//!
|
||||
//! Advisory: <https://github.com/rustfs/rustfs/security/advisories/GHSA-r5qv-rc46-hv8q>
|
||||
@@ -50,22 +50,13 @@ use uuid::Uuid;
|
||||
type HmacSha256 = Hmac<Sha256>;
|
||||
|
||||
const SIGNATURE_HEADER: &str = "x-rustfs-signature";
|
||||
pub(crate) const TIMESTAMP_HEADER: &str = "x-rustfs-timestamp";
|
||||
pub(crate) const RPC_AUTH_VERSION_HEADER: &str = "x-rustfs-rpc-auth-version";
|
||||
const TIMESTAMP_HEADER: &str = "x-rustfs-timestamp";
|
||||
const RPC_AUTH_VERSION_HEADER: &str = "x-rustfs-rpc-auth-version";
|
||||
const RPC_SIGNATURE_V2_HEADER: &str = "x-rustfs-rpc-signature-v2";
|
||||
const RPC_NONCE_HEADER: &str = "x-rustfs-rpc-nonce";
|
||||
pub(crate) const RPC_CONTENT_SHA256_HEADER: &str = "x-rustfs-content-sha256";
|
||||
pub(crate) const RPC_AUTH_VERSION_V2: &str = "2";
|
||||
pub const RPC_REPLAY_SCOPE_VERSION_HEADER: &str = "x-rustfs-rpc-replay-scope-version";
|
||||
pub const RPC_REPLAY_SCOPE_SIGNATURE_HEADER: &str = "x-rustfs-rpc-signature-v3";
|
||||
pub const RPC_REPLAY_SCOPE_NONCE_HEADER: &str = "x-rustfs-rpc-replay-nonce";
|
||||
pub const RPC_BOOT_EPOCH_HEADER: &str = "x-rustfs-rpc-boot-epoch";
|
||||
pub const RPC_BOOT_EPOCH_CHALLENGE_HEADER: &str = "x-rustfs-rpc-boot-epoch-challenge";
|
||||
pub const RPC_BOOT_EPOCH_PROOF_HEADER: &str = "x-rustfs-rpc-boot-epoch-proof";
|
||||
const RPC_REPLAY_SCOPE_VERSION_V3: &str = "3";
|
||||
const RPC_AUTH_VERSION_V2: &str = "2";
|
||||
const RPC_RESPONSE_PROOF_DOMAIN: &[u8] = b"rustfs-rpc-response-proof-v1\0";
|
||||
const RPC_REPLAY_SCOPE_DOMAIN: &[u8] = b"rustfs-rpc-replay-scope-v3\0";
|
||||
const RPC_BOOT_EPOCH_PROOF_DOMAIN: &[u8] = b"rustfs-rpc-boot-epoch-proof-v1\0";
|
||||
const UNSIGNED_PAYLOAD: &str = "UNSIGNED-PAYLOAD";
|
||||
const UNSIGNED_PAYLOAD_NONCE: &str = "unsigned";
|
||||
const SIGNATURE_VALID_DURATION: i64 = 300; // 5 minutes
|
||||
@@ -84,15 +75,9 @@ static INTERNODE_RPC_BODY_DIGEST_STRICT: LazyLock<bool> = LazyLock::new(|| {
|
||||
rustfs_config::DEFAULT_INTERNODE_RPC_BODY_DIGEST_STRICT,
|
||||
)
|
||||
});
|
||||
static INTERNODE_RPC_REPLAY_SCOPE_STRICT: LazyLock<bool> = LazyLock::new(|| {
|
||||
get_env_bool(
|
||||
rustfs_config::ENV_INTERNODE_RPC_REPLAY_SCOPE_STRICT,
|
||||
rustfs_config::DEFAULT_INTERNODE_RPC_REPLAY_SCOPE_STRICT,
|
||||
)
|
||||
});
|
||||
// Sized for peak legitimate authenticated RPC RPS x the retention window once replay scope is
|
||||
// active; overflow fails closed and increments the replay-cache overflow counter. Clamped to at
|
||||
// least 1 so a misconfigured zero cannot disable replay protection by rejecting every request.
|
||||
// Sized for peak legitimate body-bound mutation RPS x the retention window; overflow fails closed
|
||||
// and increments the replay-cache overflow counter. Clamped to at least 1 so a misconfigured zero
|
||||
// cannot disable replay protection by rejecting every body-bound request.
|
||||
static REPLAY_CACHE_CAPACITY: LazyLock<usize> = LazyLock::new(|| {
|
||||
rustfs_utils::get_env_usize(
|
||||
rustfs_config::ENV_INTERNODE_RPC_REPLAY_CACHE_CAPACITY,
|
||||
@@ -101,7 +86,6 @@ static REPLAY_CACHE_CAPACITY: LazyLock<usize> = LazyLock::new(|| {
|
||||
.max(1)
|
||||
});
|
||||
static RPC_SECRET_RESOLUTION_LOG_ONCE: Once = Once::new();
|
||||
static RPC_BOOT_EPOCH: LazyLock<Uuid> = LazyLock::new(Uuid::new_v4);
|
||||
|
||||
#[derive(Default)]
|
||||
struct RpcNonceCache {
|
||||
@@ -329,189 +313,6 @@ fn verify_signature_v2(secret: &str, scope: SignatureV2Scope<'_>, signature: &st
|
||||
mac.verify_slice(&signature).is_ok()
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
struct ReplayScope<'a> {
|
||||
audience: &'a str,
|
||||
path: &'a str,
|
||||
timestamp: &'a str,
|
||||
nonce: Uuid,
|
||||
content_sha256: &'a str,
|
||||
boot_epoch: Uuid,
|
||||
}
|
||||
|
||||
fn update_replay_scope(mac: &mut HmacSha256, scope: ReplayScope<'_>) {
|
||||
mac.update(RPC_REPLAY_SCOPE_DOMAIN);
|
||||
for part in [
|
||||
scope.audience.as_bytes(),
|
||||
b"|",
|
||||
scope.path.as_bytes(),
|
||||
b"|POST|",
|
||||
scope.timestamp.as_bytes(),
|
||||
b"|",
|
||||
scope.nonce.as_bytes(),
|
||||
b"|",
|
||||
scope.content_sha256.as_bytes(),
|
||||
b"|",
|
||||
scope.boot_epoch.as_bytes(),
|
||||
] {
|
||||
mac.update(part);
|
||||
}
|
||||
}
|
||||
|
||||
fn generate_replay_scope_signature(secret: &str, scope: ReplayScope<'_>) -> std::io::Result<String> {
|
||||
let mut mac =
|
||||
<HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
|
||||
update_replay_scope(&mut mac, scope);
|
||||
Ok(general_purpose::STANDARD.encode(mac.finalize().into_bytes()))
|
||||
}
|
||||
|
||||
fn verify_replay_scope_signature(secret: &str, scope: ReplayScope<'_>, signature: &str) -> bool {
|
||||
let Ok(signature) = general_purpose::STANDARD.decode(signature) else {
|
||||
return false;
|
||||
};
|
||||
let Ok(mut mac) = <HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()) else {
|
||||
return false;
|
||||
};
|
||||
update_replay_scope(&mut mac, scope);
|
||||
mac.verify_slice(&signature).is_ok()
|
||||
}
|
||||
|
||||
fn update_boot_epoch_proof(mac: &mut HmacSha256, audience: &str, challenge: Uuid, boot_epoch: Uuid) {
|
||||
mac.update(RPC_BOOT_EPOCH_PROOF_DOMAIN);
|
||||
mac.update(audience.as_bytes());
|
||||
mac.update(b"|");
|
||||
mac.update(challenge.as_bytes());
|
||||
mac.update(boot_epoch.as_bytes());
|
||||
}
|
||||
|
||||
fn generate_boot_epoch_proof(secret: &str, audience: &str, challenge: Uuid, boot_epoch: Uuid) -> std::io::Result<String> {
|
||||
if audience.is_empty() || challenge.is_nil() || boot_epoch.is_nil() {
|
||||
return Err(std::io::Error::other("Invalid RPC boot epoch proof scope"));
|
||||
}
|
||||
let mut mac =
|
||||
<HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
|
||||
update_boot_epoch_proof(&mut mac, audience, challenge, boot_epoch);
|
||||
Ok(general_purpose::STANDARD.encode(mac.finalize().into_bytes()))
|
||||
}
|
||||
|
||||
fn verify_boot_epoch_proof(secret: &str, audience: &str, challenge: Uuid, boot_epoch: Uuid, proof: &str) -> std::io::Result<()> {
|
||||
if audience.is_empty() || challenge.is_nil() || boot_epoch.is_nil() {
|
||||
return Err(std::io::Error::other("Invalid RPC boot epoch proof scope"));
|
||||
}
|
||||
let proof = general_purpose::STANDARD
|
||||
.decode(proof)
|
||||
.map_err(|_| std::io::Error::other("Invalid RPC boot epoch proof"))?;
|
||||
let mut mac =
|
||||
<HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
|
||||
update_boot_epoch_proof(&mut mac, audience, challenge, boot_epoch);
|
||||
mac.verify_slice(&proof)
|
||||
.map_err(|_| std::io::Error::new(std::io::ErrorKind::PermissionDenied, "Invalid RPC boot epoch proof"))
|
||||
}
|
||||
|
||||
fn non_nil_uuid(value: &str, name: &str) -> std::io::Result<Uuid> {
|
||||
let value = Uuid::parse_str(value).map_err(|_| std::io::Error::other(format!("Invalid {name}")))?;
|
||||
(!value.is_nil())
|
||||
.then_some(value)
|
||||
.ok_or_else(|| std::io::Error::other(format!("Invalid {name}")))
|
||||
}
|
||||
|
||||
fn parse_tonic_rpc_path(path: &str) -> std::io::Result<(&str, &str)> {
|
||||
path.strip_prefix('/')
|
||||
.and_then(|path| path.split_once('/'))
|
||||
.filter(|(service, rpc_method)| !service.is_empty() && !rpc_method.is_empty() && !rpc_method.contains('/'))
|
||||
.ok_or_else(|| std::io::Error::other("Invalid RPC request path"))
|
||||
}
|
||||
|
||||
/// The process-unique epoch included in every replay-scoped server verification.
|
||||
///
|
||||
/// A fresh process gets a fresh value, so a signature captured before a server restart cannot be
|
||||
/// admitted even though the bounded in-memory nonce cache necessarily starts empty again.
|
||||
pub fn tonic_rpc_boot_epoch() -> Uuid {
|
||||
*RPC_BOOT_EPOCH
|
||||
}
|
||||
|
||||
/// Build the additive replay-scope headers for a request that already carries rolling-upgrade-safe
|
||||
/// v1/v2 metadata. `timestamp` and `content_sha256` are deliberately reused from the v2 scope so
|
||||
/// old servers can continue validating the same request unchanged.
|
||||
pub fn gen_tonic_replay_scope_headers(
|
||||
audience: &str,
|
||||
path: &str,
|
||||
timestamp: &str,
|
||||
content_sha256: &str,
|
||||
boot_epoch: Uuid,
|
||||
) -> std::io::Result<HeaderMap> {
|
||||
if audience.is_empty() || !path.starts_with('/') || !valid_content_sha256(content_sha256) || boot_epoch.is_nil() {
|
||||
return Err(std::io::Error::other("Invalid replay-scoped RPC signing scope"));
|
||||
}
|
||||
parse_tonic_rpc_path(path)?;
|
||||
timestamp
|
||||
.parse::<i64>()
|
||||
.map_err(|_| std::io::Error::other("Invalid timestamp format"))?;
|
||||
|
||||
let nonce = Uuid::new_v4();
|
||||
let signature = generate_replay_scope_signature(
|
||||
&get_shared_secret()?,
|
||||
ReplayScope {
|
||||
audience,
|
||||
path,
|
||||
timestamp,
|
||||
nonce,
|
||||
content_sha256,
|
||||
boot_epoch,
|
||||
},
|
||||
)?;
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(RPC_REPLAY_SCOPE_VERSION_HEADER, HeaderValue::from_static(RPC_REPLAY_SCOPE_VERSION_V3));
|
||||
headers.insert(
|
||||
RPC_REPLAY_SCOPE_SIGNATURE_HEADER,
|
||||
header_value(&signature, RPC_REPLAY_SCOPE_SIGNATURE_HEADER)?,
|
||||
);
|
||||
headers.insert(
|
||||
RPC_REPLAY_SCOPE_NONCE_HEADER,
|
||||
header_value(&nonce.to_string(), RPC_REPLAY_SCOPE_NONCE_HEADER)?,
|
||||
);
|
||||
headers.insert(RPC_BOOT_EPOCH_HEADER, header_value(&boot_epoch.to_string(), RPC_BOOT_EPOCH_HEADER)?);
|
||||
Ok(headers)
|
||||
}
|
||||
|
||||
/// Parse the optional client challenge used to authenticate a server boot-epoch advertisement.
|
||||
pub fn tonic_boot_epoch_challenge(headers: &HeaderMap) -> std::io::Result<Option<Uuid>> {
|
||||
headers
|
||||
.get(RPC_BOOT_EPOCH_CHALLENGE_HEADER)
|
||||
.map(|value| {
|
||||
value
|
||||
.to_str()
|
||||
.map_err(|_| std::io::Error::other("Invalid RPC boot epoch challenge"))
|
||||
.and_then(|value| non_nil_uuid(value, "RPC boot epoch challenge"))
|
||||
})
|
||||
.transpose()
|
||||
}
|
||||
|
||||
/// Build the authenticated response headers for a client boot-epoch challenge.
|
||||
pub fn tonic_boot_epoch_response_headers(audience: &str, challenge: Uuid) -> std::io::Result<HeaderMap> {
|
||||
let boot_epoch = tonic_rpc_boot_epoch();
|
||||
let proof = generate_boot_epoch_proof(&get_shared_secret()?, audience, challenge, boot_epoch)?;
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(RPC_BOOT_EPOCH_HEADER, header_value(&boot_epoch.to_string(), RPC_BOOT_EPOCH_HEADER)?);
|
||||
headers.insert(RPC_BOOT_EPOCH_PROOF_HEADER, header_value(&proof, RPC_BOOT_EPOCH_PROOF_HEADER)?);
|
||||
Ok(headers)
|
||||
}
|
||||
|
||||
/// Verify the server boot-epoch response for a challenge generated by this client.
|
||||
pub fn verify_tonic_boot_epoch_response(audience: &str, challenge: Uuid, headers: &HeaderMap) -> std::io::Result<Uuid> {
|
||||
let boot_epoch = headers
|
||||
.get(RPC_BOOT_EPOCH_HEADER)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.ok_or_else(|| std::io::Error::other("Missing RPC boot epoch"))
|
||||
.and_then(|value| non_nil_uuid(value, "RPC boot epoch"))?;
|
||||
let proof = headers
|
||||
.get(RPC_BOOT_EPOCH_PROOF_HEADER)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.ok_or_else(|| std::io::Error::other("Missing RPC boot epoch proof"))?;
|
||||
verify_boot_epoch_proof(&get_shared_secret()?, audience, challenge, boot_epoch, proof)?;
|
||||
Ok(boot_epoch)
|
||||
}
|
||||
|
||||
fn valid_content_sha256(value: &str) -> bool {
|
||||
value == UNSIGNED_PAYLOAD
|
||||
|| (value.len() == 64
|
||||
@@ -730,17 +531,6 @@ fn has_v2_auth_headers(headers: &HeaderMap) -> bool {
|
||||
.any(|name| headers.contains_key(*name))
|
||||
}
|
||||
|
||||
fn has_replay_scope_headers(headers: &HeaderMap) -> bool {
|
||||
[
|
||||
RPC_REPLAY_SCOPE_VERSION_HEADER,
|
||||
RPC_REPLAY_SCOPE_SIGNATURE_HEADER,
|
||||
RPC_REPLAY_SCOPE_NONCE_HEADER,
|
||||
RPC_BOOT_EPOCH_HEADER,
|
||||
]
|
||||
.iter()
|
||||
.any(|name| headers.contains_key(*name))
|
||||
}
|
||||
|
||||
/// Whether the server requires target-bound v2 authentication on every internode gRPC request,
|
||||
/// rejecting the legacy constant-target fallback instead of accepting it. Default-off rollout
|
||||
/// lever gated on the v1-fallback counter reading zero fleet-wide; see
|
||||
@@ -750,127 +540,9 @@ fn internode_rpc_signature_strict() -> bool {
|
||||
*INTERNODE_RPC_SIGNATURE_STRICT
|
||||
}
|
||||
|
||||
fn internode_rpc_replay_scope_strict() -> bool {
|
||||
*INTERNODE_RPC_REPLAY_SCOPE_STRICT
|
||||
}
|
||||
|
||||
fn verify_tonic_replay_scope_signature(audience: &str, path: &str, headers: &HeaderMap) -> std::io::Result<()> {
|
||||
if audience.is_empty() {
|
||||
return Err(std::io::Error::other("Missing RPC audience"));
|
||||
}
|
||||
parse_tonic_rpc_path(path)?;
|
||||
|
||||
let version = headers
|
||||
.get(RPC_REPLAY_SCOPE_VERSION_HEADER)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.ok_or_else(|| std::io::Error::other("Missing RPC replay scope version"))?;
|
||||
if version != RPC_REPLAY_SCOPE_VERSION_V3 {
|
||||
return Err(std::io::Error::other("Unsupported RPC replay scope version"));
|
||||
}
|
||||
let signature = headers
|
||||
.get(RPC_REPLAY_SCOPE_SIGNATURE_HEADER)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.ok_or_else(|| std::io::Error::other("Missing RPC replay scope signature"))?;
|
||||
let timestamp = headers
|
||||
.get(TIMESTAMP_HEADER)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.ok_or_else(|| std::io::Error::other("Missing timestamp header"))?;
|
||||
let signed_at = timestamp
|
||||
.parse::<i64>()
|
||||
.map_err(|_| std::io::Error::other("Invalid timestamp format"))?;
|
||||
check_timestamp(signed_at)?;
|
||||
let nonce = headers
|
||||
.get(RPC_REPLAY_SCOPE_NONCE_HEADER)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.ok_or_else(|| std::io::Error::other("Missing RPC replay scope nonce"))
|
||||
.and_then(|value| non_nil_uuid(value, "RPC replay scope nonce"))?;
|
||||
let content_sha256 = headers
|
||||
.get(RPC_CONTENT_SHA256_HEADER)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.ok_or_else(|| std::io::Error::other("Missing RPC content SHA-256"))?;
|
||||
if !valid_content_sha256(content_sha256) {
|
||||
return Err(std::io::Error::other("Invalid RPC content SHA-256"));
|
||||
}
|
||||
let boot_epoch = headers
|
||||
.get(RPC_BOOT_EPOCH_HEADER)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.ok_or_else(|| std::io::Error::other("Missing RPC boot epoch"))
|
||||
.and_then(|value| non_nil_uuid(value, "RPC boot epoch"))?;
|
||||
let secret = get_shared_secret()?;
|
||||
if !verify_replay_scope_signature(
|
||||
&secret,
|
||||
ReplayScope {
|
||||
audience,
|
||||
path,
|
||||
timestamp,
|
||||
nonce,
|
||||
content_sha256,
|
||||
boot_epoch,
|
||||
},
|
||||
signature,
|
||||
) {
|
||||
return Err(std::io::Error::other("Invalid RPC replay scope signature"));
|
||||
}
|
||||
if boot_epoch != tonic_rpc_boot_epoch() {
|
||||
return Err(std::io::Error::other("RPC boot epoch is stale"));
|
||||
}
|
||||
check_and_record_nonce(nonce, signed_at)
|
||||
}
|
||||
|
||||
/// Verify gRPC authentication, preferring v2 without downgrade on malformed v2 metadata.
|
||||
pub fn verify_tonic_rpc_signature(audience: &str, path: &str, headers: &HeaderMap) -> std::io::Result<()> {
|
||||
verify_tonic_rpc_signature_with_policy(
|
||||
audience,
|
||||
path,
|
||||
headers,
|
||||
internode_rpc_signature_strict(),
|
||||
internode_rpc_replay_scope_strict(),
|
||||
false,
|
||||
)
|
||||
}
|
||||
|
||||
/// Verify gRPC authentication while allowing the narrowly scoped v2 `Ping` bootstrap used to
|
||||
/// obtain an authenticated server boot epoch when replay-scope strictness is enabled.
|
||||
pub fn verify_tonic_rpc_signature_with_bootstrap(
|
||||
audience: &str,
|
||||
path: &str,
|
||||
headers: &HeaderMap,
|
||||
allow_replay_scope_bootstrap: bool,
|
||||
) -> std::io::Result<()> {
|
||||
verify_tonic_rpc_signature_with_policy(
|
||||
audience,
|
||||
path,
|
||||
headers,
|
||||
internode_rpc_signature_strict(),
|
||||
internode_rpc_replay_scope_strict(),
|
||||
allow_replay_scope_bootstrap,
|
||||
)
|
||||
}
|
||||
|
||||
fn verify_tonic_rpc_signature_with_policy(
|
||||
audience: &str,
|
||||
path: &str,
|
||||
headers: &HeaderMap,
|
||||
signature_strict: bool,
|
||||
replay_scope_strict: bool,
|
||||
allow_replay_scope_bootstrap: bool,
|
||||
) -> std::io::Result<()> {
|
||||
if has_replay_scope_headers(headers) {
|
||||
return verify_tonic_replay_scope_signature(audience, path, headers);
|
||||
}
|
||||
|
||||
// Only a method-bound v2 Ping with a syntactically valid challenge may bootstrap a strict
|
||||
// client after its peer restarts. Legacy metadata never gets this exception.
|
||||
let bootstrap = allow_replay_scope_bootstrap
|
||||
&& has_v2_auth_headers(headers)
|
||||
&& tonic_boot_epoch_challenge(headers).is_ok_and(|challenge| challenge.is_some());
|
||||
if replay_scope_strict && !bootstrap {
|
||||
return Err(std::io::Error::other("RPC replay-scoped authentication required"));
|
||||
}
|
||||
|
||||
verify_tonic_rpc_signature_with_strictness(audience, path, headers, signature_strict)?;
|
||||
global_internode_metrics().record_replay_scope_fallback();
|
||||
Ok(())
|
||||
verify_tonic_rpc_signature_with_strictness(audience, path, headers, internode_rpc_signature_strict())
|
||||
}
|
||||
|
||||
/// [`verify_tonic_rpc_signature`] with the strict gate injected as a parameter, so both rollout
|
||||
@@ -1601,104 +1273,6 @@ mod tests {
|
||||
assert_eq!(error.to_string(), "Invalid RPC v2 signature");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replay_scope_binds_path_epoch_and_random_nonce() {
|
||||
ensure_test_rpc_secret();
|
||||
let path = "/node_service.NodeService/Ping";
|
||||
let mut headers = gen_tonic_signature_headers("node-a:9000", "node_service.NodeService", "Ping", None)
|
||||
.expect("v2 compatibility headers should build");
|
||||
let timestamp = headers
|
||||
.get(TIMESTAMP_HEADER)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.expect("v2 timestamp")
|
||||
.to_string();
|
||||
let content_sha256 = headers
|
||||
.get(RPC_CONTENT_SHA256_HEADER)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.expect("v2 content digest")
|
||||
.to_string();
|
||||
headers.extend(
|
||||
gen_tonic_replay_scope_headers("node-a:9000", path, ×tamp, &content_sha256, tonic_rpc_boot_epoch())
|
||||
.expect("replay-scope headers should build"),
|
||||
);
|
||||
|
||||
assert!(
|
||||
verify_tonic_rpc_signature_with_policy("node-a:9000", path, &headers, false, false, false).is_ok(),
|
||||
"the first replay-scoped request must be accepted"
|
||||
);
|
||||
let replay = verify_tonic_rpc_signature_with_policy("node-a:9000", path, &headers, false, false, false)
|
||||
.expect_err("the random replay-scope nonce must be single-use");
|
||||
assert_eq!(replay.to_string(), "RPC request replay detected");
|
||||
|
||||
let path_error = verify_tonic_replay_scope_signature("node-a:9000", "/node_service.NodeService/SignalService", &headers)
|
||||
.expect_err("a replay-scoped signature must not move to another method");
|
||||
assert_eq!(path_error.to_string(), "Invalid RPC replay scope signature");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replay_scope_rejects_partial_metadata_and_stale_epoch_without_fallback() {
|
||||
ensure_test_rpc_secret();
|
||||
let path = "/node_service.NodeService/Ping";
|
||||
let mut partial = gen_tonic_signature_headers("node-a:9000", "node_service.NodeService", "Ping", None)
|
||||
.expect("v2 compatibility headers should build");
|
||||
partial.insert(RPC_REPLAY_SCOPE_VERSION_HEADER, HeaderValue::from_static(RPC_REPLAY_SCOPE_VERSION_V3));
|
||||
let error = verify_tonic_rpc_signature_with_policy("node-a:9000", path, &partial, false, false, false)
|
||||
.expect_err("partial replay-scope metadata must never downgrade to v2");
|
||||
assert_eq!(error.to_string(), "Missing RPC replay scope signature");
|
||||
|
||||
let timestamp = partial
|
||||
.get(TIMESTAMP_HEADER)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.expect("v2 timestamp")
|
||||
.to_string();
|
||||
let content_sha256 = partial
|
||||
.get(RPC_CONTENT_SHA256_HEADER)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.expect("v2 content digest")
|
||||
.to_string();
|
||||
let stale_epoch = Uuid::new_v4();
|
||||
partial.extend(
|
||||
gen_tonic_replay_scope_headers("node-a:9000", path, ×tamp, &content_sha256, stale_epoch)
|
||||
.expect("replay-scope headers should build"),
|
||||
);
|
||||
let stale = verify_tonic_rpc_signature_with_policy("node-a:9000", path, &partial, false, false, false)
|
||||
.expect_err("a signature from a prior server boot epoch must be rejected");
|
||||
assert_eq!(stale.to_string(), "RPC boot epoch is stale");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replay_scope_strictness_allows_only_authenticated_ping_bootstrap() {
|
||||
ensure_test_rpc_secret();
|
||||
let mut headers = gen_tonic_signature_headers("node-a:9000", "node_service.NodeService", "Ping", None)
|
||||
.expect("v2 compatibility headers should build");
|
||||
let rejected =
|
||||
verify_tonic_rpc_signature_with_policy("node-a:9000", "/node_service.NodeService/Ping", &headers, false, true, false)
|
||||
.expect_err("strict replay scope must reject stripped new metadata");
|
||||
assert_eq!(rejected.to_string(), "RPC replay-scoped authentication required");
|
||||
|
||||
headers.insert(
|
||||
RPC_BOOT_EPOCH_CHALLENGE_HEADER,
|
||||
HeaderValue::from_str(&Uuid::new_v4().to_string()).expect("UUID header"),
|
||||
);
|
||||
assert!(
|
||||
verify_tonic_rpc_signature_with_policy("node-a:9000", "/node_service.NodeService/Ping", &headers, false, true, true,)
|
||||
.is_ok(),
|
||||
"only the signed Ping bootstrap may obtain a new server epoch in strict mode"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn boot_epoch_response_proof_binds_audience_challenge_and_epoch() {
|
||||
ensure_test_rpc_secret();
|
||||
let challenge = Uuid::new_v4();
|
||||
let headers = tonic_boot_epoch_response_headers("node-a:9000", challenge).expect("proof headers should build");
|
||||
let epoch =
|
||||
verify_tonic_boot_epoch_response("node-a:9000", challenge, &headers).expect("matching proof headers should verify");
|
||||
assert_eq!(epoch, tonic_rpc_boot_epoch());
|
||||
assert!(verify_tonic_boot_epoch_response("node-b:9000", challenge, &headers).is_err());
|
||||
assert!(verify_tonic_boot_epoch_response("node-a:9000", Uuid::new_v4(), &headers).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn malformed_v2_auth_does_not_downgrade_to_valid_legacy_signature() {
|
||||
ensure_test_rpc_secret();
|
||||
|
||||
@@ -26,18 +26,13 @@ pub(crate) mod runtime_sources;
|
||||
pub use background_monitor::shutdown_background_monitors;
|
||||
pub(crate) use background_monitor::spawn_background_monitor;
|
||||
pub use client::{
|
||||
AuthenticatedChannel, TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client,
|
||||
node_service_time_out_client_no_auth,
|
||||
TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client, node_service_time_out_client_no_auth,
|
||||
};
|
||||
// Re-exported through `api::rpc`; not every item is consumed inside this crate.
|
||||
#[allow(unused_imports)]
|
||||
pub use http_auth::{
|
||||
TONIC_RPC_PREFIX, build_auth_headers, gen_signature_headers, gen_tonic_replay_scope_headers, gen_tonic_signature_headers,
|
||||
normalize_tonic_rpc_audience, set_tonic_canonical_body_digest, set_tonic_mutation_body_digest, sign_ns_scanner_capability,
|
||||
sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, verify_ns_scanner_capability,
|
||||
verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest,
|
||||
verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof, verify_tonic_rpc_signature,
|
||||
verify_tonic_rpc_signature_with_bootstrap,
|
||||
TONIC_RPC_PREFIX, build_auth_headers, gen_signature_headers, gen_tonic_signature_headers, normalize_tonic_rpc_audience,
|
||||
set_tonic_canonical_body_digest, set_tonic_mutation_body_digest, sign_ns_scanner_capability, sign_tonic_rpc_response_proof,
|
||||
verify_ns_scanner_capability, verify_rpc_signature, verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest,
|
||||
verify_tonic_rpc_response_proof, verify_tonic_rpc_signature,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use internode_data_transport::TcpHttpInternodeDataTransport;
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
// limitations under the License.
|
||||
|
||||
use crate::cluster::rpc::client::{
|
||||
AuthenticatedChannel, TonicInterceptor, embedded_tonic_status, gen_tonic_signature_interceptor, heal_control_time_out_client,
|
||||
TonicInterceptor, embedded_tonic_status, gen_tonic_signature_interceptor, heal_control_time_out_client,
|
||||
is_network_like_status, message_has_network_needle, node_service_time_out_client, tier_mutation_control_time_out_client,
|
||||
};
|
||||
use crate::cluster::rpc::{set_tonic_canonical_body_digest, set_tonic_mutation_body_digest, verify_tonic_rpc_response_proof};
|
||||
@@ -58,7 +58,7 @@ use std::{
|
||||
collections::HashMap,
|
||||
io::Cursor,
|
||||
sync::{
|
||||
Arc, Weak,
|
||||
Arc,
|
||||
atomic::{AtomicBool, Ordering},
|
||||
},
|
||||
time::SystemTime,
|
||||
@@ -66,6 +66,7 @@ use std::{
|
||||
use tokio::{net::TcpStream, time::Duration};
|
||||
use tonic::Request;
|
||||
use tonic::service::interceptor::InterceptedService;
|
||||
use tonic::transport::Channel;
|
||||
use tracing::{debug, info, warn};
|
||||
use uuid::Uuid;
|
||||
|
||||
@@ -221,21 +222,6 @@ fn validate_heal_control_response_proof(canonical_response: &[u8], proof: &[u8])
|
||||
.map_err(|_| Error::other("peer returned an invalid heal control response proof"))
|
||||
}
|
||||
|
||||
fn decode_remote_version_state_capability(expected_member: &str, result: &[u8]) -> Result<Uuid> {
|
||||
let (topology_member, process_epoch) = rustfs_protos::decode_remote_version_state_capability(result).map_err(Error::other)?;
|
||||
if topology_member != expected_member {
|
||||
return Err(Error::other(
|
||||
"peer returned a remote version state capability for a different topology member",
|
||||
));
|
||||
}
|
||||
let server_epoch =
|
||||
Uuid::from_slice(process_epoch).map_err(|_| Error::other("peer returned an invalid remote version state epoch"))?;
|
||||
if server_epoch.is_nil() {
|
||||
return Err(Error::other("peer returned a nil remote version state epoch"));
|
||||
}
|
||||
Ok(server_epoch)
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct PeerLiveEventsBatch {
|
||||
pub events: Vec<u8>,
|
||||
@@ -247,7 +233,6 @@ pub struct PeerLiveEventsBatch {
|
||||
pub struct PeerRestClient {
|
||||
pub host: XHost,
|
||||
pub grid_host: String,
|
||||
topology_member: String,
|
||||
offline: Arc<AtomicBool>,
|
||||
recovery_running: Arc<AtomicBool>,
|
||||
}
|
||||
@@ -340,54 +325,14 @@ impl PeerRestClient {
|
||||
}
|
||||
|
||||
pub fn new(host: XHost, grid_host: String) -> Self {
|
||||
let topology_member = host.to_string();
|
||||
Self {
|
||||
host,
|
||||
grid_host,
|
||||
topology_member,
|
||||
offline: Arc::new(AtomicBool::new(false)),
|
||||
recovery_running: Arc::new(AtomicBool::new(false)),
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_topology_host(peer_host_port: &str, grid_host: &str) -> Result<XHost> {
|
||||
let url = url::Url::parse(grid_host).map_err(|_| Error::other("peer grid host is not a valid URL"))?;
|
||||
if !matches!(url.scheme(), "http" | "https")
|
||||
|| !url.username().is_empty()
|
||||
|| url.password().is_some()
|
||||
|| url.query().is_some()
|
||||
|| url.fragment().is_some()
|
||||
|| url.path() != "/"
|
||||
{
|
||||
return Err(Error::other("peer grid host has an invalid URL shape"));
|
||||
}
|
||||
let url_host = url.host().ok_or_else(|| Error::other("peer grid host is missing a host"))?;
|
||||
let topology_host = match url.port() {
|
||||
Some(port) => format!("{url_host}:{port}"),
|
||||
None => url_host.to_string(),
|
||||
};
|
||||
let explicit_port = url.port();
|
||||
let name = match url_host {
|
||||
url::Host::Domain(domain) => domain.to_string(),
|
||||
url::Host::Ipv4(address) => address.to_string(),
|
||||
url::Host::Ipv6(address) if explicit_port.is_none() => format!("[{address}]"),
|
||||
url::Host::Ipv6(address) => address.to_string(),
|
||||
};
|
||||
let port = url
|
||||
.port_or_known_default()
|
||||
.filter(|port| *port > 0)
|
||||
.ok_or_else(|| Error::other("peer grid host is missing a valid port"))?;
|
||||
let host = XHost {
|
||||
name,
|
||||
port,
|
||||
is_port_set: explicit_port.is_some(),
|
||||
};
|
||||
if topology_host != peer_host_port {
|
||||
return Err(Error::other("peer topology host does not match its grid URL"));
|
||||
}
|
||||
Ok(host)
|
||||
}
|
||||
|
||||
fn build_clients_from_slots(
|
||||
slots: Vec<(String, Option<String>, bool)>,
|
||||
) -> (Vec<Option<Self>>, Vec<Option<Self>>, Vec<String>) {
|
||||
@@ -401,14 +346,10 @@ impl PeerRestClient {
|
||||
}
|
||||
|
||||
let client = match grid_host {
|
||||
Some(grid_host) => match Self::parse_topology_host(&peer_host_port, &grid_host) {
|
||||
Ok(host) => {
|
||||
let mut client = PeerRestClient::new(host, grid_host);
|
||||
client.topology_member = peer_host_port.clone();
|
||||
Some(client)
|
||||
}
|
||||
Some(grid_host) => match XHost::try_from(peer_host_port.clone()) {
|
||||
Ok(host) => Some(PeerRestClient::new(host, grid_host)),
|
||||
Err(err) => {
|
||||
warn!(peer = %peer_host_port, "peer topology host parse failed while constructing peer client: {err:?}");
|
||||
warn!(peer = %peer_host_port, "Xhost parse failed while constructing peer client: {err:?}");
|
||||
None
|
||||
}
|
||||
},
|
||||
@@ -449,7 +390,7 @@ impl PeerRestClient {
|
||||
(remote, all, remote_topology_hosts)
|
||||
}
|
||||
|
||||
pub async fn get_client(&self) -> Result<NodeServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>> {
|
||||
pub async fn get_client(&self) -> Result<NodeServiceClient<InterceptedService<Channel, TonicInterceptor>>> {
|
||||
if self.offline.load(Ordering::Acquire) {
|
||||
self.mark_offline_and_spawn_recovery();
|
||||
return Err(Error::other(format!("peer {} is temporarily offline", self.grid_host)));
|
||||
@@ -470,7 +411,7 @@ impl PeerRestClient {
|
||||
&self,
|
||||
) -> Result<
|
||||
rustfs_protos::proto_gen::node_service::heal_control_service_client::HealControlServiceClient<
|
||||
InterceptedService<AuthenticatedChannel, TonicInterceptor>,
|
||||
InterceptedService<Channel, TonicInterceptor>,
|
||||
>,
|
||||
> {
|
||||
if self.offline.load(Ordering::Acquire) {
|
||||
@@ -491,7 +432,7 @@ impl PeerRestClient {
|
||||
|
||||
async fn get_tier_mutation_control_client(
|
||||
&self,
|
||||
) -> Result<TierMutationControlServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>> {
|
||||
) -> Result<TierMutationControlServiceClient<InterceptedService<Channel, TonicInterceptor>>> {
|
||||
if self.offline.load(Ordering::Acquire) {
|
||||
self.mark_offline_and_spawn_recovery();
|
||||
return Err(Error::other(format!("peer {} is temporarily offline", self.grid_host)));
|
||||
@@ -557,8 +498,9 @@ impl PeerRestClient {
|
||||
}
|
||||
|
||||
let grid_host = self.grid_host.clone();
|
||||
let offline = Arc::downgrade(&self.offline);
|
||||
let recovery_running = Arc::downgrade(&self.recovery_running);
|
||||
let offline = Arc::clone(&self.offline);
|
||||
let recovery_running = Arc::clone(&self.recovery_running);
|
||||
let span = Self::recovery_monitor_span(&grid_host);
|
||||
// The offline flag and its recovery are the silent half of
|
||||
// rustfs/backlog#888: log the monitor's start and its success so an
|
||||
// "offline then back" episode leaves a trace on the observing node.
|
||||
@@ -567,34 +509,13 @@ impl PeerRestClient {
|
||||
grid_host = %self.grid_host,
|
||||
"peer RPC connection marked offline after a network-like failure; starting background recovery monitor"
|
||||
);
|
||||
drop(Self::spawn_recovery_monitor(grid_host, offline, recovery_running));
|
||||
}
|
||||
|
||||
fn spawn_recovery_monitor(
|
||||
grid_host: String,
|
||||
offline: Weak<AtomicBool>,
|
||||
recovery_running: Weak<AtomicBool>,
|
||||
) -> tokio::task::JoinHandle<()> {
|
||||
let span = Self::recovery_monitor_span(&grid_host);
|
||||
super::spawn_background_monitor(span, async move {
|
||||
let mut delay = get_drive_active_check_interval();
|
||||
let connect_timeout = get_drive_active_check_timeout();
|
||||
|
||||
for attempt in 1..=PEER_REST_RECOVERY_MAX_ATTEMPTS {
|
||||
if offline.strong_count() == 0 || recovery_running.strong_count() == 0 {
|
||||
return;
|
||||
}
|
||||
tokio::time::sleep(delay).await;
|
||||
if offline.strong_count() == 0 || recovery_running.strong_count() == 0 {
|
||||
return;
|
||||
}
|
||||
if Self::perform_connectivity_check(&grid_host, connect_timeout).await.is_ok() {
|
||||
let Some(offline) = offline.upgrade() else {
|
||||
return;
|
||||
};
|
||||
let Some(recovery_running) = recovery_running.upgrade() else {
|
||||
return;
|
||||
};
|
||||
offline.store(false, Ordering::Release);
|
||||
recovery_running.store(false, Ordering::Release);
|
||||
info!(
|
||||
@@ -614,10 +535,8 @@ impl PeerRestClient {
|
||||
attempts = PEER_REST_RECOVERY_MAX_ATTEMPTS,
|
||||
"peer recovery monitor reached max attempts; will retry on next request"
|
||||
);
|
||||
if let Some(recovery_running) = recovery_running.upgrade() {
|
||||
recovery_running.store(false, Ordering::Release);
|
||||
}
|
||||
})
|
||||
recovery_running.store(false, Ordering::Release);
|
||||
});
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -1235,15 +1154,6 @@ impl PeerRestClient {
|
||||
validate_heal_control_capability_proof(&canonical_ack, &proof)
|
||||
}
|
||||
|
||||
pub async fn probe_remote_version_state(&self, topology_fingerprint: String) -> Result<(String, Uuid)> {
|
||||
let probe = rustfs_protos::remote_version_state_capability_probe(Uuid::new_v4().as_bytes());
|
||||
let result = self
|
||||
.heal_control(rustfs_protos::HEAL_CONTROL_PROTOCOL_VERSION, topology_fingerprint, probe)
|
||||
.await?;
|
||||
let epoch = decode_remote_version_state_capability(&self.topology_member, &result)?;
|
||||
Ok((self.topology_member.clone(), epoch))
|
||||
}
|
||||
|
||||
pub async fn load_bucket_metadata(&self, bucket: &str, scanner_maintenance_change: bool) -> Result<()> {
|
||||
self.finalize_result(
|
||||
async {
|
||||
@@ -1867,13 +1777,9 @@ fn tier_config_reload_status_outcome(status: tonic::Status) -> TierConfigReloadO
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::config::com::STORAGE_CLASS_SUB_SYS;
|
||||
use crate::layout::{disks_layout::DisksLayout, endpoints::SetupType};
|
||||
use rustfs_config::{ENV_KUBERNETES_SERVICE_HOST, ENV_LOCAL_ENDPOINT_HOST, ENV_STARTUP_TOPOLOGY_WAIT_MODE};
|
||||
use serde_json::Value;
|
||||
use serial_test::serial;
|
||||
use std::io::{self, Write};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use temp_env::async_with_vars;
|
||||
use tracing_subscriber::{Registry, fmt::MakeWriter, layer::SubscriberExt};
|
||||
|
||||
#[test]
|
||||
@@ -1991,115 +1897,30 @@ mod tests {
|
||||
fn build_clients_from_slots_preserves_missing_remote_topology_slots() {
|
||||
let slots = vec![
|
||||
("127.0.0.1:9000".to_string(), None, true),
|
||||
(
|
||||
"rustfs-1.invalid:9001".to_string(),
|
||||
Some("http://rustfs-1.invalid:9001".to_string()),
|
||||
false,
|
||||
),
|
||||
("rustfs-2.invalid".to_string(), Some("http://rustfs-2.invalid".to_string()), false),
|
||||
("127.0.0.1:9001".to_string(), Some("http://127.0.0.1:9001".to_string()), false),
|
||||
("127.0.0.1:notaport".to_string(), Some("http://127.0.0.1:notaport".to_string()), false),
|
||||
("127.0.0.1:9003".to_string(), None, false),
|
||||
];
|
||||
|
||||
let (remote, all, remote_topology_hosts) = PeerRestClient::build_clients_from_slots(slots);
|
||||
|
||||
assert_eq!(remote.len(), 4, "local node is excluded but remote slots are not compacted away");
|
||||
assert_eq!(all.len(), 5, "all slots preserve the sorted cluster topology shape");
|
||||
assert_eq!(remote.len(), 3, "local node is excluded but remote slots are not compacted away");
|
||||
assert_eq!(all.len(), 4, "all slots preserve the sorted cluster topology shape");
|
||||
assert_eq!(
|
||||
remote_topology_hosts,
|
||||
vec![
|
||||
"rustfs-1.invalid:9001".to_string(),
|
||||
"rustfs-2.invalid".to_string(),
|
||||
"127.0.0.1:9001".to_string(),
|
||||
"127.0.0.1:notaport".to_string(),
|
||||
"127.0.0.1:9003".to_string()
|
||||
]
|
||||
);
|
||||
let unresolved = remote[0]
|
||||
.as_ref()
|
||||
.expect("temporarily unresolved remote peer should retain a client");
|
||||
assert_eq!(unresolved.host.to_string(), "rustfs-1.invalid:9001");
|
||||
let default_port = remote[1]
|
||||
.as_ref()
|
||||
.expect("temporarily unresolved scheme-default remote peer should retain a client");
|
||||
assert_eq!(default_port.host.to_string(), "rustfs-2.invalid");
|
||||
assert_eq!(default_port.host.port, 80);
|
||||
assert!(!default_port.host.is_port_set);
|
||||
assert!(remote[2].is_none(), "unparseable remote peer should remain observable as a missing slot");
|
||||
assert!(remote[3].is_none(), "missing grid host should remain observable as a missing slot");
|
||||
assert!(remote[0].is_some(), "valid remote peer should get a client");
|
||||
assert!(remote[1].is_none(), "unparseable remote peer should remain observable as a missing slot");
|
||||
assert!(remote[2].is_none(), "missing grid host should remain observable as a missing slot");
|
||||
assert!(all[0].is_none(), "local node is represented by the local server_info row");
|
||||
assert!(all[1].is_some());
|
||||
assert!(all[2].is_some());
|
||||
assert!(all[2].is_none());
|
||||
assert!(all[3].is_none());
|
||||
assert!(all[4].is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn topology_host_parser_preserves_names_and_bracketed_ipv6() {
|
||||
let domain = PeerRestClient::parse_topology_host("rustfs-1.invalid", "https://rustfs-1.invalid")
|
||||
.expect("unresolved HTTPS topology host should parse without DNS");
|
||||
assert_eq!(domain.to_string(), "rustfs-1.invalid");
|
||||
assert_eq!(domain.port, 443);
|
||||
assert!(!domain.is_port_set);
|
||||
|
||||
let ipv6 = PeerRestClient::parse_topology_host("[2001:db8::1]:9000", "http://[2001:db8::1]:9000")
|
||||
.expect("bracketed IPv6 topology host should parse without changing its identity");
|
||||
assert_eq!(ipv6.to_string(), "[2001:db8::1]:9000");
|
||||
|
||||
let default_port_ipv6 = PeerRestClient::parse_topology_host("[2001:db8::2]", "http://[2001:db8::2]")
|
||||
.expect("scheme-default IPv6 topology host should parse without DNS");
|
||||
assert_eq!(default_port_ipv6.to_string(), "[2001:db8::2]");
|
||||
assert_eq!(default_port_ipv6.port, 80);
|
||||
assert!(!default_port_ipv6.is_port_set);
|
||||
|
||||
assert!(PeerRestClient::parse_topology_host("peer.invalid:0", "http://peer.invalid:0").is_err());
|
||||
assert!(PeerRestClient::parse_topology_host("peer-a.invalid:9000", "http://peer-b.invalid:9000").is_err());
|
||||
assert!(PeerRestClient::parse_topology_host("peer.invalid:9000", "http://peer.invalid:9000/unexpected").is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn unresolved_default_port_endpoint_topology_retains_all_peer_clients() {
|
||||
let volumes = (0..4)
|
||||
.map(|index| format!("http://rustfs-{index}.invalid:80/data{index}"))
|
||||
.collect::<Vec<_>>();
|
||||
let layout = DisksLayout::from_volumes(&volumes).expect("distributed default-port topology should parse");
|
||||
|
||||
async_with_vars(
|
||||
[
|
||||
(ENV_STARTUP_TOPOLOGY_WAIT_MODE, Some("orchestrated")),
|
||||
(ENV_LOCAL_ENDPOINT_HOST, Some("rustfs-0.invalid")),
|
||||
(ENV_KUBERNETES_SERVICE_HOST, None),
|
||||
],
|
||||
async {
|
||||
let (server_pools, setup_type) = EndpointServerPools::create_server_endpoints("0.0.0.0:80", &layout)
|
||||
.await
|
||||
.expect("explicit local identity should avoid peer DNS during endpoint construction");
|
||||
assert_eq!(setup_type, SetupType::DistErasure);
|
||||
|
||||
let (remote, all, remote_topology_hosts) =
|
||||
PeerRestClient::build_clients_from_slots(server_pools.peer_grid_host_slots_sorted());
|
||||
assert_eq!(remote.len(), 3);
|
||||
assert!(
|
||||
remote.iter().all(Option::is_some),
|
||||
"unresolved remote peers must retain reconnectable clients"
|
||||
);
|
||||
assert_eq!(all.len(), 4);
|
||||
assert_eq!(all.iter().filter(|client| client.is_none()).count(), 1);
|
||||
assert_eq!(remote_topology_hosts.len(), 3);
|
||||
assert!(
|
||||
remote_topology_hosts.iter().all(|host| !host.contains(':')),
|
||||
"scheme-default ports must preserve the legacy topology identity"
|
||||
);
|
||||
assert!(
|
||||
remote
|
||||
.iter()
|
||||
.flatten()
|
||||
.all(|client| client.host.port == 80 && !client.host.is_port_set),
|
||||
"scheme-default peers must retain the effective dial port"
|
||||
);
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -2649,22 +2470,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_version_state_capability_decoder_fails_closed() {
|
||||
let epoch = Uuid::new_v4();
|
||||
let result = rustfs_protos::encode_remote_version_state_capability("node-a:9000", epoch.as_bytes())
|
||||
.expect("small capability response should encode");
|
||||
assert_eq!(
|
||||
decode_remote_version_state_capability("node-a:9000", &result).expect("valid epoch should decode"),
|
||||
epoch
|
||||
);
|
||||
assert!(decode_remote_version_state_capability("node-b:9000", &result).is_err());
|
||||
assert!(decode_remote_version_state_capability("node-a:9000", &result[..result.len() - 1]).is_err());
|
||||
let nil = rustfs_protos::encode_remote_version_state_capability("node-a:9000", Uuid::nil().as_bytes())
|
||||
.expect("small capability response should encode");
|
||||
assert!(decode_remote_version_state_capability("node-a:9000", &nil).is_err());
|
||||
}
|
||||
|
||||
struct TierMutationResponseFixture<'a> {
|
||||
version: u32,
|
||||
phase: TierMutationRpcPhase,
|
||||
@@ -2889,31 +2694,6 @@ mod tests {
|
||||
assert!(!client.offline.load(Ordering::Acquire));
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn dropped_peer_client_releases_and_stops_its_recovery_monitor() {
|
||||
let client = test_peer_client();
|
||||
client.offline.store(true, Ordering::Release);
|
||||
client.recovery_running.store(true, Ordering::Release);
|
||||
let offline = Arc::downgrade(&client.offline);
|
||||
let recovery_running = Arc::downgrade(&client.recovery_running);
|
||||
let handle = PeerRestClient::spawn_recovery_monitor(client.grid_host.clone(), offline.clone(), recovery_running.clone());
|
||||
let started = tokio::time::Instant::now();
|
||||
|
||||
drop(client);
|
||||
|
||||
assert!(offline.upgrade().is_none(), "detached recovery must not retain offline state");
|
||||
assert!(
|
||||
recovery_running.upgrade().is_none(),
|
||||
"detached recovery must not retain its running state"
|
||||
);
|
||||
handle.await.expect("recovery monitor should not panic");
|
||||
assert_eq!(
|
||||
tokio::time::Instant::now(),
|
||||
started,
|
||||
"recovery monitor should stop before advancing to its first delayed probe"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn peer_rest_client_finalize_result_keeps_online_for_app_errors_mentioning_unavailable() {
|
||||
// Regression: application error text containing "unavailable" (a
|
||||
@@ -2961,11 +2741,6 @@ mod tests {
|
||||
.with_span_list(true),
|
||||
);
|
||||
let _guard = tracing::subscriber::set_default(subscriber);
|
||||
// The `recovery-monitor` callsite is shared with the production
|
||||
// `mark_offline_and_spawn_recovery` path that sibling tests exercise from
|
||||
// subscriber-less threads; without this the span can be cached as
|
||||
// `Interest::never()` and silently degrade to `Span::none()`.
|
||||
let _callsite_pin = crate::test_tracing::pin_callsite_interest_for_test();
|
||||
|
||||
let client = test_peer_client();
|
||||
let span = tracing::info_span!("request-span", request_id = "req-peer-rest");
|
||||
|
||||
@@ -14,8 +14,7 @@
|
||||
|
||||
use crate::bucket::metadata_sys;
|
||||
use crate::cluster::rpc::client::{
|
||||
AuthenticatedChannel, TonicInterceptor, gen_tonic_signature_interceptor, is_network_like_disk_error,
|
||||
node_service_time_out_client,
|
||||
TonicInterceptor, gen_tonic_signature_interceptor, is_network_like_disk_error, node_service_time_out_client,
|
||||
};
|
||||
use crate::cluster::rpc::set_tonic_mutation_body_digest;
|
||||
use crate::disk::error::DiskError;
|
||||
@@ -41,84 +40,16 @@ use rustfs_protos::proto_gen::node_service::node_service_client::NodeServiceClie
|
||||
use rustfs_protos::proto_gen::node_service::{
|
||||
DeleteBucketRequest, GetBucketInfoRequest, HealBucketRequest, ListBucketRequest, MakeBucketRequest,
|
||||
};
|
||||
#[cfg(test)]
|
||||
use std::sync::{
|
||||
Mutex as StdMutex,
|
||||
atomic::{AtomicBool, Ordering},
|
||||
};
|
||||
use std::{collections::HashMap, fmt::Debug, sync::Arc, time::Duration};
|
||||
#[cfg(test)]
|
||||
use tokio::sync::Notify;
|
||||
use tokio::{net::TcpStream, sync::RwLock, time};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tonic::Request;
|
||||
use tonic::service::interceptor::InterceptedService;
|
||||
use tonic::transport::Channel;
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
type Client = Arc<Box<dyn PeerS3Client>>;
|
||||
|
||||
#[cfg(test)]
|
||||
#[derive(Default)]
|
||||
pub(crate) struct DeleteBucketEmptyScanBarrier {
|
||||
arrived: AtomicBool,
|
||||
arrived_notify: Notify,
|
||||
released: AtomicBool,
|
||||
release_notify: Notify,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl DeleteBucketEmptyScanBarrier {
|
||||
pub(crate) async fn wait_until_paused(&self) {
|
||||
loop {
|
||||
let notified = self.arrived_notify.notified();
|
||||
if self.arrived.load(Ordering::Acquire) {
|
||||
return;
|
||||
}
|
||||
notified.await;
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn release(&self) {
|
||||
self.released.store(true, Ordering::Release);
|
||||
self.release_notify.notify_waiters();
|
||||
}
|
||||
|
||||
async fn pause(&self) {
|
||||
self.arrived.store(true, Ordering::Release);
|
||||
self.arrived_notify.notify_waiters();
|
||||
loop {
|
||||
let notified = self.release_notify.notified();
|
||||
if self.released.load(Ordering::Acquire) {
|
||||
return;
|
||||
}
|
||||
notified.await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
static DELETE_BUCKET_EMPTY_SCAN_BARRIER: StdMutex<Option<Arc<DeleteBucketEmptyScanBarrier>>> = StdMutex::new(None);
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn install_delete_bucket_empty_scan_barrier() -> Arc<DeleteBucketEmptyScanBarrier> {
|
||||
let barrier = Arc::new(DeleteBucketEmptyScanBarrier::default());
|
||||
*DELETE_BUCKET_EMPTY_SCAN_BARRIER
|
||||
.lock()
|
||||
.expect("empty scan barrier lock should not be poisoned") = Some(barrier.clone());
|
||||
barrier
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
async fn pause_after_delete_bucket_empty_scan() {
|
||||
let barrier = DELETE_BUCKET_EMPTY_SCAN_BARRIER
|
||||
.lock()
|
||||
.expect("empty scan barrier lock should not be poisoned")
|
||||
.take();
|
||||
if let Some(barrier) = barrier {
|
||||
barrier.pause().await;
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct ScannerBucketListing {
|
||||
pub buckets: Vec<BucketInfo>,
|
||||
@@ -719,22 +650,24 @@ impl PeerS3Client for LocalPeerS3Client {
|
||||
return Err(Error::ErasureWriteQuorum);
|
||||
}
|
||||
|
||||
if opts.force_if_empty && !opts.force {
|
||||
let force = if opts.force_if_empty && !opts.force {
|
||||
for disk in local_disks.iter() {
|
||||
if has_xlmeta_files(&disk.path().join(bucket)).await.map_err(Error::Io)? {
|
||||
return Err(Error::VolumeNotEmpty);
|
||||
}
|
||||
}
|
||||
#[cfg(test)]
|
||||
pause_after_delete_bucket_empty_scan().await;
|
||||
}
|
||||
true
|
||||
} else {
|
||||
opts.force
|
||||
};
|
||||
|
||||
let mut futures = Vec::with_capacity(local_disks.len());
|
||||
|
||||
for disk in local_disks.iter() {
|
||||
// `force_if_empty` is validation-only. Passing it as force would let
|
||||
// a PutObject committed after the scan be removed recursively.
|
||||
futures.push(disk.delete_volume(bucket, opts.force));
|
||||
// Non-force delete refuses a non-empty bucket (VolumeNotEmpty), which
|
||||
// the recreate loop below turns into BucketNotEmpty; only an explicit
|
||||
// force delete removes recursively (backlog#799 B1).
|
||||
futures.push(disk.delete_volume(bucket, force));
|
||||
}
|
||||
|
||||
let results = join_all(futures).await;
|
||||
@@ -797,15 +730,6 @@ pub struct RemotePeerS3Client {
|
||||
}
|
||||
|
||||
impl RemotePeerS3Client {
|
||||
fn encode_delete_bucket_options(opts: &DeleteBucketOptions) -> Result<String> {
|
||||
let mut remote_opts = opts.clone();
|
||||
// Older peers promote `force_if_empty` to recursive force after their
|
||||
// metadata scan. Keep this coordinator-only hint off the wire so a
|
||||
// mixed-version delete fails closed on non-empty directory remnants.
|
||||
remote_opts.force_if_empty = false;
|
||||
serde_json::to_string(&remote_opts).map_err(Into::into)
|
||||
}
|
||||
|
||||
fn recovery_monitor_span(addr: &str) -> tracing::Span {
|
||||
tracing::info_span!(
|
||||
"recovery-monitor",
|
||||
@@ -832,7 +756,7 @@ impl RemotePeerS3Client {
|
||||
client
|
||||
}
|
||||
|
||||
pub async fn get_client(&self) -> Result<NodeServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>> {
|
||||
pub async fn get_client(&self) -> Result<NodeServiceClient<InterceptedService<Channel, TonicInterceptor>>> {
|
||||
node_service_time_out_client(&self.addr, TonicInterceptor::Signature(gen_tonic_signature_interceptor()))
|
||||
.await
|
||||
.map_err(|err| Error::other(format!("can not get client, err: {err}")))
|
||||
@@ -1116,7 +1040,7 @@ impl PeerS3Client for RemotePeerS3Client {
|
||||
async fn delete_bucket(&self, bucket: &str, opts: &DeleteBucketOptions) -> Result<()> {
|
||||
self.execute_with_timeout(
|
||||
|| async {
|
||||
let options = Self::encode_delete_bucket_options(opts)?;
|
||||
let options = serde_json::to_string(opts)?;
|
||||
let mut client = self.get_client().await?;
|
||||
|
||||
let mut request = Request::new(DeleteBucketRequest {
|
||||
@@ -1490,49 +1414,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_delete_bucket_options_fail_closed_for_legacy_peers() {
|
||||
let encoded = RemotePeerS3Client::encode_delete_bucket_options(&DeleteBucketOptions {
|
||||
no_lock: true,
|
||||
no_recreate: true,
|
||||
force_if_empty: true,
|
||||
..Default::default()
|
||||
})
|
||||
.expect("remote delete options should serialize");
|
||||
let legacy_opts: DeleteBucketOptions =
|
||||
serde_json::from_str(&encoded).expect("legacy peer should decode remote delete options");
|
||||
|
||||
assert!(legacy_opts.no_lock);
|
||||
assert!(legacy_opts.no_recreate);
|
||||
assert!(!legacy_opts.force);
|
||||
assert!(!legacy_opts.force_if_empty);
|
||||
|
||||
let legacy_recursive_force = if legacy_opts.force_if_empty && !legacy_opts.force {
|
||||
true
|
||||
} else {
|
||||
legacy_opts.force
|
||||
};
|
||||
assert!(
|
||||
!legacy_recursive_force,
|
||||
"legacy peer must not upgrade empty-only delete to recursive force"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_delete_bucket_options_preserve_explicit_force() {
|
||||
let encoded = RemotePeerS3Client::encode_delete_bucket_options(&DeleteBucketOptions {
|
||||
force: true,
|
||||
force_if_empty: true,
|
||||
..Default::default()
|
||||
})
|
||||
.expect("remote force-delete options should serialize");
|
||||
let remote_opts: DeleteBucketOptions =
|
||||
serde_json::from_str(&encoded).expect("remote peer should decode force-delete options");
|
||||
|
||||
assert!(remote_opts.force);
|
||||
assert!(!remote_opts.force_if_empty);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_execute_with_timeout_marks_remote_peer_faulty_on_network_like_error() {
|
||||
let client = test_remote_peer("http://peer-network-error:9000");
|
||||
@@ -1690,54 +1571,6 @@ mod tests {
|
||||
reset_local_disk_test_state().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn local_peer_force_if_empty_preserves_unclassified_file_in_selected_pool() {
|
||||
reset_local_disk_test_state().await;
|
||||
|
||||
let temp_dir = TempDir::new().expect("create temp dir for empty-only delete regression");
|
||||
let disks = init_test_local_disks_for_pools(
|
||||
&temp_dir,
|
||||
&[(0, 1), (1, 1)],
|
||||
"local-peer-force-if-empty-preserves-unclassified-file",
|
||||
)
|
||||
.await;
|
||||
let bucket = "empty-only-delete-bucket";
|
||||
let marker = "object/commit-marker";
|
||||
let data = bytes::Bytes::from_static(b"committed object data");
|
||||
|
||||
disks[1]
|
||||
.make_volume(bucket)
|
||||
.await
|
||||
.expect("bucket should be created in the selected pool");
|
||||
disks[1]
|
||||
.write_all(bucket, marker, data.clone())
|
||||
.await
|
||||
.expect("unclassified committed file should be written");
|
||||
|
||||
let err = LocalPeerS3Client::new_with_local_disks(None, Some(vec![1]), disks.clone())
|
||||
.delete_bucket(
|
||||
bucket,
|
||||
&DeleteBucketOptions {
|
||||
force_if_empty: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect_err("empty-only delete must not recursively remove an unclassified file");
|
||||
|
||||
assert_eq!(err, Error::VolumeNotEmpty);
|
||||
assert_eq!(
|
||||
disks[1]
|
||||
.read_all(bucket, marker)
|
||||
.await
|
||||
.expect("unclassified committed file should be preserved"),
|
||||
data
|
||||
);
|
||||
|
||||
reset_local_disk_test_state().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn heal_bucket_local_recreates_missing_bucket_volumes() {
|
||||
|
||||
@@ -13,8 +13,8 @@
|
||||
// limitations under the License.
|
||||
|
||||
use crate::cluster::rpc::client::{
|
||||
AuthenticatedChannel, TonicInterceptor, gen_tonic_signature_interceptor, is_network_like_disk_error,
|
||||
node_service_time_out_client, node_service_time_out_client_for_class, node_service_time_out_client_no_auth,
|
||||
TonicInterceptor, gen_tonic_signature_interceptor, is_network_like_disk_error, node_service_time_out_client,
|
||||
node_service_time_out_client_for_class, node_service_time_out_client_no_auth,
|
||||
};
|
||||
use crate::cluster::rpc::http_auth::set_tonic_canonical_body_digest;
|
||||
use crate::cluster::rpc::internode_data_transport::{
|
||||
@@ -71,7 +71,7 @@ use tokio::{
|
||||
time::timeout,
|
||||
};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tonic::{Code, Request, service::interceptor::InterceptedService};
|
||||
use tonic::{Code, Request, service::interceptor::InterceptedService, transport::Channel};
|
||||
use tracing::{debug, trace, warn};
|
||||
use uuid::Uuid;
|
||||
|
||||
@@ -1083,7 +1083,7 @@ impl RemoteDisk {
|
||||
internode_offline_bypass_reason(&self.addr).map(Error::other)
|
||||
}
|
||||
|
||||
async fn get_client(&self) -> Result<NodeServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>> {
|
||||
async fn get_client(&self) -> Result<NodeServiceClient<InterceptedService<Channel, TonicInterceptor>>> {
|
||||
if let Some(err) = self.offline_bypass_error() {
|
||||
return Err(err);
|
||||
}
|
||||
@@ -1096,7 +1096,7 @@ impl RemoteDisk {
|
||||
/// Routes onto the isolated bulk channel pool so large transfers cannot head-of-line block
|
||||
/// lock/health RPCs (grpc-optimization P1). Falls back to the control channel when isolation
|
||||
/// is disabled.
|
||||
async fn get_bulk_client(&self) -> Result<NodeServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>> {
|
||||
async fn get_bulk_client(&self) -> Result<NodeServiceClient<InterceptedService<Channel, TonicInterceptor>>> {
|
||||
if let Some(err) = self.offline_bypass_error() {
|
||||
return Err(err);
|
||||
}
|
||||
@@ -3005,7 +3005,6 @@ mod tests {
|
||||
use crate::cluster::rpc::internode_data_transport::{InternodeDataTransportCapabilities, TcpHttpInternodeDataTransport};
|
||||
use crate::runtime::sources as runtime_sources;
|
||||
use serde_json::Value;
|
||||
use serial_test::serial;
|
||||
use std::io::{self as std_io, Write};
|
||||
use std::pin::Pin;
|
||||
use std::sync::{Arc, Mutex, Mutex as StdMutex, Once};
|
||||
@@ -3019,20 +3018,6 @@ mod tests {
|
||||
|
||||
static INIT: Once = Once::new();
|
||||
|
||||
// `#[serial(internode_metrics)]` marks every test that observes
|
||||
// `global_internode_metrics()`. Those counters are a process-wide singleton:
|
||||
// some of these tests snapshot a counter, run one decode, and assert on the
|
||||
// delta, while others deliberately record decode errors or call
|
||||
// `reset_internode_metrics_for_test()`. Run concurrently in one process they
|
||||
// corrupt each other's deltas — a sibling's error bumps the "no decode error"
|
||||
// assertion off zero, and a sibling's reset can drive an `after > before`
|
||||
// assertion backwards.
|
||||
//
|
||||
// The marker only takes effect under the `cargo test` fallback; nextest
|
||||
// already isolates each test in its own process, so every test there gets its
|
||||
// own copy of the counters (see `docs/testing/README.md`). Any new test that
|
||||
// reads or mutates the global internode metrics belongs in this group.
|
||||
|
||||
#[test]
|
||||
fn snapshot_lease_response_requires_current_protocol_and_valid_token() {
|
||||
let token = SnapshotLeaseToken::new();
|
||||
@@ -3321,7 +3306,6 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial(internode_metrics)]
|
||||
fn read_multiple_response_decode_prefers_msgpack_payloads() {
|
||||
let endpoint = sample_remote_endpoint();
|
||||
let msgpack_resp = sample_read_multiple_resp("msgpack", b"binary");
|
||||
@@ -3344,7 +3328,6 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial(internode_metrics)]
|
||||
fn read_multiple_response_decode_falls_back_to_json_payloads() {
|
||||
let endpoint = sample_remote_endpoint();
|
||||
let json_resp = sample_read_multiple_resp("json", b"fallback");
|
||||
@@ -3366,7 +3349,6 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial(internode_metrics)]
|
||||
fn rename_data_response_accepts_legacy_json_without_decode_error() {
|
||||
crate::cluster::rpc::runtime_sources::reset_internode_metrics_for_test();
|
||||
let response = RenameDataResp {
|
||||
@@ -3518,7 +3500,6 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial(internode_metrics)]
|
||||
fn read_multiple_response_decode_reports_corrupt_msgpack_item() {
|
||||
let endpoint = sample_remote_endpoint();
|
||||
let response = ReadMultipleResponse {
|
||||
@@ -3544,7 +3525,6 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial(internode_metrics)]
|
||||
fn read_multiple_response_decode_reports_corrupt_json_item() {
|
||||
let endpoint = sample_remote_endpoint();
|
||||
let response = ReadMultipleResponse {
|
||||
@@ -3585,7 +3565,6 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial(internode_metrics)]
|
||||
fn batch_read_version_response_decode_prefers_msgpack_payloads() {
|
||||
let endpoint = sample_remote_endpoint();
|
||||
let msgpack_resp = sample_batch_read_version_resp(7, "msgpack-object", true);
|
||||
@@ -3606,7 +3585,6 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial(internode_metrics)]
|
||||
fn batch_read_version_response_rejects_invalid_success_metadata() {
|
||||
let endpoint = sample_remote_endpoint();
|
||||
let mut response_item = sample_batch_read_version_resp(0, "invalid-object", true);
|
||||
@@ -3626,7 +3604,6 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial(internode_metrics)]
|
||||
fn batch_read_version_response_decode_reports_corrupt_msgpack_item() {
|
||||
let endpoint = sample_remote_endpoint();
|
||||
let response = BatchReadVersionResponse {
|
||||
@@ -3653,7 +3630,6 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial(internode_metrics)]
|
||||
fn batch_read_version_response_decode_reports_corrupt_json_item() {
|
||||
let endpoint = sample_remote_endpoint();
|
||||
let response = BatchReadVersionResponse {
|
||||
@@ -4443,7 +4419,6 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial(internode_metrics)]
|
||||
async fn test_remote_disk_create_file_retries_once_on_retryable_open_write_error() {
|
||||
let transport = RetryingOpenWriteInternodeDataTransport::with_steps(vec![
|
||||
OpenWriteTestStep::Error(DiskError::from(rustfs_rio::new_test_internode_http_io_error(
|
||||
@@ -4482,7 +4457,6 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial(internode_metrics)]
|
||||
async fn test_remote_disk_read_file_stream_retries_once_on_retryable_open_read_error() {
|
||||
// A transient reset-by-peer on a shard read during the read-after-write window must be
|
||||
// absorbed by one re-dial rather than eroding read quorum (issue #2761).
|
||||
@@ -4814,11 +4788,9 @@ mod tests {
|
||||
async fn test_remote_disk_endpoints_with_different_schemes() {
|
||||
let test_cases = vec![
|
||||
("http://server:9000", "server:9000"),
|
||||
("http://plain-server:80", "plain-server"),
|
||||
("http://plain-server", "plain-server"),
|
||||
("https://secure-server:443", "secure-server"),
|
||||
("https://secure-server:443", "secure-server"), // Default HTTPS port is omitted
|
||||
("http://192.168.1.100:8080", "192.168.1.100:8080"),
|
||||
("https://secure-server", "secure-server"),
|
||||
("https://secure-server", "secure-server"), // No port specified
|
||||
];
|
||||
|
||||
for (url_str, expected_hostname) in test_cases {
|
||||
@@ -5334,11 +5306,6 @@ mod tests {
|
||||
.with_span_list(true),
|
||||
);
|
||||
let _guard = tracing::subscriber::set_default(subscriber);
|
||||
// The `recovery-monitor` span and the monitor's own log events are
|
||||
// production callsites that sibling tests exercise from subscriber-less
|
||||
// threads; without this they can be cached as `Interest::never()` and go
|
||||
// silently missing here.
|
||||
let _callsite_pin = crate::test_tracing::pin_callsite_interest_for_test();
|
||||
|
||||
let endpoint = Endpoint {
|
||||
url: url::Url::parse("http://127.0.0.1:59996/data").expect("endpoint URL should parse"),
|
||||
@@ -5393,11 +5360,6 @@ mod tests {
|
||||
.with_span_list(true),
|
||||
);
|
||||
let _guard = tracing::subscriber::set_default(subscriber);
|
||||
// The `recovery-monitor` span and the monitor's own log events are
|
||||
// production callsites that sibling tests exercise from subscriber-less
|
||||
// threads; without this they can be cached as `Interest::never()` and go
|
||||
// silently missing here.
|
||||
let _callsite_pin = crate::test_tracing::pin_callsite_interest_for_test();
|
||||
|
||||
let addr = "http://127.0.0.1:59997".to_string();
|
||||
let endpoint = Endpoint {
|
||||
|
||||
@@ -12,9 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::cluster::rpc::client::{
|
||||
AuthenticatedChannel, TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client,
|
||||
};
|
||||
use crate::cluster::rpc::client::{TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client};
|
||||
use crate::cluster::rpc::set_tonic_mutation_body_digest;
|
||||
use async_trait::async_trait;
|
||||
use bytes::Bytes;
|
||||
@@ -31,6 +29,7 @@ use std::time::Duration;
|
||||
use tokio::time::timeout;
|
||||
use tonic::Request;
|
||||
use tonic::service::interceptor::InterceptedService;
|
||||
use tonic::transport::Channel;
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
/// Remote lock client implementation
|
||||
@@ -79,7 +78,7 @@ impl RemoteClient {
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn get_client(&self) -> Result<NodeServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>> {
|
||||
pub async fn get_client(&self) -> Result<NodeServiceClient<InterceptedService<Channel, TonicInterceptor>>> {
|
||||
// P3-2 offline bypass (now covering the lock path too): fast-fail a peer already marked
|
||||
// offline instead of paying the connect timeout, so dsync reaches quorum sooner. Does not
|
||||
// change quorum; the self-healing re-probe keeps the peer recoverable.
|
||||
|
||||
+20
-199
@@ -37,9 +37,7 @@ use crate::{
|
||||
runtime::instance::{InstanceContext, bootstrap_ctx},
|
||||
runtime::sources as runtime_sources,
|
||||
set_disk::{PreparedGetObjectMetadata, SetDisks},
|
||||
store::init_format::{
|
||||
check_format_erasure_values, load_format_erasure_all, save_format_file, select_format_erasure_in_quorum,
|
||||
},
|
||||
store::init_format::{check_format_erasure_values, get_format_erasure_in_quorum, load_format_erasure_all, save_format_file},
|
||||
};
|
||||
use futures::{
|
||||
future::join_all,
|
||||
@@ -949,7 +947,7 @@ impl crate::storage_api_contracts::heal::HealOperations for Sets {
|
||||
|
||||
#[tracing::instrument(skip(self))]
|
||||
async fn heal_format(&self, dry_run: bool) -> Result<(HealResultItem, Option<Error>)> {
|
||||
let (disks, init_errs) = init_storage_disks_with_errors(
|
||||
let (disks, _) = init_storage_disks_with_errors(
|
||||
&self.endpoints.endpoints,
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
@@ -957,36 +955,15 @@ impl crate::storage_api_contracts::heal::HealOperations for Sets {
|
||||
},
|
||||
)
|
||||
.await;
|
||||
let (formats, mut errs) = load_format_erasure_all(&disks, true).await;
|
||||
for (err, init_err) in errs.iter_mut().zip(init_errs) {
|
||||
if init_err.is_some() {
|
||||
*err = init_err;
|
||||
}
|
||||
}
|
||||
if errs.iter().any(|err| {
|
||||
matches!(
|
||||
err,
|
||||
Some(DiskError::InconsistentDisk | DiskError::CorruptedFormat | DiskError::CorruptedBackend)
|
||||
)
|
||||
}) {
|
||||
return Ok((HealResultItem::default(), Some(StorageError::CorruptedFormat)));
|
||||
}
|
||||
let (formats, errs) = load_format_erasure_all(&disks, true).await;
|
||||
if let Err(err) = check_format_erasure_values(&formats, self.set_drive_count) {
|
||||
info!("failed to check formats erasure values: {}", err);
|
||||
return Ok((HealResultItem::default(), Some(err)));
|
||||
}
|
||||
let (ref_format, quorum_members) = match select_format_erasure_in_quorum(&formats, 0) {
|
||||
Ok((format, members)) if format.shared_identity() == self.format.shared_identity() => (format, members),
|
||||
Ok(_) => return Ok((HealResultItem::default(), Some(StorageError::CorruptedFormat))),
|
||||
let ref_format = match get_format_erasure_in_quorum(&formats) {
|
||||
Ok(format) => format,
|
||||
Err(err) => return Ok((HealResultItem::default(), Some(err))),
|
||||
};
|
||||
if formats
|
||||
.iter()
|
||||
.zip(quorum_members)
|
||||
.any(|(format, member)| format.is_some() && !member)
|
||||
{
|
||||
return Ok((HealResultItem::default(), Some(StorageError::CorruptedFormat)));
|
||||
}
|
||||
let mut res = HealResultItem {
|
||||
heal_item_type: HealItemType::Metadata.to_string(),
|
||||
detail: "disk-format".to_string(),
|
||||
@@ -1008,6 +985,11 @@ impl crate::storage_api_contracts::heal::HealOperations for Sets {
|
||||
return Ok((res, Some(StorageError::NoHealRequired)));
|
||||
}
|
||||
|
||||
// if !self.format.eq(&ref_format) {
|
||||
// info!("format ({:?}) not eq ref_format ({:?})", self.format, ref_format);
|
||||
// return Ok((res, Some(Error::new(DiskError::CorruptedFormat))));
|
||||
// }
|
||||
|
||||
let (new_format_sets, _) = new_heal_format_sets(&ref_format, self.set_count, self.set_drive_count, &formats, &errs);
|
||||
if !dry_run {
|
||||
let mut tmp_new_formats = vec![None; self.set_count * self.set_drive_count];
|
||||
@@ -1316,7 +1298,7 @@ mod tests {
|
||||
assert_eq!(result, (Some(3), Some(1), Some(0)));
|
||||
}
|
||||
|
||||
async fn two_set_test_sets() -> (Vec<tempfile::TempDir>, Arc<Sets>) {
|
||||
async fn multipart_listing_test_sets() -> (Vec<tempfile::TempDir>, Arc<Sets>) {
|
||||
let format = FormatV3::new(2, 2);
|
||||
let mut temp_dirs = Vec::new();
|
||||
let mut all_endpoints = Vec::new();
|
||||
@@ -1357,8 +1339,8 @@ mod tests {
|
||||
Arc::new(RwLock::new(disks)),
|
||||
2,
|
||||
1,
|
||||
set_index,
|
||||
0,
|
||||
set_index,
|
||||
endpoints,
|
||||
format.clone(),
|
||||
vec![Arc::new(LocalClient::new()), Arc::new(LocalClient::new())],
|
||||
@@ -1391,114 +1373,11 @@ mod tests {
|
||||
(temp_dirs, sets)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn set_format_heal_accepts_quorum_from_a_nonzero_set() {
|
||||
let (_temp_dirs, sets) = two_set_test_sets().await;
|
||||
|
||||
let (result, err) = sets.disk_set[1]
|
||||
.heal_format(false)
|
||||
.await
|
||||
.expect("the second erasure set should load its own format quorum");
|
||||
|
||||
assert!(matches!(err, Some(StorageError::NoHealRequired)), "unexpected heal result: {err:?}");
|
||||
assert_eq!(result.disk_count, 2);
|
||||
assert_eq!(result.set_count, 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn format_heal_rejects_foreign_majorities_at_set_and_pool_scopes() {
|
||||
let (_temp_dirs, _canonical_format, sets) = setup_heal_format_sets(2, true).await;
|
||||
let set_disks = set_level_heal_view(&sets).await;
|
||||
|
||||
let (_, set_err) = set_disks
|
||||
.heal_format(false)
|
||||
.await
|
||||
.expect("set format heal should report a typed mismatch");
|
||||
assert!(
|
||||
matches!(set_err, Some(StorageError::CorruptedFormat)),
|
||||
"foreign set majority must not replace the cached format: {set_err:?}"
|
||||
);
|
||||
|
||||
let (_, pool_err) = sets
|
||||
.heal_format(false)
|
||||
.await
|
||||
.expect("pool format heal should report a typed mismatch");
|
||||
assert!(
|
||||
matches!(pool_err, Some(StorageError::CorruptedFormat)),
|
||||
"foreign pool majority must not replace the cached format: {pool_err:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn pool_format_heal_rejects_a_wrong_slot_minority() {
|
||||
let (_temp_dirs, canonical_format, sets) = setup_heal_format_sets(3, false).await;
|
||||
let mut poisoned_format = canonical_format.clone();
|
||||
poisoned_format.erasure.this = canonical_format.erasure.sets[0][0];
|
||||
replace_heal_test_format(&sets, 2, &poisoned_format).await;
|
||||
let probe_err = new_disk(
|
||||
&sets.endpoints.endpoints.as_ref()[2],
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect_err("a wrong-slot local format must fail disk initialization");
|
||||
assert_eq!(probe_err, DiskError::InconsistentDisk);
|
||||
|
||||
let (_, pool_err) = sets
|
||||
.heal_format(false)
|
||||
.await
|
||||
.expect("pool format heal should report a typed slot mismatch");
|
||||
assert!(
|
||||
matches!(pool_err, Some(StorageError::CorruptedFormat)),
|
||||
"a wrong-slot minority must not be reported as no-heal-required: {pool_err:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
read_heal_test_format(&sets, 2).await,
|
||||
poisoned_format,
|
||||
"format heal must not overwrite a wrong-slot disk"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn format_heal_rejects_a_foreign_minority_at_set_and_pool_scopes() {
|
||||
let (_temp_dirs, canonical_format, sets) = setup_heal_format_sets(3, false).await;
|
||||
let mut poisoned_format = canonical_format.clone();
|
||||
poisoned_format.id = Uuid::new_v4();
|
||||
poisoned_format.erasure.this = poisoned_format.erasure.sets[0][2];
|
||||
replace_heal_test_format(&sets, 2, &poisoned_format).await;
|
||||
let set_disks = set_level_heal_view(&sets).await;
|
||||
|
||||
let (_, set_err) = set_disks
|
||||
.heal_format(false)
|
||||
.await
|
||||
.expect("set format heal should report a typed identity mismatch");
|
||||
assert!(
|
||||
matches!(set_err, Some(StorageError::CorruptedFormat)),
|
||||
"a foreign minority must not be reported as no-heal-required: {set_err:?}"
|
||||
);
|
||||
|
||||
let (_, pool_err) = sets
|
||||
.heal_format(false)
|
||||
.await
|
||||
.expect("pool format heal should report a typed identity mismatch");
|
||||
assert!(
|
||||
matches!(pool_err, Some(StorageError::CorruptedFormat)),
|
||||
"a foreign minority must not be reported as no-heal-required: {pool_err:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
read_heal_test_format(&sets, 2).await,
|
||||
poisoned_format,
|
||||
"format heal must not overwrite a foreign disk"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn list_multipart_uploads_merges_all_sets_without_pagination_loss() {
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::Erasure).await;
|
||||
let (_temp_dirs, sets) = two_set_test_sets().await;
|
||||
let (_temp_dirs, sets) = multipart_listing_test_sets().await;
|
||||
let bucket = format!("multipart-list-{}", Uuid::new_v4().simple());
|
||||
sets.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
@@ -1737,15 +1616,11 @@ mod tests {
|
||||
// formatting the first `num_formatted` of them against a shared reference
|
||||
// format and leaving the rest unformatted. Returns the live TempDir handles
|
||||
// (must be kept alive), the reference format, and the assembled `Sets`.
|
||||
// `disk_set` is intentionally empty: these tests only exercise paths that
|
||||
// return before pool-level healing delegates into a set.
|
||||
async fn setup_heal_format_sets(num_formatted: usize, foreign_identity: bool) -> (Vec<tempfile::TempDir>, FormatV3, Sets) {
|
||||
// `disk_set` is intentionally empty: these tests only drive `heal_format`
|
||||
// with `dry_run == true`, which never touches `disk_set`.
|
||||
async fn setup_heal_format_sets(num_formatted: usize) -> (Vec<tempfile::TempDir>, FormatV3, Sets) {
|
||||
const SET_DRIVE_COUNT: usize = 3;
|
||||
let ref_format = FormatV3::new(1, SET_DRIVE_COUNT);
|
||||
let mut stored_format = ref_format.clone();
|
||||
if foreign_identity {
|
||||
stored_format.id = Uuid::new_v4();
|
||||
}
|
||||
|
||||
let mut dirs = Vec::with_capacity(SET_DRIVE_COUNT);
|
||||
let mut endpoints = Vec::with_capacity(SET_DRIVE_COUNT);
|
||||
@@ -1770,8 +1645,8 @@ mod tests {
|
||||
)
|
||||
.await
|
||||
.expect("disk should be created");
|
||||
let mut disk_format = stored_format.clone();
|
||||
disk_format.erasure.this = stored_format.erasure.sets[0][i];
|
||||
let mut disk_format = ref_format.clone();
|
||||
disk_format.erasure.this = ref_format.erasure.sets[0][i];
|
||||
save_format_file(&Some(disk), &Some(disk_format))
|
||||
.await
|
||||
.expect("format should be saved");
|
||||
@@ -1802,60 +1677,6 @@ mod tests {
|
||||
(dirs, ref_format, sets)
|
||||
}
|
||||
|
||||
async fn set_level_heal_view(sets: &Sets) -> Arc<SetDisks> {
|
||||
let endpoints = sets.endpoints.endpoints.as_ref().clone();
|
||||
let mut disks = Vec::with_capacity(endpoints.len());
|
||||
for endpoint in &endpoints {
|
||||
disks.push(Some(
|
||||
new_disk(
|
||||
endpoint,
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("fresh set-level disk handle should open"),
|
||||
));
|
||||
}
|
||||
|
||||
SetDisks::new(
|
||||
"test-owner".to_string(),
|
||||
Arc::new(RwLock::new(disks)),
|
||||
endpoints.len(),
|
||||
1,
|
||||
0,
|
||||
0,
|
||||
endpoints,
|
||||
sets.format.clone(),
|
||||
Vec::new(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn replace_heal_test_format(sets: &Sets, disk_index: usize, format: &FormatV3) {
|
||||
let disk = new_disk(
|
||||
&sets.endpoints.endpoints.as_ref()[disk_index],
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("heal test disk should open");
|
||||
save_format_file(&Some(disk.clone()), &Some(format.clone()))
|
||||
.await
|
||||
.expect("poisoned test format should be written");
|
||||
}
|
||||
|
||||
async fn read_heal_test_format(sets: &Sets, disk_index: usize) -> FormatV3 {
|
||||
let path = std::path::Path::new(&sets.endpoints.endpoints.as_ref()[disk_index].get_file_path())
|
||||
.join(crate::disk::RUSTFS_META_BUCKET)
|
||||
.join(crate::disk::FORMAT_CONFIG_FILE);
|
||||
let data = tokio::fs::read(path).await.expect("test format should be readable");
|
||||
FormatV3::try_from(data.as_slice()).expect("test format should parse")
|
||||
}
|
||||
|
||||
// Regression for #956 (NoHealRequired path): with every disk already
|
||||
// formatted, `heal_format` reports exactly one drive record per disk
|
||||
// (N = set_count * set_drive_count), each carrying a real endpoint. Before
|
||||
@@ -1864,7 +1685,7 @@ mod tests {
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn heal_format_no_heal_required_reports_one_record_per_disk() {
|
||||
let (_dirs, _ref_format, sets) = setup_heal_format_sets(3, false).await;
|
||||
let (_dirs, _ref_format, sets) = setup_heal_format_sets(3).await;
|
||||
|
||||
let (res, err) = sets.heal_format(true).await.expect("heal_format should succeed");
|
||||
// All disks formatted -> NoHealRequired early return, still returns `res`.
|
||||
@@ -1894,7 +1715,7 @@ mod tests {
|
||||
#[serial]
|
||||
async fn heal_format_heal_path_reports_one_record_per_disk_aligned() {
|
||||
// Disks 0 and 1 formatted (quorum), disk 2 unformatted.
|
||||
let (_dirs, _ref_format, sets) = setup_heal_format_sets(2, false).await;
|
||||
let (_dirs, _ref_format, sets) = setup_heal_format_sets(2).await;
|
||||
|
||||
let (res, err) = sets.heal_format(true).await.expect("heal_format should succeed");
|
||||
// Unformatted disk present -> heal path, not NoHealRequired.
|
||||
|
||||
@@ -381,291 +381,6 @@ fn classify_delete_volume_error(err: std::io::Error) -> DiskError {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
struct EmptyDirectoryFrame {
|
||||
path: PathBuf,
|
||||
name_in_parent: std::ffi::CString,
|
||||
entries: rustix::fs::Dir,
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
fn empty_tree_io_error(err: rustix::io::Errno) -> std::io::Error {
|
||||
match err {
|
||||
rustix::io::Errno::NOTDIR | rustix::io::Errno::LOOP => std::io::Error::from(ErrorKind::DirectoryNotEmpty),
|
||||
_ => err.into(),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
fn remove_empty_directory_tree_unix_with(
|
||||
root: &Path,
|
||||
mut before_descend: impl FnMut(&Path) -> std::io::Result<()>,
|
||||
mut before_remove: impl FnMut(&Path) -> std::io::Result<()>,
|
||||
) -> std::io::Result<()> {
|
||||
use rustix::{
|
||||
fs::{AtFlags, Dir, Mode, OFlags, fstat, open, openat, statat, unlinkat},
|
||||
io::Errno,
|
||||
};
|
||||
use std::os::fd::AsFd;
|
||||
use std::os::unix::ffi::OsStrExt;
|
||||
|
||||
let flags = OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC;
|
||||
let root_parent_path = root
|
||||
.parent()
|
||||
.ok_or_else(|| std::io::Error::from(ErrorKind::DirectoryNotEmpty))?;
|
||||
let root_name = root
|
||||
.file_name()
|
||||
.ok_or_else(|| std::io::Error::from(ErrorKind::DirectoryNotEmpty))?
|
||||
.as_bytes();
|
||||
let root_name = std::ffi::CString::new(root_name).map_err(|_| std::io::Error::from(ErrorKind::DirectoryNotEmpty))?;
|
||||
let root_parent = open(root_parent_path, flags, Mode::empty()).map_err(empty_tree_io_error)?;
|
||||
let root_fd = openat(&root_parent, root_name.as_c_str(), flags, Mode::empty()).map_err(empty_tree_io_error)?;
|
||||
// Each frame owns one directory iterator/FD, so memory and descriptors are
|
||||
// bounded by path depth rather than by the number of empty remnants.
|
||||
let mut stack = vec![EmptyDirectoryFrame {
|
||||
path: root.to_path_buf(),
|
||||
name_in_parent: root_name,
|
||||
entries: Dir::new(root_fd).map_err(empty_tree_io_error)?,
|
||||
}];
|
||||
|
||||
while let Some(mut frame) = stack.pop() {
|
||||
let next_child = loop {
|
||||
let Some(entry) = frame.entries.next() else {
|
||||
break None;
|
||||
};
|
||||
let entry = entry.map_err(std::io::Error::from)?;
|
||||
let name = entry.file_name();
|
||||
if name.to_bytes() == b"." || name.to_bytes() == b".." {
|
||||
continue;
|
||||
}
|
||||
|
||||
let name = name.to_owned();
|
||||
let child_path = frame.path.join(std::ffi::OsStr::from_bytes(name.as_bytes()));
|
||||
before_descend(&child_path)?;
|
||||
break Some((child_path, name));
|
||||
};
|
||||
|
||||
if let Some((child_path, name)) = next_child {
|
||||
let parent = frame.entries.fd().map_err(std::io::Error::from)?;
|
||||
let child = match openat(parent, name.as_c_str(), flags, Mode::empty()) {
|
||||
Ok(child) => child,
|
||||
// A concurrent cleanup may remove an empty child after readdir
|
||||
// returns it. Resume the parent instead of treating the whole
|
||||
// bucket as missing and leaving its root behind.
|
||||
Err(Errno::NOENT) => {
|
||||
stack.push(frame);
|
||||
continue;
|
||||
}
|
||||
Err(err) => return Err(empty_tree_io_error(err)),
|
||||
};
|
||||
stack.push(frame);
|
||||
stack.push(EmptyDirectoryFrame {
|
||||
path: child_path,
|
||||
name_in_parent: name,
|
||||
entries: Dir::new(child).map_err(empty_tree_io_error)?,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
before_remove(&frame.path)?;
|
||||
let parent = if let Some(parent) = stack.last() {
|
||||
parent.entries.fd().map_err(std::io::Error::from)?
|
||||
} else {
|
||||
root_parent.as_fd()
|
||||
};
|
||||
let expected = fstat(frame.entries.fd().map_err(std::io::Error::from)?).map_err(empty_tree_io_error)?;
|
||||
let current = statat(parent, frame.name_in_parent.as_c_str(), AtFlags::SYMLINK_NOFOLLOW).map_err(empty_tree_io_error)?;
|
||||
if current.st_dev != expected.st_dev || current.st_ino != expected.st_ino {
|
||||
return Err(std::io::Error::from(ErrorKind::DirectoryNotEmpty));
|
||||
}
|
||||
match unlinkat(parent, frame.name_in_parent.as_c_str(), AtFlags::REMOVEDIR).map_err(empty_tree_io_error) {
|
||||
Ok(()) => {}
|
||||
Err(err) if err.kind() == ErrorKind::NotFound => {}
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
async fn remove_empty_directory_tree_with(
|
||||
root: &Path,
|
||||
before_descend: impl FnMut(&Path) -> std::io::Result<()>,
|
||||
before_remove: impl FnMut(&Path) -> std::io::Result<()>,
|
||||
) -> std::io::Result<()> {
|
||||
remove_empty_directory_tree_unix_with(root, before_descend, before_remove)
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
async fn remove_empty_directory_tree(root: &Path) -> std::io::Result<()> {
|
||||
let root = root.to_path_buf();
|
||||
tokio::task::spawn_blocking(move || remove_empty_directory_tree_unix_with(&root, |_| Ok(()), |_| Ok(()))).await?
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
#[derive(Debug)]
|
||||
struct LockedEmptyDirectory {
|
||||
handle: winapi_util::Handle,
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
fn validate_windows_empty_directory(file_attributes: u64) -> std::io::Result<()> {
|
||||
const FILE_ATTRIBUTE_DIRECTORY: u64 = 0x10;
|
||||
const FILE_ATTRIBUTE_REPARSE_POINT: u64 = 0x400;
|
||||
|
||||
if file_attributes & FILE_ATTRIBUTE_DIRECTORY == 0 || file_attributes & FILE_ATTRIBUTE_REPARSE_POINT != 0 {
|
||||
return Err(std::io::Error::from(ErrorKind::DirectoryNotEmpty));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
async fn lock_windows_empty_directory(path: &Path, canonical_root: Option<&Path>) -> std::io::Result<LockedEmptyDirectory> {
|
||||
use std::os::windows::fs::OpenOptionsExt;
|
||||
use windows_sys::Win32::{
|
||||
Foundation::GENERIC_READ,
|
||||
Storage::FileSystem::{DELETE, FILE_SHARE_READ},
|
||||
};
|
||||
|
||||
const FILE_FLAG_BACKUP_SEMANTICS: u32 = 0x0200_0000;
|
||||
const FILE_FLAG_OPEN_REPARSE_POINT: u32 = 0x0020_0000;
|
||||
|
||||
let path = path.to_path_buf();
|
||||
let canonical_root = canonical_root.map(Path::to_path_buf);
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let file = std::fs::OpenOptions::new()
|
||||
.access_mode(GENERIC_READ | DELETE)
|
||||
.share_mode(FILE_SHARE_READ)
|
||||
.custom_flags(FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OPEN_REPARSE_POINT)
|
||||
.open(&path)?;
|
||||
let handle = winapi_util::Handle::from_file(file);
|
||||
let info = winapi_util::file::information(&handle)?;
|
||||
validate_windows_empty_directory(info.file_attributes())?;
|
||||
if let Some(canonical_root) = canonical_root {
|
||||
let canonical_path = std::fs::canonicalize(path)?;
|
||||
if !canonical_path.starts_with(canonical_root) {
|
||||
return Err(std::io::Error::from(ErrorKind::DirectoryNotEmpty));
|
||||
}
|
||||
}
|
||||
Ok::<_, std::io::Error>(LockedEmptyDirectory { handle })
|
||||
})
|
||||
.await?
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
// SAFETY: This helper only passes an owned live handle and one initialized
|
||||
// FILE_DISPOSITION_INFO to the synchronous Windows deletion API.
|
||||
#[allow(unsafe_code)]
|
||||
async fn remove_windows_empty_directory(directory: LockedEmptyDirectory) -> std::io::Result<()> {
|
||||
tokio::task::spawn_blocking(move || {
|
||||
use std::os::windows::io::AsRawHandle;
|
||||
use windows_sys::Win32::Storage::FileSystem::{FILE_DISPOSITION_INFO, FileDispositionInfo, SetFileInformationByHandle};
|
||||
|
||||
let disposition = FILE_DISPOSITION_INFO { DeleteFile: true };
|
||||
let disposition_size = u32::try_from(std::mem::size_of_val(&disposition))
|
||||
.map_err(|_| std::io::Error::other("FILE_DISPOSITION_INFO size exceeds the Win32 API limit"))?;
|
||||
let handle = directory.handle.as_raw_handle();
|
||||
// SAFETY: `handle` is owned by `directory` and stays live for this synchronous
|
||||
// call. `disposition` is initialized with the exact structure and byte size
|
||||
// required by `FileDispositionInfo`; Windows does not retain the pointer.
|
||||
let deleted = unsafe {
|
||||
SetFileInformationByHandle(handle, FileDispositionInfo, std::ptr::from_ref(&disposition).cast(), disposition_size)
|
||||
};
|
||||
if deleted == 0 {
|
||||
Err(std::io::Error::last_os_error())
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
})
|
||||
.await?
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
struct WindowsEmptyDirectoryFrame {
|
||||
path: PathBuf,
|
||||
directory: LockedEmptyDirectory,
|
||||
entries: fs::ReadDir,
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
async fn remove_empty_directory_tree_with(
|
||||
root: &Path,
|
||||
mut before_descend: impl FnMut(&Path) -> std::io::Result<()>,
|
||||
mut before_remove: impl FnMut(&Path) -> std::io::Result<()>,
|
||||
) -> std::io::Result<()> {
|
||||
let root_directory = lock_windows_empty_directory(root, None).await?;
|
||||
let canonical_root = fs::canonicalize(root).await?;
|
||||
let root_entries = fs::read_dir(root).await?;
|
||||
|
||||
// Holding each validated directory without delete sharing keeps its path
|
||||
// generation stable until handle-relative deletion. State is O(depth).
|
||||
let mut stack = vec![WindowsEmptyDirectoryFrame {
|
||||
path: root.to_path_buf(),
|
||||
directory: root_directory,
|
||||
entries: root_entries,
|
||||
}];
|
||||
|
||||
while let Some(mut frame) = stack.pop() {
|
||||
match frame.entries.next_entry().await {
|
||||
Ok(Some(entry)) => {
|
||||
let child = entry.path();
|
||||
before_descend(&child)?;
|
||||
let child_directory = match lock_windows_empty_directory(&child, Some(&canonical_root)).await {
|
||||
Ok(directory) => directory,
|
||||
Err(err) if err.kind() == ErrorKind::NotFound => {
|
||||
stack.push(frame);
|
||||
continue;
|
||||
}
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
let child_entries = match fs::read_dir(&child).await {
|
||||
Ok(entries) => entries,
|
||||
Err(err) if err.kind() == ErrorKind::NotFound => {
|
||||
stack.push(frame);
|
||||
continue;
|
||||
}
|
||||
Err(err) if err.kind() == ErrorKind::NotADirectory => {
|
||||
return Err(std::io::Error::from(ErrorKind::DirectoryNotEmpty));
|
||||
}
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
stack.push(frame);
|
||||
stack.push(WindowsEmptyDirectoryFrame {
|
||||
path: child,
|
||||
directory: child_directory,
|
||||
entries: child_entries,
|
||||
});
|
||||
}
|
||||
Ok(None) => {
|
||||
before_remove(&frame.path)?;
|
||||
drop(frame.entries);
|
||||
match remove_windows_empty_directory(frame.directory).await {
|
||||
Ok(()) => {}
|
||||
Err(err) if err.kind() == ErrorKind::NotFound => {}
|
||||
Err(err) if err.kind() == ErrorKind::NotADirectory => {
|
||||
return Err(std::io::Error::from(ErrorKind::DirectoryNotEmpty));
|
||||
}
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
}
|
||||
Err(err) if err.kind() == ErrorKind::NotFound => {}
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
async fn remove_empty_directory_tree(root: &Path) -> std::io::Result<()> {
|
||||
remove_empty_directory_tree_with(root, |_| Ok(()), |_| Ok(())).await
|
||||
}
|
||||
|
||||
#[cfg(all(not(unix), not(windows)))]
|
||||
async fn remove_empty_directory_tree(root: &Path) -> std::io::Result<()> {
|
||||
fs::remove_dir(root).await
|
||||
}
|
||||
|
||||
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
|
||||
const LOG_SUBSYSTEM_DISK_LOCAL: &str = "disk_local";
|
||||
const EVENT_DISK_LOCAL_STARTUP_CLEANUP: &str = "disk_local_startup_cleanup";
|
||||
@@ -1841,8 +1556,6 @@ static RENAME_DATA_FAIL_COMMIT_RENAME: std::sync::Mutex<Option<String>> = std::s
|
||||
#[cfg(test)]
|
||||
static LOCAL_INLINE_ROLLBACK_HARDLINK_FAILURE: std::sync::Mutex<Option<PathBuf>> = std::sync::Mutex::new(None);
|
||||
#[cfg(test)]
|
||||
static RENAME_DATA_REMOVE_DST_BASE_BEFORE_COMMIT: std::sync::Mutex<Option<(String, PathBuf)>> = std::sync::Mutex::new(None);
|
||||
#[cfg(test)]
|
||||
static DELETE_VERSION_FAIL_AFTER_DATA_STAGED: std::sync::Mutex<Vec<String>> = std::sync::Mutex::new(Vec::new());
|
||||
#[cfg(test)]
|
||||
static DELETE_VERSION_FAIL_AFTER_COMMIT: std::sync::Mutex<Vec<(PathBuf, String)>> = std::sync::Mutex::new(Vec::new());
|
||||
@@ -1875,13 +1588,6 @@ fn set_local_inline_rollback_hardlink_failure(dst_path: &Path) {
|
||||
.expect("test failpoint lock should not be poisoned") = Some(dst_path.to_path_buf());
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn set_rename_data_remove_dst_base_before_commit(dst_path: &str, dst_base: &Path) {
|
||||
*RENAME_DATA_REMOVE_DST_BASE_BEFORE_COMMIT
|
||||
.lock()
|
||||
.expect("test failpoint lock should not be poisoned") = Some((dst_path.to_string(), dst_base.to_path_buf()));
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn set_delete_version_fail_after_data_staged(path: &str) {
|
||||
DELETE_VERSION_FAIL_AFTER_DATA_STAGED
|
||||
@@ -1950,19 +1656,6 @@ fn should_fail_local_inline_rollback_hardlink(dst_path: &Path) -> bool {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn remove_dst_base_before_commit(dst_path: &str) -> std::io::Result<()> {
|
||||
let mut target = RENAME_DATA_REMOVE_DST_BASE_BEFORE_COMMIT
|
||||
.lock()
|
||||
.expect("test failpoint lock should not be poisoned");
|
||||
let Some((_, base)) = target.as_ref().filter(|(target_path, _)| target_path == dst_path) else {
|
||||
return Ok(());
|
||||
};
|
||||
std::fs::remove_dir_all(base)?;
|
||||
target.take();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn should_fail_after_delete_data_staged(path: &str) -> bool {
|
||||
let mut targets = DELETE_VERSION_FAIL_AFTER_DATA_STAGED
|
||||
@@ -2012,11 +1705,6 @@ fn should_fail_local_inline_rollback_hardlink(_dst_path: &Path) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
#[cfg(not(test))]
|
||||
fn remove_dst_base_before_commit(_dst_path: &str) -> std::io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(not(test))]
|
||||
fn should_fail_after_delete_data_staged(_path: &str) -> bool {
|
||||
false
|
||||
@@ -4555,11 +4243,9 @@ impl LocalDisk {
|
||||
let tmp_path = Self::meta_path(root, RUSTFS_META_TMP_BUCKET);
|
||||
let tmp_old_path = Self::meta_path(root, RUSTFS_META_TMP_OLD_BUCKET).join(Uuid::new_v4().to_string());
|
||||
|
||||
rename_all_ignore_missing_source(&tmp_path, &tmp_old_path, root)
|
||||
.await
|
||||
.inspect_err(|err| {
|
||||
log_startup_disk_error("cleanup_tmp_rename_all", &tmp_path, err);
|
||||
})?;
|
||||
rename_all(&tmp_path, &tmp_old_path, root).await.inspect_err(|err| {
|
||||
log_startup_disk_error("cleanup_tmp_rename_all", &tmp_path, err);
|
||||
})?;
|
||||
|
||||
let tmp_deleted_path = Self::meta_path(root, RUSTFS_META_TMP_DELETED_BUCKET);
|
||||
tokio::fs::create_dir_all(&tmp_deleted_path).await.inspect_err(|err| {
|
||||
@@ -7673,10 +7359,6 @@ impl DiskAPI for LocalDisk {
|
||||
dst_path: &str,
|
||||
) -> Result<RenameDataResp> {
|
||||
crate::hp_guard!("LocalDisk::rename_data");
|
||||
// A non-force DeleteBucket must not remove a directory while a local
|
||||
// object commit is publishing into it. The peer's empty scan remains
|
||||
// optimistic; this guard establishes the local commit/delete order.
|
||||
let _volume_mutation_guard = os::disk_volume_mutation_lock(&self.root, dst_volume).read_owned().await;
|
||||
if fi.is_legacy_indexed_delete_marker() {
|
||||
fi.erasure.index = 0;
|
||||
}
|
||||
@@ -7873,7 +7555,6 @@ impl DiskAPI for LocalDisk {
|
||||
// sequential version did.
|
||||
tmp_meta_res?;
|
||||
shard_sync_res?;
|
||||
remove_dst_base_before_commit(dst_path).map_err(to_file_error)?;
|
||||
|
||||
if let Some((src_data_path, dst_data_path)) = has_data_dir_path.as_ref()
|
||||
&& let Err(err) = rename_all(src_data_path, dst_data_path, &skip_parent).await
|
||||
@@ -8133,7 +7814,6 @@ impl DiskAPI for LocalDisk {
|
||||
xlmeta.add_version(fi)?;
|
||||
let version_signature = rename_data_versions_signature(&xlmeta);
|
||||
let new_buf = xlmeta.marshal_msg()?;
|
||||
remove_dst_base_before_commit(&dst_path_for_failpoint).map_err(to_file_error)?;
|
||||
|
||||
// Write new xl.meta + rename. Inline objects carry their data
|
||||
// inside xl.meta, so this whole sequence is a metadata commit:
|
||||
@@ -8158,11 +7838,9 @@ impl DiskAPI for LocalDisk {
|
||||
{
|
||||
let old_path = dst_parent.join(old_dir.to_string()).join(STORAGE_FORMAT_FILE_BACKUP);
|
||||
let old_parent = old_path.parent().map(|p| p.to_path_buf());
|
||||
let _old_parent_guard = old_parent
|
||||
.as_deref()
|
||||
.map(|parent| os::mkdir_all_below_existing_base_std(parent, &bucket_dir))
|
||||
.transpose()
|
||||
.map_err(to_file_error)?;
|
||||
if let Some(ref old_parent) = old_parent {
|
||||
std::fs::create_dir_all(old_parent)?;
|
||||
}
|
||||
// This rollback backup is the sole restore source for a later
|
||||
// undo_write when the set-level write quorum fails. Persist it as
|
||||
// durably as the new xl.meta written above (and as the non-inline
|
||||
@@ -8188,12 +7866,6 @@ impl DiskAPI for LocalDisk {
|
||||
local_rollback_path = Some(create_local_inline_rollback_backup(&dst, &src, old_metadata)?);
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
let _commit_parent_guard = if let Some(parent) = dst.parent() {
|
||||
Some(os::mkdir_all_below_existing_base_std(parent, &bucket_dir).map_err(to_file_error)?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let commit_result = if should_fail_commit_rename(&dst_path_for_failpoint) {
|
||||
Err(std::io::Error::other("test fail during metadata commit rename"))
|
||||
} else {
|
||||
@@ -8202,8 +7874,7 @@ impl DiskAPI for LocalDisk {
|
||||
Err(err) if err.kind() == ErrorKind::NotFound && !src.exists() => Ok(()),
|
||||
Err(err) if err.kind() == ErrorKind::NotFound => {
|
||||
if let Some(parent) = dst.parent() {
|
||||
let _parent_guard =
|
||||
os::mkdir_all_below_existing_base_std(parent, &bucket_dir).map_err(to_file_error)?;
|
||||
std::fs::create_dir_all(parent)?;
|
||||
}
|
||||
std::fs::rename(&src, &dst).map_err(to_file_error)?;
|
||||
Ok(())
|
||||
@@ -9120,16 +8791,18 @@ impl DiskAPI for LocalDisk {
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
async fn delete_volume(&self, volume: &str, force_delete: bool) -> Result<()> {
|
||||
let p = self.get_bucket_path(volume)?;
|
||||
let _volume_mutation_guard = os::disk_volume_mutation_lock(&self.root, volume).write_owned().await;
|
||||
|
||||
// Non-force removes empty directory remnants children-first with
|
||||
// non-recursive rmdir calls. A file that exists during the scan, or
|
||||
// appears before its parent is removed, fails closed with
|
||||
// VolumeNotEmpty. Only an explicit force delete removes recursively.
|
||||
// Non-force is non-recursive: `remove_dir` (rmdir) fails atomically with
|
||||
// `DirectoryNotEmpty` -> VolumeNotEmpty if the bucket still holds any
|
||||
// object data, so a misclassified "dangling" bucket on the heal path
|
||||
// (or a non-force S3 DeleteBucket on a populated bucket) can never be
|
||||
// recursively wiped. Only an explicit `force_delete` (e.g. S3 force
|
||||
// bucket delete) removes recursively. Mirrors MinIO's
|
||||
// xlStorage.DeleteVol (Remove vs RemoveAll). (backlog#799 B1)
|
||||
let res = if force_delete {
|
||||
fs::remove_dir_all(&p).await
|
||||
} else {
|
||||
remove_empty_directory_tree(&p).await
|
||||
fs::remove_dir(&p).await
|
||||
};
|
||||
|
||||
if let Err(err) = res {
|
||||
@@ -9388,35 +9061,6 @@ mod test {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
#[test]
|
||||
fn windows_empty_tree_requires_non_reparse_directory() {
|
||||
validate_windows_empty_directory(0x10).expect("ordinary directories should be accepted");
|
||||
assert!(validate_windows_empty_directory(0).is_err());
|
||||
assert!(validate_windows_empty_directory(0x10 | 0x400).is_err());
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
#[tokio::test]
|
||||
async fn windows_empty_tree_blocks_replacement_at_final_delete_boundary() {
|
||||
let root = tempfile::tempdir().expect("temporary root should be created");
|
||||
let bucket_path = root.path().join("bucket");
|
||||
let child_path = bucket_path.join("child");
|
||||
fs::create_dir_all(&child_path).await.expect("bucket child should be created");
|
||||
let canonical_root = fs::canonicalize(&bucket_path).await.expect("bucket path should canonicalize");
|
||||
let directory = lock_windows_empty_directory(&child_path, Some(&canonical_root))
|
||||
.await
|
||||
.expect("child directory should be locked");
|
||||
|
||||
std::fs::rename(&child_path, bucket_path.join("replacement"))
|
||||
.expect_err("the locked directory must not be replaceable at the final deletion boundary");
|
||||
remove_windows_empty_directory(directory)
|
||||
.await
|
||||
.expect("handle-relative deletion should remove the locked directory");
|
||||
|
||||
assert!(!child_path.exists(), "the exact locked directory should be removed");
|
||||
}
|
||||
|
||||
async fn ensure_test_volume(disk: &LocalDisk, volume: &str) {
|
||||
match disk.make_volume(volume).await {
|
||||
Ok(()) | Err(DiskError::VolumeExists) => {}
|
||||
@@ -11143,72 +10787,6 @@ mod test {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(rename_data_deleted_bucket)]
|
||||
async fn rename_data_non_inline_does_not_recreate_bucket_deleted_before_commit() {
|
||||
let dir = tempfile::tempdir().expect("temp dir should be created");
|
||||
let endpoint = Endpoint::try_from(dir.path().to_string_lossy().as_ref()).expect("endpoint should parse");
|
||||
let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created");
|
||||
let bucket = "deleted-before-non-inline-commit";
|
||||
let object = "prefix/object";
|
||||
let tmp_object = "tmp-non-inline-delete-race";
|
||||
let data_dir = Uuid::parse_str("aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee").expect("data dir should parse");
|
||||
ensure_test_volume(&disk, bucket).await;
|
||||
ensure_test_volume(&disk, RUSTFS_META_TMP_BUCKET).await;
|
||||
|
||||
let staged_data = disk
|
||||
.get_object_path(RUSTFS_META_TMP_BUCKET, &format!("{tmp_object}/{data_dir}/part.1"))
|
||||
.expect("staged data path should resolve");
|
||||
fs::create_dir_all(staged_data.parent().expect("staged data should have a parent"))
|
||||
.await
|
||||
.expect("staged data parent should be created");
|
||||
fs::write(&staged_data, b"payload")
|
||||
.await
|
||||
.expect("staged shard should be written");
|
||||
|
||||
let bucket_path = disk.get_bucket_path(bucket).expect("bucket path should resolve");
|
||||
set_rename_data_remove_dst_base_before_commit(object, &bucket_path);
|
||||
let fi = test_file_info(object, Uuid::new_v4(), Some(data_dir), None);
|
||||
let err = disk
|
||||
.rename_data(RUSTFS_META_TMP_BUCKET, tmp_object, fi, bucket, object)
|
||||
.await
|
||||
.expect_err("commit must fail after the destination bucket is deleted");
|
||||
|
||||
assert_eq!(err, DiskError::FileNotFound);
|
||||
assert!(!bucket_path.exists(), "rename_data must not recreate the deleted bucket");
|
||||
assert!(staged_data.exists(), "failed commit must preserve the staged shard");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(rename_data_deleted_bucket)]
|
||||
async fn rename_data_inline_does_not_recreate_bucket_deleted_before_commit() {
|
||||
let dir = tempfile::tempdir().expect("temp dir should be created");
|
||||
let endpoint = Endpoint::try_from(dir.path().to_string_lossy().as_ref()).expect("endpoint should parse");
|
||||
let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created");
|
||||
let bucket = "deleted-before-inline-commit";
|
||||
let object = "prefix/object";
|
||||
let tmp_object = "tmp-inline-delete-race";
|
||||
ensure_test_volume(&disk, bucket).await;
|
||||
ensure_test_volume(&disk, RUSTFS_META_TMP_BUCKET).await;
|
||||
|
||||
let bucket_path = disk.get_bucket_path(bucket).expect("bucket path should resolve");
|
||||
set_rename_data_remove_dst_base_before_commit(object, &bucket_path);
|
||||
let fi = test_file_info(object, Uuid::new_v4(), None, Some(Bytes::from_static(b"inline payload")));
|
||||
let err = disk
|
||||
.rename_data(RUSTFS_META_TMP_BUCKET, tmp_object, fi, bucket, object)
|
||||
.await
|
||||
.expect_err("inline commit must fail after the destination bucket is deleted");
|
||||
|
||||
assert_eq!(err, DiskError::FileNotFound);
|
||||
assert!(!bucket_path.exists(), "inline rename_data must not recreate the deleted bucket");
|
||||
assert!(
|
||||
disk.get_object_path(RUSTFS_META_TMP_BUCKET, &format!("{tmp_object}/{STORAGE_FORMAT_FILE}"))
|
||||
.expect("staged metadata path should resolve")
|
||||
.exists(),
|
||||
"failed inline commit must preserve staged metadata"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_resolve_durability_mode_mapping() {
|
||||
// Default: nothing set -> strict (current main behavior).
|
||||
@@ -13067,19 +12645,6 @@ mod test {
|
||||
assert!(format_info.last_check.is_none(), "cached format timestamp should be cleared");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cleanup_tmp_on_startup_allows_missing_tmp_directory() {
|
||||
use tempfile::tempdir;
|
||||
|
||||
let dir = tempdir().expect("operation should succeed");
|
||||
|
||||
LocalDisk::cleanup_tmp_on_startup(dir.path(), Arc::new(AtomicU32::new(0)), Arc::new(Notify::new()))
|
||||
.await
|
||||
.expect("missing temporary directory should already be clean");
|
||||
|
||||
assert!(LocalDisk::meta_path(dir.path(), RUSTFS_META_TMP_DELETED_BUCKET).exists());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cleanup_tmp_on_startup_moves_existing_tmp_and_recreates_trash() {
|
||||
use tempfile::tempdir;
|
||||
@@ -14952,154 +14517,6 @@ mod test {
|
||||
let _ = fs::remove_dir_all(&test_dir).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_volume_non_force_removes_nested_empty_directories() {
|
||||
let root = tempfile::tempdir().expect("temporary disk root should be created");
|
||||
let endpoint = Endpoint::try_from(root.path().to_string_lossy().as_ref()).expect("endpoint should parse");
|
||||
let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created");
|
||||
let bucket = "nested-empty-bucket";
|
||||
|
||||
disk.make_volume(bucket).await.expect("bucket should be created");
|
||||
fs::create_dir_all(disk.path().join(bucket).join("a/b/c"))
|
||||
.await
|
||||
.expect("nested empty directories should be created");
|
||||
|
||||
disk.delete_volume(bucket, false)
|
||||
.await
|
||||
.expect("non-force delete should remove an empty directory tree");
|
||||
|
||||
assert!(matches!(disk.stat_volume(bucket).await, Err(DiskError::VolumeNotFound)));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn empty_tree_delete_preserves_xlmeta_published_after_scan() {
|
||||
let root = tempfile::tempdir().expect("temporary disk root should be created");
|
||||
let bucket_path = root.path().join("bucket");
|
||||
let object_path = bucket_path.join("object").join(STORAGE_FORMAT_FILE);
|
||||
fs::create_dir_all(object_path.parent().expect("object path should have a parent"))
|
||||
.await
|
||||
.expect("empty object directory should be created");
|
||||
|
||||
let data = b"committed object metadata";
|
||||
let mut published = false;
|
||||
let err = remove_empty_directory_tree_with(
|
||||
&bucket_path,
|
||||
|_| Ok(()),
|
||||
|directory| {
|
||||
if !published && directory == object_path.parent().expect("object path should have a parent") {
|
||||
std::fs::write(&object_path, data)?;
|
||||
published = true;
|
||||
}
|
||||
Ok(())
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect_err("rmdir should refuse metadata published after the directory scan");
|
||||
|
||||
assert!(published, "test barrier should publish metadata before rmdir");
|
||||
assert!(matches!(classify_delete_volume_error(err), DiskError::VolumeNotEmpty));
|
||||
assert_eq!(std::fs::read(&object_path).expect("object metadata should be preserved"), data);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn empty_tree_delete_rejects_child_replaced_with_external_symlink() {
|
||||
use std::os::unix::fs::symlink;
|
||||
|
||||
let root = tempfile::tempdir().expect("temporary root should be created");
|
||||
let bucket_path = root.path().join("bucket");
|
||||
let child_path = bucket_path.join("child");
|
||||
let outside_path = root.path().join("outside");
|
||||
let outside_empty = outside_path.join("must-remain");
|
||||
fs::create_dir_all(&child_path).await.expect("bucket child should be created");
|
||||
fs::create_dir_all(&outside_empty)
|
||||
.await
|
||||
.expect("outside directory should be created");
|
||||
|
||||
let mut replaced = false;
|
||||
let err = remove_empty_directory_tree_with(
|
||||
&bucket_path,
|
||||
|child| {
|
||||
if !replaced && child == child_path {
|
||||
std::fs::remove_dir(&child_path)?;
|
||||
symlink(&outside_path, &child_path)?;
|
||||
replaced = true;
|
||||
}
|
||||
Ok(())
|
||||
},
|
||||
|_| Ok(()),
|
||||
)
|
||||
.await
|
||||
.expect_err("a replaced child must fail closed");
|
||||
|
||||
assert!(replaced, "test barrier should replace the child before it is opened");
|
||||
assert!(matches!(classify_delete_volume_error(err), DiskError::VolumeNotEmpty));
|
||||
assert!(outside_empty.exists(), "bucket deletion must not remove directories outside the bucket");
|
||||
assert!(
|
||||
std::fs::symlink_metadata(&child_path)
|
||||
.expect("replacement symlink should remain")
|
||||
.file_type()
|
||||
.is_symlink()
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn empty_tree_delete_rechecks_parent_after_child_disappears() {
|
||||
let root = tempfile::tempdir().expect("temporary root should be created");
|
||||
let bucket_path = root.path().join("bucket");
|
||||
let child_path = bucket_path.join("child");
|
||||
fs::create_dir_all(&child_path).await.expect("bucket child should be created");
|
||||
|
||||
let mut removed = false;
|
||||
remove_empty_directory_tree_with(
|
||||
&bucket_path,
|
||||
|child| {
|
||||
if !removed && child == child_path {
|
||||
std::fs::remove_dir(&child_path)?;
|
||||
removed = true;
|
||||
}
|
||||
Ok(())
|
||||
},
|
||||
|_| Ok(()),
|
||||
)
|
||||
.await
|
||||
.expect("a vanished empty child should not leave the bucket root behind");
|
||||
|
||||
assert!(removed, "test hook should remove the child before openat");
|
||||
assert!(!bucket_path.exists(), "parent should be rechecked and removed after the child disappears");
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn empty_tree_delete_rejects_root_generation_replacement() {
|
||||
let root = tempfile::tempdir().expect("temporary root should be created");
|
||||
let bucket_path = root.path().join("bucket");
|
||||
let moved_path = root.path().join("bucket-before-replacement");
|
||||
fs::create_dir(&bucket_path).await.expect("bucket should be created");
|
||||
|
||||
let mut replaced = false;
|
||||
let err = remove_empty_directory_tree_with(
|
||||
&bucket_path,
|
||||
|_| Ok(()),
|
||||
|directory| {
|
||||
if !replaced && directory == bucket_path {
|
||||
std::fs::rename(&bucket_path, &moved_path)?;
|
||||
std::fs::create_dir(&bucket_path)?;
|
||||
replaced = true;
|
||||
}
|
||||
Ok(())
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect_err("a replacement root directory must fail closed");
|
||||
|
||||
assert!(replaced, "test hook should replace the root generation before rmdir");
|
||||
assert!(matches!(classify_delete_volume_error(err), DiskError::VolumeNotEmpty));
|
||||
assert!(moved_path.exists(), "the originally scanned root should not be removed by its old name");
|
||||
assert!(bucket_path.exists(), "the replacement root should remain after identity validation fails");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_local_disk_volume_operations() {
|
||||
let test_dir = "./test_local_disk_volumes";
|
||||
|
||||
@@ -25,7 +25,7 @@ use std::{
|
||||
sync::{Arc, LazyLock, Weak},
|
||||
};
|
||||
use tokio::fs;
|
||||
use tokio::sync::{OwnedSemaphorePermit, RwLock, Semaphore, SemaphorePermit};
|
||||
use tokio::sync::{OwnedSemaphorePermit, Semaphore, SemaphorePermit};
|
||||
use tracing::warn;
|
||||
|
||||
/// Check path length according to OS limits.
|
||||
@@ -123,8 +123,6 @@ const TEST_GLOBAL_FILE_SYNCS: usize = 64;
|
||||
|
||||
static FILE_SYNC_PERMITS: LazyLock<Semaphore> = LazyLock::new(|| Semaphore::new(global_file_sync_limit()));
|
||||
static DISK_FILE_SYNC_LIMITERS: LazyLock<Mutex<HashMap<PathBuf, Weak<Semaphore>>>> = LazyLock::new(|| Mutex::new(HashMap::new()));
|
||||
static DISK_VOLUME_MUTATION_LOCKS: LazyLock<Mutex<HashMap<PathBuf, Weak<RwLock<()>>>>> =
|
||||
LazyLock::new(|| Mutex::new(HashMap::new()));
|
||||
|
||||
fn default_global_file_sync_limit(cpu_count: usize, max_blocking_threads: usize) -> usize {
|
||||
let cpu_scaled = cpu_count
|
||||
@@ -159,24 +157,6 @@ pub(crate) fn disk_file_sync_limiter(root: &Path) -> Arc<Semaphore> {
|
||||
limiter
|
||||
}
|
||||
|
||||
/// Serialize a bucket's local metadata commits with physical bucket removal.
|
||||
///
|
||||
/// The key includes the canonical disk root, so independently reconnected
|
||||
/// [`LocalDisk`](super::local::LocalDisk) instances share the same lock while
|
||||
/// disconnected disks do not keep the registry alive.
|
||||
pub(crate) fn disk_volume_mutation_lock(root: &Path, volume: &str) -> Arc<RwLock<()>> {
|
||||
let key = root.join(volume);
|
||||
let mut locks = DISK_VOLUME_MUTATION_LOCKS.lock();
|
||||
locks.retain(|_, lock| lock.strong_count() > 0);
|
||||
if let Some(lock) = locks.get(&key).and_then(Weak::upgrade) {
|
||||
return lock;
|
||||
}
|
||||
|
||||
let lock = Arc::new(RwLock::new(()));
|
||||
locks.insert(key, Arc::downgrade(&lock));
|
||||
lock
|
||||
}
|
||||
|
||||
/// Always acquire the per-disk permit before the process-wide permit. Keeping
|
||||
/// this order uniform prevents one slow disk from reserving global capacity
|
||||
/// while it waits for its own concurrency slot.
|
||||
@@ -592,14 +572,15 @@ async fn reliable_rename_inner(
|
||||
base_dir: impl AsRef<Path>,
|
||||
warn_on_missing_source: bool,
|
||||
) -> io::Result<()> {
|
||||
let parent_guard = match dst_file_path.as_ref().parent() {
|
||||
Some(parent) => Some(mkdir_all_below_existing_base(parent, base_dir.as_ref()).await?),
|
||||
None => None,
|
||||
};
|
||||
if let Some(parent) = dst_file_path.as_ref().parent()
|
||||
&& !file_exists(parent)
|
||||
{
|
||||
reliable_mkdir_all(parent, base_dir.as_ref()).await?;
|
||||
}
|
||||
|
||||
let mut i = 0;
|
||||
loop {
|
||||
if let Err(e) = rename_into_existing_parent(src_file_path.as_ref(), dst_file_path.as_ref(), parent_guard.as_ref()) {
|
||||
if let Err(e) = super::fs::rename_std(src_file_path.as_ref(), dst_file_path.as_ref()) {
|
||||
if should_retry_rename(&e, i) {
|
||||
i += 1;
|
||||
continue;
|
||||
@@ -616,159 +597,6 @@ async fn reliable_rename_inner(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
fn rename_into_existing_parent(
|
||||
src_file_path: &Path,
|
||||
dst_file_path: &Path,
|
||||
parent_guard: Option<&ExistingBaseDirectoryGuard>,
|
||||
) -> io::Result<()> {
|
||||
use rustix::fs::{Mode, OFlags, open, renameat};
|
||||
|
||||
let Some(parent_guard) = parent_guard else {
|
||||
return super::fs::rename_std(src_file_path, dst_file_path);
|
||||
};
|
||||
let src_parent = src_file_path
|
||||
.parent()
|
||||
.ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "rename source must have a parent directory"))?;
|
||||
let src_name = src_file_path
|
||||
.file_name()
|
||||
.ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "rename source must have a file name"))?;
|
||||
let dst_name = dst_file_path
|
||||
.file_name()
|
||||
.ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "rename destination must have a file name"))?;
|
||||
let src_parent = open(
|
||||
src_parent,
|
||||
OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC,
|
||||
Mode::empty(),
|
||||
)
|
||||
.map_err(io::Error::from)?;
|
||||
let dst_parent = parent_guard
|
||||
.last()
|
||||
.ok_or_else(|| io::Error::other("rename destination parent guard is empty"))?;
|
||||
|
||||
renameat(&src_parent, src_name, dst_parent, dst_name).map_err(io::Error::from)
|
||||
}
|
||||
|
||||
#[cfg(not(unix))]
|
||||
fn rename_into_existing_parent(
|
||||
src_file_path: &Path,
|
||||
dst_file_path: &Path,
|
||||
_parent_guard: Option<&ExistingBaseDirectoryGuard>,
|
||||
) -> io::Result<()> {
|
||||
super::fs::rename_std(src_file_path, dst_file_path)
|
||||
}
|
||||
|
||||
async fn mkdir_all_below_existing_base(dir_path: &Path, base_dir: &Path) -> io::Result<ExistingBaseDirectoryGuard> {
|
||||
let dir_path = dir_path.to_path_buf();
|
||||
let base_dir = base_dir.to_path_buf();
|
||||
|
||||
tokio::task::spawn_blocking(move || mkdir_all_below_existing_base_std(&dir_path, &base_dir)).await?
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
pub(crate) type ExistingBaseDirectoryGuard = Vec<winapi_util::Handle>;
|
||||
|
||||
#[cfg(unix)]
|
||||
pub(crate) type ExistingBaseDirectoryGuard = Vec<std::os::fd::OwnedFd>;
|
||||
|
||||
#[cfg(all(not(unix), not(windows)))]
|
||||
pub(crate) type ExistingBaseDirectoryGuard = ();
|
||||
|
||||
#[cfg(windows)]
|
||||
fn lock_windows_directory(path: &Path) -> io::Result<winapi_util::Handle> {
|
||||
use std::os::windows::fs::OpenOptionsExt;
|
||||
|
||||
const FILE_ATTRIBUTE_DIRECTORY: u64 = 0x10;
|
||||
const FILE_ATTRIBUTE_REPARSE_POINT: u32 = 0x400;
|
||||
const FILE_FLAG_BACKUP_SEMANTICS: u32 = 0x0200_0000;
|
||||
const FILE_FLAG_OPEN_REPARSE_POINT: u32 = 0x0020_0000;
|
||||
const FILE_SHARE_READ: u32 = 0x1;
|
||||
|
||||
let file = std::fs::OpenOptions::new()
|
||||
.read(true)
|
||||
.share_mode(FILE_SHARE_READ)
|
||||
.custom_flags(FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OPEN_REPARSE_POINT)
|
||||
.open(path)?;
|
||||
let handle = winapi_util::Handle::from_file(file);
|
||||
let info = winapi_util::file::information(&handle)?;
|
||||
if info.file_attributes() & FILE_ATTRIBUTE_DIRECTORY == 0
|
||||
|| info.file_attributes() & u64::from(FILE_ATTRIBUTE_REPARSE_POINT) != 0
|
||||
{
|
||||
return Err(io::Error::from(io::ErrorKind::NotADirectory));
|
||||
}
|
||||
Ok(handle)
|
||||
}
|
||||
|
||||
pub(crate) fn mkdir_all_below_existing_base_std(dir_path: &Path, base_dir: &Path) -> io::Result<ExistingBaseDirectoryGuard> {
|
||||
let relative = dir_path
|
||||
.strip_prefix(base_dir)
|
||||
.map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "rename destination must remain below its base directory"))?;
|
||||
for component in relative.components() {
|
||||
if !matches!(component, Component::Normal(_) | Component::CurDir) {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::InvalidInput,
|
||||
"rename destination contains an invalid path component",
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use rustix::fs::{Mode, OFlags, mkdirat, open, openat};
|
||||
use rustix::io::Errno;
|
||||
|
||||
let flags = OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC;
|
||||
let mode = Mode::RWXU | Mode::RWXG | Mode::RWXO;
|
||||
let mut parents = vec![open(base_dir, flags, Mode::empty()).map_err(io::Error::from)?];
|
||||
|
||||
for component in relative.components() {
|
||||
let Component::Normal(component) = component else {
|
||||
continue;
|
||||
};
|
||||
let parent = parents
|
||||
.last()
|
||||
.expect("base directory guard should contain the base directory");
|
||||
match mkdirat(parent, component, mode) {
|
||||
Ok(()) => {}
|
||||
Err(Errno::EXIST) => {}
|
||||
Err(err) => return Err(err.into()),
|
||||
}
|
||||
parents.push(openat(parent, component, flags, Mode::empty()).map_err(io::Error::from)?);
|
||||
}
|
||||
|
||||
Ok(parents)
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
{
|
||||
let mut handles = vec![lock_windows_directory(base_dir)?];
|
||||
let mut current = base_dir.to_path_buf();
|
||||
for component in relative.components() {
|
||||
let Component::Normal(component) = component else {
|
||||
continue;
|
||||
};
|
||||
current.push(component);
|
||||
match std::fs::create_dir(¤t) {
|
||||
Ok(()) => {}
|
||||
Err(err) if err.kind() == io::ErrorKind::AlreadyExists => {}
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
handles.push(lock_windows_directory(¤t)?);
|
||||
}
|
||||
|
||||
Ok(handles)
|
||||
}
|
||||
|
||||
#[cfg(all(not(unix), not(windows)))]
|
||||
{
|
||||
let _ = relative;
|
||||
Err(io::Error::new(
|
||||
io::ErrorKind::Unsupported,
|
||||
"safe recursive directory creation is unavailable on this platform",
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
fn warn_reliable_rename_failure(src_file_path: &Path, dst_file_path: &Path, base_dir: &Path, err: &io::Error) {
|
||||
warn!(
|
||||
"reliable_rename failed. src_file_path: {:?}, dst_file_path: {:?}, base_dir: {:?}, err: {:?}",
|
||||
@@ -899,20 +727,6 @@ mod tests {
|
||||
Arc::new(Semaphore::new(MAX_PARALLEL_FILE_SYNCS))
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn disk_volume_mutation_lock_is_shared_per_root_and_volume() {
|
||||
let temp_dir = tempdir().expect("create temp dir");
|
||||
let first = disk_volume_mutation_lock(temp_dir.path(), "bucket");
|
||||
let second = disk_volume_mutation_lock(temp_dir.path(), "bucket");
|
||||
let other = disk_volume_mutation_lock(temp_dir.path(), "other-bucket");
|
||||
|
||||
assert!(Arc::ptr_eq(&first, &second), "reconnected disks must share a bucket mutation lock");
|
||||
assert!(!Arc::ptr_eq(&first, &other), "different buckets must not serialize each other");
|
||||
|
||||
let _write_guard = first.write().await;
|
||||
assert!(second.try_read().is_err(), "a bucket delete lock must exclude local commits");
|
||||
}
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
struct CapturedLogs {
|
||||
buffer: Arc<Mutex<Vec<u8>>>,
|
||||
@@ -957,26 +771,9 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
/// Holds a `warn_capture()` capture alive: the thread-local subscriber, plus
|
||||
/// the pin that keeps tracing's process-global callsite-interest cache from
|
||||
/// being decided by some other test's thread.
|
||||
struct WarnCaptureGuard {
|
||||
_subscriber: tracing::subscriber::DefaultGuard,
|
||||
_callsite_pin: tracing::Dispatch,
|
||||
}
|
||||
|
||||
/// Capture WARN-level output on the current thread; tokio tests here run on
|
||||
/// the current-thread runtime, so the guard covers the whole test body.
|
||||
///
|
||||
/// The callsite pin matters because `warn_reliable_rename_failure` is a
|
||||
/// single production callsite shared with tests that call `rename_all`
|
||||
/// *without* installing a subscriber — `rename_all_missing_source_returns_file_not_found`
|
||||
/// is one. Whichever thread reaches it first fixes its `Interest`
|
||||
/// process-wide, so without the pin that sibling can cache
|
||||
/// `Interest::never()` and the WARN never fires here at all, leaving the
|
||||
/// "must keep the WARN" assertions staring at empty output. See
|
||||
/// [`crate::test_tracing::pin_callsite_interest_for_test`].
|
||||
fn warn_capture() -> (CapturedLogs, WarnCaptureGuard) {
|
||||
fn warn_capture() -> (CapturedLogs, tracing::subscriber::DefaultGuard) {
|
||||
let logs = CapturedLogs::default();
|
||||
let subscriber = tracing_subscriber::fmt()
|
||||
.with_max_level(tracing::Level::WARN)
|
||||
@@ -984,10 +781,7 @@ mod tests {
|
||||
.with_ansi(false)
|
||||
.without_time()
|
||||
.finish();
|
||||
let guard = WarnCaptureGuard {
|
||||
_subscriber: tracing::subscriber::set_default(subscriber),
|
||||
_callsite_pin: crate::test_tracing::pin_callsite_interest_for_test(),
|
||||
};
|
||||
let guard = tracing::subscriber::set_default(subscriber);
|
||||
(logs, guard)
|
||||
}
|
||||
|
||||
@@ -1174,112 +968,6 @@ mod tests {
|
||||
assert_eq!(std::fs::read(dst.join("nested").join("part.1")).expect("read moved part"), b"payload");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rename_all_does_not_recreate_missing_base_directory() {
|
||||
let temp_dir = tempdir().expect("create temp dir");
|
||||
let base = temp_dir.path().join("bucket");
|
||||
std::fs::create_dir(&base).expect("create destination base");
|
||||
let src = temp_dir.path().join("staged-object");
|
||||
std::fs::write(&src, b"payload").expect("write staged object");
|
||||
let dst = base.join("object").join("xl.meta");
|
||||
std::fs::remove_dir(&base).expect("delete destination base before commit");
|
||||
|
||||
let err = rename_all(&src, &dst, &base)
|
||||
.await
|
||||
.expect_err("rename must not recreate a deleted destination base");
|
||||
|
||||
assert!(matches!(err, DiskError::FileNotFound));
|
||||
assert!(src.exists(), "failed commit must preserve the staged source");
|
||||
assert!(!base.exists(), "failed commit must not recreate the deleted bucket");
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn rename_all_rejects_a_replaced_base_with_an_existing_parent() {
|
||||
use std::os::unix::fs::symlink;
|
||||
|
||||
let temp_dir = tempdir().expect("create temp dir");
|
||||
let base = temp_dir.path().join("bucket");
|
||||
let outside = temp_dir.path().join("outside");
|
||||
std::fs::create_dir(&base).expect("create destination base");
|
||||
std::fs::create_dir_all(outside.join("object")).expect("create outside destination parent");
|
||||
let src = temp_dir.path().join("staged-object");
|
||||
std::fs::write(&src, b"payload").expect("write staged object");
|
||||
let dst = base.join("object").join("xl.meta");
|
||||
|
||||
std::fs::remove_dir(&base).expect("remove destination base before replacement");
|
||||
symlink(&outside, &base).expect("replace destination base with a symlink");
|
||||
|
||||
rename_all(&src, &dst, &base)
|
||||
.await
|
||||
.expect_err("rename must reject an existing destination parent below a replaced base");
|
||||
|
||||
assert!(src.exists(), "rejected rename must preserve the staged source");
|
||||
assert!(
|
||||
!outside.join("object/xl.meta").exists(),
|
||||
"rename must not publish through the replacement symlink"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
#[test]
|
||||
fn windows_parent_guard_blocks_base_and_intermediate_replacement() {
|
||||
let temp_dir = tempdir().expect("create temp dir");
|
||||
let base = temp_dir.path().join("bucket");
|
||||
std::fs::create_dir(&base).expect("create destination base");
|
||||
let parent = base.join("object").join("nested");
|
||||
let guard = mkdir_all_below_existing_base_std(&parent, &base).expect("create and lock destination parents");
|
||||
|
||||
std::fs::rename(&base, temp_dir.path().join("replacement-base"))
|
||||
.expect_err("the locked base must not be replaceable before commit");
|
||||
std::fs::rename(base.join("object"), base.join("replacement-object"))
|
||||
.expect_err("a locked intermediate directory must not be replaceable before commit");
|
||||
|
||||
drop(guard);
|
||||
std::fs::rename(base.join("object"), base.join("replacement-object"))
|
||||
.expect("replacement should succeed after the commit guard is released");
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn rename_parent_creation_rejects_symlinked_base() {
|
||||
use std::os::unix::fs::symlink;
|
||||
|
||||
let temp_dir = tempdir().expect("create temp dir");
|
||||
let outside = temp_dir.path().join("outside");
|
||||
std::fs::create_dir(&outside).expect("create outside directory");
|
||||
let base = temp_dir.path().join("bucket");
|
||||
symlink(&outside, &base).expect("create symlinked base");
|
||||
|
||||
mkdir_all_below_existing_base(&base.join("object"), &base)
|
||||
.await
|
||||
.expect_err("symlinked base must be rejected");
|
||||
|
||||
assert!(!outside.join("object").exists(), "parent creation must remain confined to the base");
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn rename_parent_creation_rejects_symlink_below_base() {
|
||||
use std::os::unix::fs::symlink;
|
||||
|
||||
let temp_dir = tempdir().expect("create temp dir");
|
||||
let base = temp_dir.path().join("bucket");
|
||||
let outside = temp_dir.path().join("outside");
|
||||
std::fs::create_dir(&base).expect("create destination base");
|
||||
std::fs::create_dir(&outside).expect("create outside directory");
|
||||
symlink(&outside, base.join("linked")).expect("create symlink below base");
|
||||
|
||||
mkdir_all_below_existing_base(&base.join("linked/object"), &base)
|
||||
.await
|
||||
.expect_err("symlink below base must be rejected");
|
||||
|
||||
assert!(
|
||||
!outside.join("object").exists(),
|
||||
"parent creation must not follow a symlink outside the base"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn fsync_dir_succeeds_on_directory() {
|
||||
let temp_dir = tempdir().expect("create temp dir");
|
||||
|
||||
@@ -203,7 +203,7 @@ fn get_all_sets<T: AsRef<str>>(set_drive_count: usize, is_ellipses: bool, args:
|
||||
for args in set_args.iter() {
|
||||
for arg in args {
|
||||
if unique_args.contains(arg) {
|
||||
return Err(Error::other("input arguments contain a duplicate endpoint after ellipsis expansion"));
|
||||
return Err(Error::other(format!("Input args {arg} has duplicate ellipses")));
|
||||
}
|
||||
unique_args.insert(arg);
|
||||
}
|
||||
@@ -924,15 +924,4 @@ mod test {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn layout_errors_do_not_echo_url_credentials() {
|
||||
for volumes in [
|
||||
vec!["http://:duplicate-secret@server/path", "http://:duplicate-secret@server/path"],
|
||||
vec!["http://:ellipsis...secret@server/path"],
|
||||
] {
|
||||
let err = DisksLayout::from_volumes(&volumes).unwrap_err();
|
||||
assert!(!err.to_string().contains("secret"), "layout error leaked endpoint credentials: {err}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -88,7 +88,6 @@ impl TryFrom<&str> for Endpoint {
|
||||
// - All field should be empty except Host and Path.
|
||||
if !((url.scheme() == "http" || url.scheme() == "https")
|
||||
&& url.username().is_empty()
|
||||
&& url.password().is_none()
|
||||
&& url.fragment().is_none()
|
||||
&& url.query().is_none())
|
||||
{
|
||||
@@ -367,12 +366,6 @@ mod test {
|
||||
expected_type: None,
|
||||
expected_err: Some(Error::other("invalid URL endpoint format")),
|
||||
},
|
||||
TestCase {
|
||||
arg: "http://:topsecret@server/path",
|
||||
expected_endpoint: None,
|
||||
expected_type: None,
|
||||
expected_err: Some(Error::other("invalid URL endpoint format")),
|
||||
},
|
||||
TestCase {
|
||||
arg: "http://:/path",
|
||||
expected_endpoint: None,
|
||||
@@ -512,18 +505,8 @@ mod test {
|
||||
let endpoint = Endpoint::try_from("http://example.com:9000/path").unwrap();
|
||||
assert_eq!(endpoint.host_port(), "example.com:9000");
|
||||
|
||||
for endpoint in [
|
||||
Endpoint::try_from("http://example.com/path").unwrap(),
|
||||
Endpoint::try_from("http://example.com:80/path").unwrap(),
|
||||
] {
|
||||
assert_eq!(endpoint.host_port(), "example.com");
|
||||
}
|
||||
for endpoint in [
|
||||
Endpoint::try_from("https://example.com/path").unwrap(),
|
||||
Endpoint::try_from("https://example.com:443/path").unwrap(),
|
||||
] {
|
||||
assert_eq!(endpoint.host_port(), "example.com");
|
||||
}
|
||||
let endpoint_no_port = Endpoint::try_from("https://example.com/path").unwrap();
|
||||
assert_eq!(endpoint_no_port.host_port(), "example.com");
|
||||
|
||||
let file_endpoint = Endpoint::try_from("/tmp/data").unwrap();
|
||||
assert_eq!(file_endpoint.host_port(), "");
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -18,7 +18,7 @@ use serde::{Deserialize, Serialize};
|
||||
use serde_json::Error as JsonError;
|
||||
use uuid::Uuid;
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq, Hash)]
|
||||
#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)]
|
||||
pub enum FormatMetaVersion {
|
||||
#[serde(rename = "1")]
|
||||
V1,
|
||||
@@ -27,7 +27,7 @@ pub enum FormatMetaVersion {
|
||||
Unknown,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq, Hash)]
|
||||
#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)]
|
||||
pub enum FormatBackend {
|
||||
#[serde(rename = "xl")]
|
||||
Erasure,
|
||||
@@ -64,7 +64,7 @@ pub struct FormatErasureV3 {
|
||||
pub distribution_algo: DistributionAlgoVersion,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq, Hash)]
|
||||
#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)]
|
||||
pub enum FormatErasureVersion {
|
||||
#[serde(rename = "1")]
|
||||
V1,
|
||||
@@ -77,7 +77,7 @@ pub enum FormatErasureVersion {
|
||||
Unknown,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq, Hash)]
|
||||
#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)]
|
||||
pub enum DistributionAlgoVersion {
|
||||
#[serde(rename = "CRCMOD")]
|
||||
V1,
|
||||
@@ -121,15 +121,6 @@ pub struct FormatV3 {
|
||||
pub disk_info: Option<DiskInfo>,
|
||||
}
|
||||
|
||||
pub(crate) type SharedFormatIdentity<'a> = (
|
||||
&'a FormatMetaVersion,
|
||||
&'a FormatBackend,
|
||||
&'a Uuid,
|
||||
&'a FormatErasureVersion,
|
||||
&'a [Vec<Uuid>],
|
||||
&'a DistributionAlgoVersion,
|
||||
);
|
||||
|
||||
impl TryFrom<&[u8]> for FormatV3 {
|
||||
type Error = JsonError;
|
||||
|
||||
@@ -207,24 +198,52 @@ impl FormatV3 {
|
||||
}
|
||||
|
||||
pub fn check_other(&self, other: &FormatV3) -> Result<()> {
|
||||
if self.shared_identity() != other.shared_identity() {
|
||||
return Err(Error::other("storage formats do not match"));
|
||||
let mut tmp = other.clone();
|
||||
let this = tmp.erasure.this;
|
||||
tmp.erasure.this = Uuid::nil();
|
||||
|
||||
if self.erasure.sets.len() != other.erasure.sets.len() {
|
||||
return Err(Error::other(format!(
|
||||
"Expected number of sets {}, got {}",
|
||||
self.erasure.sets.len(),
|
||||
other.erasure.sets.len()
|
||||
)));
|
||||
}
|
||||
|
||||
self.find_disk_index_by_disk_id(other.erasure.this).map(|_| ())
|
||||
}
|
||||
for i in 0..self.erasure.sets.len() {
|
||||
if self.erasure.sets[i].len() != other.erasure.sets[i].len() {
|
||||
return Err(Error::other(format!(
|
||||
"Each set should be of same size, expected {}, got {}",
|
||||
self.erasure.sets[i].len(),
|
||||
other.erasure.sets[i].len()
|
||||
)));
|
||||
}
|
||||
|
||||
/// Fields that must agree across every disk in one erasure format,
|
||||
/// excluding the disk-specific `this` UUID and runtime-only `disk_info`.
|
||||
pub(crate) fn shared_identity(&self) -> SharedFormatIdentity<'_> {
|
||||
(
|
||||
&self.version,
|
||||
&self.format,
|
||||
&self.id,
|
||||
&self.erasure.version,
|
||||
&self.erasure.sets,
|
||||
&self.erasure.distribution_algo,
|
||||
)
|
||||
for j in 0..self.erasure.sets[i].len() {
|
||||
if self.erasure.sets[i][j] != other.erasure.sets[i][j] {
|
||||
return Err(Error::other(format!(
|
||||
"UUID on positions {}:{} do not match with, expected {:?} got {:?}: (%w)",
|
||||
i,
|
||||
j,
|
||||
self.erasure.sets[i][j].to_string(),
|
||||
other.erasure.sets[i][j].to_string(),
|
||||
)));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for i in 0..tmp.erasure.sets.len() {
|
||||
for j in 0..tmp.erasure.sets[i].len() {
|
||||
if this == tmp.erasure.sets[i][j] {
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Err(Error::other(format!(
|
||||
"DriveID {:?} not found in any drive sets {:?}",
|
||||
this, other.erasure.sets
|
||||
)))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -418,30 +437,6 @@ mod test {
|
||||
assert!(result.is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_check_other_rejects_shared_identity_mismatches() {
|
||||
type FormatMutation = (&'static str, fn(&mut FormatV3));
|
||||
|
||||
let format = FormatV3::new(1, 2);
|
||||
let mutations: [FormatMutation; 5] = [
|
||||
("meta version", |other| other.version = FormatMetaVersion::Unknown),
|
||||
("backend", |other| other.format = FormatBackend::ErasureSingle),
|
||||
("deployment id", |other| other.id = Uuid::new_v4()),
|
||||
("erasure version", |other| other.erasure.version = FormatErasureVersion::V2),
|
||||
("distribution algorithm", |other| {
|
||||
other.erasure.distribution_algo = DistributionAlgoVersion::V2
|
||||
}),
|
||||
];
|
||||
|
||||
for (field, mutate) in mutations {
|
||||
let mut other = format.clone();
|
||||
other.erasure.this = format.erasure.sets[0][0];
|
||||
mutate(&mut other);
|
||||
|
||||
assert!(format.check_other(&other).is_err(), "{field} mismatch must be rejected");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_check_other_different_set_count() {
|
||||
let format1 = FormatV3::new(2, 4);
|
||||
|
||||
@@ -93,6 +93,3 @@ pub(crate) mod ecstore_validation_blackbox;
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) mod test_metrics;
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) mod test_tracing;
|
||||
|
||||
@@ -424,11 +424,7 @@ pub(crate) async fn record_local_disk_id(instance_ctx: &Arc<InstanceContext>, di
|
||||
pub(crate) async fn replace_local_disk_id(previous: Option<Uuid>, current: Option<Uuid>, endpoint: String) {
|
||||
let id_map = local_disk_id_map_handle();
|
||||
let mut disk_id_map = id_map.write().await;
|
||||
if let Some(previous_id) = previous
|
||||
&& disk_id_map
|
||||
.get(&previous_id)
|
||||
.is_some_and(|registered_endpoint| registered_endpoint == &endpoint)
|
||||
{
|
||||
if let Some(previous_id) = previous {
|
||||
disk_id_map.remove(&previous_id);
|
||||
}
|
||||
if let Some(current_id) = current {
|
||||
@@ -562,14 +558,10 @@ pub(crate) async fn init_tier_config_mgr(store: Arc<ECStore>) -> Result<()> {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{
|
||||
LockRegistry, clear_local_disk_id_map_for_test, local_disk_path_by_id, local_node_name, replace_local_disk_id,
|
||||
set_local_node_name,
|
||||
};
|
||||
use super::{LockRegistry, local_node_name, set_local_node_name};
|
||||
use crate::disk::endpoint::Endpoint;
|
||||
use rustfs_lock::{LocalClient, LockClient};
|
||||
use std::{collections::HashMap, sync::Arc};
|
||||
use uuid::Uuid;
|
||||
|
||||
fn url_endpoint(raw: &str) -> Endpoint {
|
||||
Endpoint {
|
||||
@@ -615,17 +607,4 @@ mod tests {
|
||||
|
||||
assert_eq!(observed, next);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn clearing_a_stale_disk_id_does_not_remove_another_endpoint() {
|
||||
clear_local_disk_id_map_for_test().await;
|
||||
let disk_id = Uuid::new_v4();
|
||||
replace_local_disk_id(None, Some(disk_id), "endpoint-a".to_string()).await;
|
||||
|
||||
replace_local_disk_id(Some(disk_id), None, "endpoint-b".to_string()).await;
|
||||
|
||||
assert_eq!(local_disk_path_by_id(&disk_id).await, Some("endpoint-a".to_string()));
|
||||
clear_local_disk_id_map_for_test().await;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -29,11 +29,11 @@ use rustfs_madmin::metrics::RealtimeMetrics;
|
||||
use rustfs_madmin::net::NetInfo;
|
||||
use rustfs_madmin::{ItemState, ServerProperties, StorageInfo};
|
||||
use rustfs_utils::XHost;
|
||||
use std::collections::{BTreeMap, HashMap, hash_map::DefaultHasher};
|
||||
use std::collections::{HashMap, hash_map::DefaultHasher};
|
||||
use std::future::Future;
|
||||
use std::hash::{Hash, Hasher};
|
||||
use std::sync::{Arc, Mutex, OnceLock};
|
||||
use std::time::{Duration, Instant, SystemTime};
|
||||
use std::time::{Duration, SystemTime};
|
||||
use tokio::time::{sleep, timeout};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing::{debug, error, info, warn};
|
||||
@@ -47,9 +47,6 @@ const EVENT_NOTIFICATION_PEER_PROPAGATION: &str = "notification_peer_propagation
|
||||
const SCANNER_ACTIVITY_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
const TIER_CONFIG_RELOAD_RETRY_BASE: Duration = Duration::from_millis(100);
|
||||
const TIER_CONFIG_RELOAD_RETRY_CAP: Duration = Duration::from_secs(5);
|
||||
const REMOTE_VERSION_STATE_PROBE_INTERVAL: Duration = Duration::from_secs(10);
|
||||
const REMOTE_VERSION_STATE_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
const REMOTE_VERSION_STATE_PROOF_TTL: Duration = Duration::from_secs(30);
|
||||
|
||||
/// Cached result from the last successful admin call to a peer.
|
||||
struct PeerAdminCache {
|
||||
@@ -94,180 +91,6 @@ lazy_static! {
|
||||
pub static ref GLOBAL_NOTIFICATION_SYS: OnceLock<Arc<NotificationSys>> = OnceLock::new();
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct RemoteVersionStateFleetProof {
|
||||
topology_fingerprint: String,
|
||||
peer_epochs: Arc<BTreeMap<String, Uuid>>,
|
||||
expires_at: Instant,
|
||||
}
|
||||
|
||||
impl RemoteVersionStateFleetProof {
|
||||
fn token(&self) -> RemoteVersionStateFleetProofToken {
|
||||
RemoteVersionStateFleetProofToken {
|
||||
topology_fingerprint: self.topology_fingerprint.clone(),
|
||||
peer_epochs: self.peer_epochs.clone(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, PartialEq, Eq)]
|
||||
pub(crate) struct RemoteVersionStateFleetProofToken {
|
||||
topology_fingerprint: String,
|
||||
peer_epochs: Arc<BTreeMap<String, Uuid>>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct RemoteVersionStateFleetProofState {
|
||||
proof: Option<RemoteVersionStateFleetProof>,
|
||||
topology_conflict: bool,
|
||||
}
|
||||
|
||||
static REMOTE_VERSION_STATE_FLEET_PROOF: OnceLock<std::sync::RwLock<RemoteVersionStateFleetProofState>> = OnceLock::new();
|
||||
static REMOTE_VERSION_STATE_PROBE_TOPOLOGY: OnceLock<String> = OnceLock::new();
|
||||
|
||||
fn remote_version_state_fleet_proof_slot() -> &'static std::sync::RwLock<RemoteVersionStateFleetProofState> {
|
||||
REMOTE_VERSION_STATE_FLEET_PROOF.get_or_init(|| std::sync::RwLock::new(RemoteVersionStateFleetProofState::default()))
|
||||
}
|
||||
|
||||
fn replace_remote_version_state_fleet_proof(proof: Option<RemoteVersionStateFleetProof>) {
|
||||
replace_remote_version_state_fleet_proof_in(remote_version_state_fleet_proof_slot(), proof);
|
||||
}
|
||||
|
||||
fn replace_remote_version_state_fleet_proof_in(
|
||||
slot: &std::sync::RwLock<RemoteVersionStateFleetProofState>,
|
||||
proof: Option<RemoteVersionStateFleetProof>,
|
||||
) {
|
||||
slot.write().unwrap_or_else(std::sync::PoisonError::into_inner).proof = proof;
|
||||
}
|
||||
|
||||
fn publish_remote_version_state_probe_result(
|
||||
slot: &std::sync::RwLock<RemoteVersionStateFleetProofState>,
|
||||
topology_fingerprint: &str,
|
||||
result: Result<BTreeMap<String, Uuid>>,
|
||||
observed_at: Instant,
|
||||
) -> Option<Error> {
|
||||
match result {
|
||||
Ok(peer_epochs) => {
|
||||
let mut state = slot.write().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
let peer_epochs = state
|
||||
.proof
|
||||
.as_ref()
|
||||
.filter(|proof| proof.topology_fingerprint == topology_fingerprint && proof.peer_epochs.as_ref() == &peer_epochs)
|
||||
.map(|proof| Arc::clone(&proof.peer_epochs))
|
||||
.unwrap_or_else(|| Arc::new(peer_epochs));
|
||||
state.proof = Some(RemoteVersionStateFleetProof {
|
||||
topology_fingerprint: topology_fingerprint.to_string(),
|
||||
peer_epochs,
|
||||
expires_at: observed_at + REMOTE_VERSION_STATE_PROOF_TTL,
|
||||
});
|
||||
None
|
||||
}
|
||||
Err(err) => {
|
||||
replace_remote_version_state_fleet_proof_in(slot, None);
|
||||
Some(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn acquire_remote_version_state_fleet_proof() -> Option<RemoteVersionStateFleetProofToken> {
|
||||
let expected_topology = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get()?;
|
||||
let state = remote_version_state_fleet_proof_slot()
|
||||
.read()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
acquire_remote_version_state_fleet_proof_from(&state, expected_topology, Instant::now())
|
||||
}
|
||||
|
||||
fn acquire_remote_version_state_fleet_proof_from(
|
||||
state: &RemoteVersionStateFleetProofState,
|
||||
expected_topology: &str,
|
||||
now: Instant,
|
||||
) -> Option<RemoteVersionStateFleetProofToken> {
|
||||
if state.topology_conflict || !remote_version_state_fleet_proof_valid_at(state.proof.as_ref(), expected_topology, now) {
|
||||
return None;
|
||||
}
|
||||
state.proof.as_ref().map(RemoteVersionStateFleetProof::token)
|
||||
}
|
||||
|
||||
pub(crate) fn remote_version_state_fleet_proof_matches(proof: &RemoteVersionStateFleetProofToken) -> bool {
|
||||
let Some(expected_topology) = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get() else {
|
||||
return false;
|
||||
};
|
||||
let state = remote_version_state_fleet_proof_slot()
|
||||
.read()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
if state.topology_conflict {
|
||||
return false;
|
||||
}
|
||||
state.proof.as_ref().is_some_and(|current| {
|
||||
current.topology_fingerprint == *expected_topology
|
||||
&& current.topology_fingerprint == proof.topology_fingerprint
|
||||
&& Arc::ptr_eq(¤t.peer_epochs, &proof.peer_epochs)
|
||||
&& Instant::now() < current.expires_at
|
||||
})
|
||||
}
|
||||
|
||||
fn remote_version_state_fleet_proof_valid_at(
|
||||
proof: Option<&RemoteVersionStateFleetProof>,
|
||||
expected_topology: &str,
|
||||
now: Instant,
|
||||
) -> bool {
|
||||
proof.is_some_and(|proof| proof.topology_fingerprint == expected_topology && now < proof.expires_at)
|
||||
}
|
||||
|
||||
fn insert_remote_version_state_peer(peer_epochs: &mut BTreeMap<String, Uuid>, peer: String, epoch: Uuid) -> Result<()> {
|
||||
if epoch.is_nil() || peer_epochs.values().any(|existing| *existing == epoch) || peer_epochs.insert(peer, epoch).is_some() {
|
||||
return Err(Error::other("remote version state capability peer identity is invalid"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
|
||||
if REMOTE_VERSION_STATE_PROBE_TOPOLOGY.set(topology_fingerprint.clone()).is_err() {
|
||||
if REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get() != Some(&topology_fingerprint) {
|
||||
let mut state = remote_version_state_fleet_proof_slot()
|
||||
.write()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
state.topology_conflict = true;
|
||||
state.proof = None;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
tokio::spawn(async move {
|
||||
loop {
|
||||
let result = match get_global_notification_sys() {
|
||||
Some(notification_sys) => {
|
||||
match timeout(
|
||||
REMOTE_VERSION_STATE_PROBE_TIMEOUT,
|
||||
notification_sys.probe_remote_version_state_fleet(&topology_fingerprint),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(result) => result,
|
||||
Err(_) => Err(Error::other("remote version state fleet capability probe timed out")),
|
||||
}
|
||||
}
|
||||
None => Err(Error::other("remote version state fleet capability notification system is unavailable")),
|
||||
};
|
||||
let topology_conflict = remote_version_state_fleet_proof_slot()
|
||||
.read()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.topology_conflict;
|
||||
if topology_conflict {
|
||||
replace_remote_version_state_fleet_proof(None);
|
||||
} else if let Some(err) = publish_remote_version_state_probe_result(
|
||||
remote_version_state_fleet_proof_slot(),
|
||||
&topology_fingerprint,
|
||||
result,
|
||||
Instant::now(),
|
||||
) {
|
||||
debug!(error = %err, "remote version state fleet capability probe failed closed");
|
||||
}
|
||||
sleep(REMOTE_VERSION_STATE_PROBE_INTERVAL).await;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
pub async fn new_global_notification_sys(eps: EndpointServerPools) -> Result<()> {
|
||||
let _ = GLOBAL_NOTIFICATION_SYS
|
||||
.set(Arc::new(NotificationSys::new(eps).await))
|
||||
@@ -292,17 +115,7 @@ pub struct NotificationSys {
|
||||
|
||||
impl NotificationSys {
|
||||
pub async fn new(eps: EndpointServerPools) -> Self {
|
||||
let expected_remote_hosts = eps
|
||||
.peer_grid_host_slots_sorted()
|
||||
.into_iter()
|
||||
.filter_map(|(peer, _, is_local)| (!is_local).then_some(peer))
|
||||
.collect::<Vec<_>>();
|
||||
let (peer_clients, all_peer_clients, peer_topology_hosts) = PeerRestClient::new_clients_with_topology(eps).await;
|
||||
let peer_topology_hosts = if peer_topology_hosts.is_empty() {
|
||||
expected_remote_hosts
|
||||
} else {
|
||||
peer_topology_hosts
|
||||
};
|
||||
let peer_admin_caches = (0..peer_clients.len()).map(|_| Mutex::new(PeerAdminCache::new())).collect();
|
||||
Self {
|
||||
peer_clients,
|
||||
@@ -312,24 +125,6 @@ impl NotificationSys {
|
||||
tier_config_reload_workers: Default::default(),
|
||||
}
|
||||
}
|
||||
|
||||
async fn probe_remote_version_state_fleet(&self, topology_fingerprint: &str) -> Result<BTreeMap<String, Uuid>> {
|
||||
if self.peer_clients.len() != self.peer_topology_hosts.len() {
|
||||
return Err(Error::other("remote version state capability fleet membership is incomplete"));
|
||||
}
|
||||
let probes = self.peer_clients.iter().map(|client| async {
|
||||
let client = client
|
||||
.as_ref()
|
||||
.ok_or_else(|| Error::other("remote version state capability peer is unreachable"))?;
|
||||
client.probe_remote_version_state(topology_fingerprint.to_string()).await
|
||||
});
|
||||
let mut peer_epochs = BTreeMap::new();
|
||||
for result in join_all(probes).await {
|
||||
let (peer, epoch) = result?;
|
||||
insert_remote_version_state_peer(&mut peer_epochs, peer, epoch)?;
|
||||
}
|
||||
Ok(peer_epochs)
|
||||
}
|
||||
}
|
||||
|
||||
pub struct NotificationPeerErr {
|
||||
@@ -2044,10 +1839,15 @@ fn synthesized_disks(host: &str, endpoints: &EndpointServerPools, state: ItemSta
|
||||
/// Whether `peer_host` refers to the same node as an endpoint whose
|
||||
/// `host_port()` is `ep_host_port`.
|
||||
///
|
||||
/// Current topology clients preserve the endpoint `hostname:port`, so the
|
||||
/// direct comparison is the normal path. The resolution fallback keeps
|
||||
/// compatibility with older or manually constructed clients whose `XHost`
|
||||
/// contains a resolved `IP:port` (rustfs/rustfs#4607 follow-up).
|
||||
/// `PeerRestClient::host` is an `XHost`, which resolves names to an address on
|
||||
/// construction (`hosts_sorted` -> `XHost::try_from` -> `to_socket_addrs`), so
|
||||
/// `peer_host` is the resolved `IP:port`. An endpoint's `host_port()`, however,
|
||||
/// is `url.host():port` — still the raw `hostname:port` on hostname-based
|
||||
/// deployments. A plain string compare therefore misses on hostname clusters,
|
||||
/// leaving the synthesized/degraded drive list empty and `unknownDisks` at 0
|
||||
/// (rustfs/rustfs#4607 follow-up). Compare directly first (fast path / IP
|
||||
/// deployments), then canonicalize the endpoint side through the same `XHost`
|
||||
/// resolution and compare again.
|
||||
fn endpoint_host_matches(peer_host: &str, ep_host_port: &str) -> bool {
|
||||
if peer_host == ep_host_port {
|
||||
return true;
|
||||
@@ -2090,183 +1890,6 @@ fn aggregate_scanner_dirty_usage_acknowledgement_results(
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn remote_version_state_fleet_proof_rejects_stale_or_mismatched_membership() {
|
||||
let now = Instant::now();
|
||||
let mut peer_epochs = BTreeMap::new();
|
||||
peer_epochs.insert("peer-a".to_string(), Uuid::new_v4());
|
||||
let proof = RemoteVersionStateFleetProof {
|
||||
topology_fingerprint: "topology-a".to_string(),
|
||||
peer_epochs: Arc::new(peer_epochs),
|
||||
expires_at: now + Duration::from_secs(1),
|
||||
};
|
||||
|
||||
assert!(remote_version_state_fleet_proof_valid_at(Some(&proof), "topology-a", now));
|
||||
assert!(!remote_version_state_fleet_proof_valid_at(Some(&proof), "topology-b", now));
|
||||
assert!(!remote_version_state_fleet_proof_valid_at(Some(&proof), "topology-a", proof.expires_at));
|
||||
assert!(!remote_version_state_fleet_proof_valid_at(None, "topology-a", now));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_version_state_fleet_proof_rejects_nil_process_epoch() {
|
||||
let mut peer_epochs = BTreeMap::new();
|
||||
|
||||
assert!(insert_remote_version_state_peer(&mut peer_epochs, "peer-a".to_string(), Uuid::nil()).is_err());
|
||||
assert!(peer_epochs.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_version_state_fleet_proof_accepts_single_node_membership() {
|
||||
let now = Instant::now();
|
||||
let proof = RemoteVersionStateFleetProof {
|
||||
topology_fingerprint: "topology-a".to_string(),
|
||||
peer_epochs: Arc::new(BTreeMap::new()),
|
||||
expires_at: now + Duration::from_secs(1),
|
||||
};
|
||||
|
||||
assert!(remote_version_state_fleet_proof_valid_at(Some(&proof), "topology-a", now));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_version_state_fleet_proof_token_changes_with_process_epoch() {
|
||||
let now = Instant::now();
|
||||
let proof = RemoteVersionStateFleetProof {
|
||||
topology_fingerprint: "topology-a".to_string(),
|
||||
peer_epochs: Arc::new(BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())])),
|
||||
expires_at: now + Duration::from_secs(1),
|
||||
};
|
||||
let captured = proof.token();
|
||||
let restarted = RemoteVersionStateFleetProof {
|
||||
topology_fingerprint: proof.topology_fingerprint.clone(),
|
||||
peer_epochs: Arc::new(BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())])),
|
||||
expires_at: proof.expires_at,
|
||||
};
|
||||
|
||||
assert!(captured != restarted.token());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_version_state_fleet_proof_renewal_preserves_only_same_epoch_token() {
|
||||
let slot = std::sync::RwLock::new(RemoteVersionStateFleetProofState::default());
|
||||
let now = Instant::now();
|
||||
let epoch = Uuid::new_v4();
|
||||
let peers = BTreeMap::from([("peer-a".to_string(), epoch)]);
|
||||
assert!(publish_remote_version_state_probe_result(&slot, "topology-a", Ok(peers.clone()), now).is_none());
|
||||
let original = slot
|
||||
.read()
|
||||
.expect("proof slot should not poison")
|
||||
.proof
|
||||
.as_ref()
|
||||
.expect("successful probe should publish proof")
|
||||
.token();
|
||||
|
||||
assert!(
|
||||
publish_remote_version_state_probe_result(&slot, "topology-a", Ok(peers), now + Duration::from_millis(1)).is_none()
|
||||
);
|
||||
let renewed = slot
|
||||
.read()
|
||||
.expect("proof slot should not poison")
|
||||
.proof
|
||||
.as_ref()
|
||||
.expect("renewal should retain proof")
|
||||
.token();
|
||||
assert!(Arc::ptr_eq(&original.peer_epochs, &renewed.peer_epochs));
|
||||
|
||||
let restarted = BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())]);
|
||||
assert!(
|
||||
publish_remote_version_state_probe_result(&slot, "topology-a", Ok(restarted), now + Duration::from_millis(2))
|
||||
.is_none()
|
||||
);
|
||||
let replaced = slot
|
||||
.read()
|
||||
.expect("proof slot should not poison")
|
||||
.proof
|
||||
.as_ref()
|
||||
.expect("restarted peer should publish a new proof")
|
||||
.token();
|
||||
assert!(!Arc::ptr_eq(&original.peer_epochs, &replaced.peer_epochs));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_version_state_fleet_proof_conflict_revokes_atomic_snapshot() {
|
||||
let now = Instant::now();
|
||||
let mut state = RemoteVersionStateFleetProofState {
|
||||
proof: Some(RemoteVersionStateFleetProof {
|
||||
topology_fingerprint: "topology-a".to_string(),
|
||||
peer_epochs: Arc::new(BTreeMap::new()),
|
||||
expires_at: now + Duration::from_secs(1),
|
||||
}),
|
||||
topology_conflict: false,
|
||||
};
|
||||
assert!(acquire_remote_version_state_fleet_proof_from(&state, "topology-a", now).is_some());
|
||||
|
||||
state.topology_conflict = true;
|
||||
assert!(acquire_remote_version_state_fleet_proof_from(&state, "topology-a", now).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_version_state_fleet_probe_rejects_duplicate_member_or_process_epoch() {
|
||||
let epoch = Uuid::new_v4();
|
||||
let mut peer_epochs = BTreeMap::new();
|
||||
insert_remote_version_state_peer(&mut peer_epochs, "node-a:9000".to_string(), epoch)
|
||||
.expect("first member should be admitted");
|
||||
assert!(insert_remote_version_state_peer(&mut peer_epochs, "node-b:9000".to_string(), epoch).is_err());
|
||||
assert!(insert_remote_version_state_peer(&mut peer_epochs, "node-a:9000".to_string(), Uuid::new_v4()).is_err());
|
||||
assert!(insert_remote_version_state_peer(&mut peer_epochs, "node-c:9000".to_string(), Uuid::nil()).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_version_state_fleet_probe_failure_revokes_previous_proof() {
|
||||
let slot = std::sync::RwLock::new(RemoteVersionStateFleetProofState::default());
|
||||
let now = Instant::now();
|
||||
let peer_epochs = BTreeMap::from([("node-a:9000".to_string(), Uuid::new_v4())]);
|
||||
assert!(publish_remote_version_state_probe_result(&slot, "topology-a", Ok(peer_epochs), now).is_none());
|
||||
assert!(slot.read().expect("proof slot should not poison").proof.is_some());
|
||||
|
||||
assert!(
|
||||
publish_remote_version_state_probe_result(&slot, "topology-a", Err(Error::other("peer unavailable")), now,).is_some()
|
||||
);
|
||||
assert!(slot.read().expect("proof slot should not poison").proof.is_none());
|
||||
|
||||
let peer_epochs = BTreeMap::from([("node-a:9000".to_string(), Uuid::new_v4())]);
|
||||
assert!(publish_remote_version_state_probe_result(&slot, "topology-a", Ok(peer_epochs), now).is_none());
|
||||
assert!(slot.read().expect("proof slot should not poison").proof.is_some());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn remote_version_state_fleet_probe_rejects_unreachable_member() {
|
||||
let notification_sys = NotificationSys {
|
||||
peer_clients: vec![None],
|
||||
all_peer_clients: vec![None, None],
|
||||
peer_topology_hosts: vec!["peer-a".to_string()],
|
||||
peer_admin_caches: vec![Mutex::new(PeerAdminCache::new())],
|
||||
tier_config_reload_workers: Default::default(),
|
||||
};
|
||||
|
||||
let err = notification_sys
|
||||
.probe_remote_version_state_fleet("topology-a")
|
||||
.await
|
||||
.expect_err("an unreachable configured member must fail the fleet proof");
|
||||
assert!(err.to_string().contains("unreachable"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn remote_version_state_fleet_probe_rejects_missing_member_slot() {
|
||||
let notification_sys = NotificationSys {
|
||||
peer_clients: Vec::new(),
|
||||
all_peer_clients: vec![None],
|
||||
peer_topology_hosts: vec!["peer-a".to_string()],
|
||||
peer_admin_caches: Vec::new(),
|
||||
tier_config_reload_workers: Default::default(),
|
||||
};
|
||||
|
||||
let err = notification_sys
|
||||
.probe_remote_version_state_fleet("topology-a")
|
||||
.await
|
||||
.expect_err("a missing configured member slot must fail the fleet proof");
|
||||
assert!(err.to_string().contains("incomplete"));
|
||||
}
|
||||
|
||||
fn build_props(endpoint: &str) -> ServerProperties {
|
||||
ServerProperties {
|
||||
endpoint: endpoint.to_string(),
|
||||
|
||||
@@ -72,7 +72,6 @@ use crate::{
|
||||
cluster::rpc::peer_rest_client::{PeerRestClient, PeerTierMutationState},
|
||||
config::com::{CONFIG_PREFIX, read_config, read_config_with_metadata},
|
||||
disk::{MIGRATING_META_BUCKET, RUSTFS_META_BUCKET},
|
||||
layout::endpoints::EndpointServerPools,
|
||||
object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader},
|
||||
runtime::sources as runtime_sources,
|
||||
set_disk::get_lock_acquire_timeout,
|
||||
@@ -237,7 +236,6 @@ struct TierPublishTransition {
|
||||
|
||||
struct PreparedTierDriver {
|
||||
tier_name: String,
|
||||
tier_config: TierConfig,
|
||||
config_fingerprint: TierDriverFingerprint,
|
||||
backend_identity: TierDestinationId,
|
||||
exact_get_delete: bool,
|
||||
@@ -905,17 +903,14 @@ async fn remote_tier_mutation_peers() -> io::Result<Vec<Arc<dyn TierMutationPeer
|
||||
let Some(endpoints) = runtime_sources::endpoint_pools() else {
|
||||
return Err(tier_mutation_replay_error("cluster endpoint topology is not initialized"));
|
||||
};
|
||||
remote_tier_mutation_peers_from_topology(endpoints).await
|
||||
}
|
||||
|
||||
async fn remote_tier_mutation_peers_from_topology(endpoints: EndpointServerPools) -> io::Result<Vec<Arc<dyn TierMutationPeer>>> {
|
||||
let (peers, _, remote_topology_hosts) = PeerRestClient::new_clients_with_topology(endpoints).await;
|
||||
let remote_host_count = endpoints.hosts_sorted().iter().flatten().count();
|
||||
let (peers, _) = PeerRestClient::new_clients(endpoints).await;
|
||||
let peers = peers
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.map(|peer| Arc::new(peer) as Arc<dyn TierMutationPeer>)
|
||||
.collect::<Vec<_>>();
|
||||
ensure_complete_tier_mutation_commit_peer_set(peers.len(), remote_topology_hosts.len())?;
|
||||
ensure_complete_tier_mutation_commit_peer_set(peers.len(), remote_host_count)?;
|
||||
Ok(peers)
|
||||
}
|
||||
|
||||
@@ -1347,7 +1342,6 @@ fn tier_exact_get_delete(config: &TierConfig) -> bool {
|
||||
|
||||
struct TierDriverGeneration {
|
||||
tier_name: Arc<str>,
|
||||
tier_config: TierConfig,
|
||||
generation: DriverRevision,
|
||||
// Process-local only: this may reflect credential changes and must never be persisted or logged.
|
||||
config_fingerprint: TierDriverFingerprint,
|
||||
@@ -1355,9 +1349,6 @@ struct TierDriverGeneration {
|
||||
backend_identity: TierDestinationId,
|
||||
exact_get_delete: bool,
|
||||
driver: SharedWarmBackend,
|
||||
reconciler: tokio::sync::OnceCell<
|
||||
Option<Arc<dyn crate::services::tier::warm_backend::TransitionCandidateReconciler + Send + Sync + 'static>>,
|
||||
>,
|
||||
accepting: AtomicBool,
|
||||
active_leases: AtomicUsize,
|
||||
drained: tokio::sync::Notify,
|
||||
@@ -1482,35 +1473,6 @@ impl TierOperationLease {
|
||||
self.inner.backend_identity
|
||||
}
|
||||
|
||||
pub(crate) async fn probe_transition_candidate_for(
|
||||
&self,
|
||||
object: &str,
|
||||
transaction_id: uuid::Uuid,
|
||||
) -> io::Result<TransitionCandidateProbe> {
|
||||
let Some(reconciler) = self
|
||||
.inner
|
||||
.reconciler
|
||||
.get_or_try_init(|| async {
|
||||
crate::services::tier::warm_backend::new_transition_candidate_reconciler(&self.inner.tier_config)
|
||||
.await
|
||||
.map(|reconciler| reconciler.map(Arc::from))
|
||||
})
|
||||
.await
|
||||
.map_err(|err| io::Error::other(err.message))?
|
||||
else {
|
||||
return self.inner.driver.probe_transition_candidate(object).await;
|
||||
};
|
||||
reconciler
|
||||
.probe_transition_candidate_for(
|
||||
object,
|
||||
crate::services::tier::warm_backend::TransitionCandidateIdentity {
|
||||
transaction_id,
|
||||
destination_id: self.backend_identity(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) fn validate_remote_version_id(&self, remote_version_id: &str) -> io::Result<()> {
|
||||
self.inner.driver.validate_remote_version_id(remote_version_id)?;
|
||||
if !remote_version_id.is_empty() && !self.inner.exact_get_delete {
|
||||
@@ -2871,7 +2833,6 @@ impl TierConfigMgr {
|
||||
let exact_get_delete = tier_exact_get_delete(config);
|
||||
Some(PreparedTierDriver {
|
||||
tier_name: tier_name.to_string(),
|
||||
tier_config: config.clone(),
|
||||
config_fingerprint,
|
||||
backend_identity,
|
||||
exact_get_delete,
|
||||
@@ -2900,13 +2861,11 @@ impl TierConfigMgr {
|
||||
})?;
|
||||
let entry = Arc::new(TierDriverGeneration {
|
||||
tier_name: Arc::from(prepared.tier_name.as_str()),
|
||||
tier_config: prepared.tier_config.clone(),
|
||||
generation,
|
||||
config_fingerprint: prepared.config_fingerprint,
|
||||
backend_identity: prepared.backend_identity,
|
||||
exact_get_delete: prepared.exact_get_delete,
|
||||
driver: prepared.driver.clone(),
|
||||
reconciler: tokio::sync::OnceCell::new(),
|
||||
accepting: AtomicBool::new(true),
|
||||
active_leases: AtomicUsize::new(0),
|
||||
drained: tokio::sync::Notify::new(),
|
||||
@@ -3650,13 +3609,11 @@ impl TierConfigMgr {
|
||||
let driver: SharedWarmBackend = Arc::from(driver);
|
||||
let entry = Arc::new(TierDriverGeneration {
|
||||
tier_name: Arc::from(tier_name),
|
||||
tier_config: config.clone(),
|
||||
generation,
|
||||
config_fingerprint,
|
||||
backend_identity,
|
||||
exact_get_delete,
|
||||
driver: driver.clone(),
|
||||
reconciler: tokio::sync::OnceCell::new(),
|
||||
accepting: AtomicBool::new(true),
|
||||
active_leases: AtomicUsize::new(0),
|
||||
drained: tokio::sync::Notify::new(),
|
||||
@@ -4311,34 +4268,6 @@ fn tier_config_not_initialized_error(operation: &str) -> std::io::Error {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::layout::{
|
||||
endpoint::Endpoint,
|
||||
endpoints::{Endpoints, PoolEndpoints, SetupType},
|
||||
};
|
||||
|
||||
struct SetupTypeGuard {
|
||||
previous: SetupType,
|
||||
}
|
||||
|
||||
impl SetupTypeGuard {
|
||||
async fn switch_to(next: SetupType) -> Self {
|
||||
let previous = runtime_sources::current_setup_type().await;
|
||||
runtime_sources::set_setup_type(next).await;
|
||||
Self { previous }
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for SetupTypeGuard {
|
||||
fn drop(&mut self) {
|
||||
let previous = self.previous.clone();
|
||||
let handle = tokio::runtime::Handle::current();
|
||||
tokio::task::block_in_place(|| {
|
||||
handle.block_on(async move {
|
||||
runtime_sources::set_setup_type(previous).await;
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
fn build_s3_tier(name: &str) -> TierConfig {
|
||||
TierConfig {
|
||||
@@ -6386,42 +6315,6 @@ mod tests {
|
||||
assert!(err.to_string().contains("without peer commit clients"), "{err}");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial_test::serial]
|
||||
async fn tier_mutation_peer_composition_preserves_unresolved_topology_slots() {
|
||||
let mut endpoints = Vec::new();
|
||||
for disk_index in 0..4 {
|
||||
let mut endpoint = Endpoint::try_from(format!("http://rustfs-{disk_index}.invalid:9000/data{disk_index}").as_str())
|
||||
.expect("unresolved topology endpoint should parse without DNS");
|
||||
endpoint.is_local = disk_index == 0;
|
||||
endpoint.set_pool_index(0);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(disk_index);
|
||||
endpoints.push(endpoint);
|
||||
}
|
||||
let topology = EndpointServerPools::from(vec![PoolEndpoints {
|
||||
legacy: false,
|
||||
set_count: 1,
|
||||
drives_per_set: 4,
|
||||
endpoints: Endpoints::from(endpoints),
|
||||
cmd_line: "unresolved-tier-mutation-topology".to_string(),
|
||||
platform: "test".to_string(),
|
||||
}]);
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
|
||||
|
||||
let peers = remote_tier_mutation_peers_from_topology(topology)
|
||||
.await
|
||||
.expect("every unresolved remote topology slot should retain a tier mutation client");
|
||||
assert_eq!(
|
||||
peers.iter().map(|peer| peer.peer_label()).collect::<Vec<_>>(),
|
||||
vec![
|
||||
"http://rustfs-1.invalid:9000".to_string(),
|
||||
"http://rustfs-2.invalid:9000".to_string(),
|
||||
"http://rustfs-3.invalid:9000".to_string(),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn coordinator_fanout_prepare_failure_aborts_prepared_peers_without_cas() {
|
||||
let manager = TierConfigMgr::new();
|
||||
|
||||
@@ -73,21 +73,6 @@ pub enum TransitionCandidateProbe {
|
||||
Unsupported,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
pub(crate) struct TransitionCandidateIdentity {
|
||||
pub transaction_id: uuid::Uuid,
|
||||
pub destination_id: [u8; 32],
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
pub(crate) trait TransitionCandidateReconciler {
|
||||
async fn probe_transition_candidate_for(
|
||||
&self,
|
||||
object: &str,
|
||||
identity: TransitionCandidateIdentity,
|
||||
) -> Result<TransitionCandidateProbe, std::io::Error>;
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
pub trait WarmBackend {
|
||||
async fn validate(&self) -> Result<(), std::io::Error> {
|
||||
@@ -204,20 +189,6 @@ pub fn build_transition_put_options(storage_class: String, mut metadata: HashMap
|
||||
metadata.remove(key);
|
||||
}
|
||||
|
||||
for suffix in [
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TRANSACTION_ID,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
|
||||
] {
|
||||
for key in [
|
||||
rustfs_utils::http::metadata_compat::internal_key_rustfs(suffix),
|
||||
format!("{}{}", rustfs_utils::http::metadata_compat::MINIO_INTERNAL_PREFIX, suffix),
|
||||
] {
|
||||
if let Some(value) = metadata.remove(&key) {
|
||||
metadata.insert(format!("x-amz-meta-{key}"), value);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
opts.user_metadata = metadata;
|
||||
opts
|
||||
}
|
||||
@@ -475,51 +446,6 @@ pub async fn new_warm_backend(tier: &TierConfig, probe: bool) -> Result<WarmBack
|
||||
Ok(d)
|
||||
}
|
||||
|
||||
pub(crate) async fn new_transition_candidate_reconciler(
|
||||
tier: &TierConfig,
|
||||
) -> Result<Option<Box<dyn TransitionCandidateReconciler + Send + Sync + 'static>>, AdminError> {
|
||||
let reconciler: Box<dyn TransitionCandidateReconciler + Send + Sync + 'static> = match tier.tier_type {
|
||||
TierType::S3 => Box::new(
|
||||
WarmBackendS3::new(tier.s3.as_ref().ok_or_else(|| ERR_TIER_INVALID_CONFIG.clone())?, &tier.name)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
let mut admin_err = ERR_TIER_INVALID_CONFIG.clone();
|
||||
admin_err.message = err.to_string();
|
||||
admin_err
|
||||
})?,
|
||||
),
|
||||
TierType::MinIO => Box::new(
|
||||
WarmBackendMinIO::new(tier.minio.as_ref().ok_or_else(|| ERR_TIER_INVALID_CONFIG.clone())?, &tier.name)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
let mut admin_err = ERR_TIER_INVALID_CONFIG.clone();
|
||||
admin_err.message = err.to_string();
|
||||
admin_err
|
||||
})?,
|
||||
),
|
||||
TierType::RustFS => Box::new(
|
||||
WarmBackendRustFS::new(tier.rustfs.as_ref().ok_or_else(|| ERR_TIER_INVALID_CONFIG.clone())?, &tier.name)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
let mut admin_err = ERR_TIER_INVALID_CONFIG.clone();
|
||||
admin_err.message = err.to_string();
|
||||
admin_err
|
||||
})?,
|
||||
),
|
||||
TierType::R2 => Box::new(
|
||||
WarmBackendR2::new(tier.r2.as_ref().ok_or_else(|| ERR_TIER_INVALID_CONFIG.clone())?, &tier.name)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
let mut admin_err = ERR_TIER_INVALID_CONFIG.clone();
|
||||
admin_err.message = err.to_string();
|
||||
admin_err
|
||||
})?,
|
||||
),
|
||||
_ => return Ok(None),
|
||||
};
|
||||
Ok(Some(reconciler))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -849,37 +775,6 @@ mod tests {
|
||||
assert!(!opts.user_metadata.contains_key(X_AMZ_REPLICATION_STATUS.as_str()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_transition_put_options_persists_both_candidate_identity_keys_as_s3_metadata() {
|
||||
let mut metadata = HashMap::new();
|
||||
rustfs_utils::http::metadata_compat::insert_str(
|
||||
&mut metadata,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TRANSACTION_ID,
|
||||
"aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa".to_string(),
|
||||
);
|
||||
rustfs_utils::http::metadata_compat::insert_str(
|
||||
&mut metadata,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
|
||||
"5a".repeat(32),
|
||||
);
|
||||
|
||||
let opts = build_transition_put_options("COLD".to_string(), metadata);
|
||||
|
||||
for suffix in [
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TRANSACTION_ID,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
|
||||
] {
|
||||
assert!(opts.user_metadata.contains_key(&format!(
|
||||
"x-amz-meta-{}",
|
||||
rustfs_utils::http::metadata_compat::internal_key_rustfs(suffix)
|
||||
)));
|
||||
assert!(opts.user_metadata.contains_key(&format!(
|
||||
"x-amz-meta-{}{suffix}",
|
||||
rustfs_utils::http::metadata_compat::MINIO_INTERNAL_PREFIX
|
||||
)));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_transition_put_options_requests_no_checksum_and_content_md5() {
|
||||
// Regression for rustfs/rustfs#4811: transition uploads must leave the
|
||||
|
||||
@@ -135,20 +135,6 @@ impl WarmBackend for WarmBackendMinIO {
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl crate::services::tier::warm_backend::TransitionCandidateReconciler for WarmBackendMinIO {
|
||||
async fn probe_transition_candidate_for(
|
||||
&self,
|
||||
object: &str,
|
||||
identity: crate::services::tier::warm_backend::TransitionCandidateIdentity,
|
||||
) -> Result<TransitionCandidateProbe, std::io::Error> {
|
||||
crate::services::tier::warm_backend::TransitionCandidateReconciler::probe_transition_candidate_for(
|
||||
&self.0, object, identity,
|
||||
)
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
fn optimal_part_size(object_size: i64) -> Result<i64, std::io::Error> {
|
||||
let mut object_size = object_size;
|
||||
if object_size == -1 {
|
||||
|
||||
@@ -135,20 +135,6 @@ impl WarmBackend for WarmBackendR2 {
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl crate::services::tier::warm_backend::TransitionCandidateReconciler for WarmBackendR2 {
|
||||
async fn probe_transition_candidate_for(
|
||||
&self,
|
||||
object: &str,
|
||||
identity: crate::services::tier::warm_backend::TransitionCandidateIdentity,
|
||||
) -> Result<TransitionCandidateProbe, std::io::Error> {
|
||||
crate::services::tier::warm_backend::TransitionCandidateReconciler::probe_transition_candidate_for(
|
||||
&self.0, object, identity,
|
||||
)
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
fn optimal_part_size(object_size: i64) -> Result<i64, std::io::Error> {
|
||||
let mut object_size = object_size;
|
||||
if object_size == -1 {
|
||||
|
||||
@@ -132,20 +132,6 @@ impl WarmBackend for WarmBackendRustFS {
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl crate::services::tier::warm_backend::TransitionCandidateReconciler for WarmBackendRustFS {
|
||||
async fn probe_transition_candidate_for(
|
||||
&self,
|
||||
object: &str,
|
||||
identity: crate::services::tier::warm_backend::TransitionCandidateIdentity,
|
||||
) -> Result<TransitionCandidateProbe, std::io::Error> {
|
||||
crate::services::tier::warm_backend::TransitionCandidateReconciler::probe_transition_candidate_for(
|
||||
&self.0, object, identity,
|
||||
)
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
fn optimal_part_size(object_size: i64) -> Result<i64, std::io::Error> {
|
||||
let mut object_size = object_size;
|
||||
if object_size == -1 {
|
||||
|
||||
@@ -29,7 +29,6 @@ use crate::client::{
|
||||
api_remove::{RemoveObjectOptions, RemoveObjectResult},
|
||||
api_s3_datatypes::ListVersionsResult,
|
||||
credentials::{Credentials, SignatureType, Static, Value},
|
||||
provider_versions::validate_remote_version_id,
|
||||
transition_api::{BucketLookupType, Options, TransitionClient, TransitionCore},
|
||||
transition_api::{ReadCloser, ReaderImpl},
|
||||
};
|
||||
@@ -37,10 +36,7 @@ use crate::error::ErrorResponse;
|
||||
use crate::error::error_resp_to_object_err;
|
||||
use crate::services::tier::{
|
||||
tier_config::TierS3,
|
||||
warm_backend::{
|
||||
TransitionCandidateIdentity, TransitionCandidateProbe, TransitionCandidateReconciler, WarmBackend, WarmBackendGetOpts,
|
||||
build_transition_put_options,
|
||||
},
|
||||
warm_backend::{TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts, build_transition_put_options},
|
||||
};
|
||||
use http::HeaderMap;
|
||||
use rustfs_utils::egress::validate_outbound_url;
|
||||
@@ -193,188 +189,44 @@ impl WarmBackendS3 {
|
||||
remote_bucket_versioning_from_status(config.status.as_ref().map(|status| status.as_str()))
|
||||
}
|
||||
|
||||
async fn probe_transition_candidate_versions(
|
||||
&self,
|
||||
object: &str,
|
||||
bucket_versioning: RemoteBucketVersioning,
|
||||
) -> Result<TransitionCandidateProbe, std::io::Error> {
|
||||
let remote_object = self.get_dest(object);
|
||||
async fn list_transition_candidate_versions(&self, object: &str) -> Result<ListVersionsResult, std::io::Error> {
|
||||
let mut opts = ListObjectsOptions::default();
|
||||
opts.set("prefix", &remote_object);
|
||||
opts.set("max-keys", "1000");
|
||||
|
||||
let mut key_marker = String::new();
|
||||
let mut version_id_marker = String::new();
|
||||
let mut candidates = TransitionCandidateVersions::default();
|
||||
loop {
|
||||
let versions = self
|
||||
.client
|
||||
.list_object_versions_query(&self.bucket, &opts, &key_marker, &version_id_marker, "")
|
||||
.await?;
|
||||
candidates.extend(&remote_object, &versions);
|
||||
if candidates.is_ambiguous() {
|
||||
return Ok(TransitionCandidateProbe::Ambiguous);
|
||||
}
|
||||
if !versions.is_truncated {
|
||||
return classify_transition_candidates(candidates, bucket_versioning);
|
||||
}
|
||||
|
||||
advance_version_markers(&mut key_marker, &mut version_id_marker, &versions)?;
|
||||
}
|
||||
}
|
||||
|
||||
async fn probe_transition_candidate_identity(
|
||||
&self,
|
||||
object: &str,
|
||||
identity: TransitionCandidateIdentity,
|
||||
bucket_versioning: RemoteBucketVersioning,
|
||||
) -> Result<TransitionCandidateProbe, std::io::Error> {
|
||||
let remote_object = self.get_dest(object);
|
||||
let mut opts = ListObjectsOptions::default();
|
||||
opts.set("prefix", &remote_object);
|
||||
opts.set("max-keys", "1000");
|
||||
let mut key_marker = String::new();
|
||||
let mut version_id_marker = String::new();
|
||||
let mut matched_version = None;
|
||||
let mut saw_unproven_candidate = false;
|
||||
|
||||
loop {
|
||||
let versions = self
|
||||
.client
|
||||
.list_object_versions_query(&self.bucket, &opts, &key_marker, &version_id_marker, "")
|
||||
.await?;
|
||||
for version in versions.versions.iter().filter(|version| version.key == remote_object) {
|
||||
let mut stat_opts = GetObjectOptions::default();
|
||||
stat_opts.version_id.clone_from(&version.version_id);
|
||||
let info = self.client.stat_object(&self.bucket, &remote_object, &stat_opts).await?;
|
||||
let mut metadata = info.user_metadata;
|
||||
for (name, value) in &info.metadata {
|
||||
if (name
|
||||
.as_str()
|
||||
.starts_with(rustfs_utils::http::metadata_compat::RUSTFS_INTERNAL_PREFIX)
|
||||
|| name
|
||||
.as_str()
|
||||
.starts_with(rustfs_utils::http::metadata_compat::MINIO_INTERNAL_PREFIX))
|
||||
&& let Ok(value) = value.to_str()
|
||||
{
|
||||
metadata.insert(name.as_str().to_string(), value.to_string());
|
||||
}
|
||||
}
|
||||
if transition_candidate_metadata_matches(&metadata, identity)? {
|
||||
if matched_version.is_some() {
|
||||
return Ok(TransitionCandidateProbe::Ambiguous);
|
||||
}
|
||||
matched_version = Some(version.version_id.clone());
|
||||
} else {
|
||||
saw_unproven_candidate = true;
|
||||
}
|
||||
}
|
||||
if !versions.is_truncated {
|
||||
if matched_version.is_none() && saw_unproven_candidate {
|
||||
return Ok(TransitionCandidateProbe::Unsupported);
|
||||
}
|
||||
let candidates = TransitionCandidateVersions {
|
||||
version_id: matched_version,
|
||||
ambiguous: false,
|
||||
};
|
||||
return classify_transition_candidates(candidates, bucket_versioning);
|
||||
}
|
||||
advance_version_markers(&mut key_marker, &mut version_id_marker, &versions)?;
|
||||
}
|
||||
opts.set("prefix", &self.get_dest(object));
|
||||
opts.set("max-keys", "2");
|
||||
self.client.list_object_versions_query(&self.bucket, &opts, "", "", "").await
|
||||
}
|
||||
}
|
||||
|
||||
fn transition_candidate_metadata_matches(
|
||||
metadata: &HashMap<String, String>,
|
||||
identity: TransitionCandidateIdentity,
|
||||
) -> Result<bool, std::io::Error> {
|
||||
use rustfs_utils::http::metadata_compat::{
|
||||
SUFFIX_TRANSITION_TIER_DESTINATION_ID, SUFFIX_TRANSITION_TRANSACTION_ID, contains_key_str, get_consistent_str,
|
||||
};
|
||||
|
||||
let transaction_id = get_consistent_str(metadata, SUFFIX_TRANSITION_TRANSACTION_ID);
|
||||
let destination_id = get_consistent_str(metadata, SUFFIX_TRANSITION_TIER_DESTINATION_ID);
|
||||
if transaction_id.is_none() || destination_id.is_none() {
|
||||
if contains_key_str(metadata, SUFFIX_TRANSITION_TRANSACTION_ID)
|
||||
|| contains_key_str(metadata, SUFFIX_TRANSITION_TIER_DESTINATION_ID)
|
||||
{
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidData,
|
||||
"transition candidate identity metadata is empty or conflicting",
|
||||
));
|
||||
}
|
||||
return Ok(false);
|
||||
}
|
||||
let expected_transaction_id = identity.transaction_id.to_string();
|
||||
let expected_destination_id = rustfs_utils::crypto::hex(identity.destination_id);
|
||||
Ok(transaction_id == Some(expected_transaction_id.as_str()) && destination_id == Some(expected_destination_id.as_str()))
|
||||
}
|
||||
|
||||
fn classify_transition_candidates(
|
||||
candidates: TransitionCandidateVersions,
|
||||
fn classify_transition_candidate_versions(
|
||||
remote_object: &str,
|
||||
bucket_versioning: RemoteBucketVersioning,
|
||||
) -> Result<TransitionCandidateProbe, std::io::Error> {
|
||||
let probe = candidates.classify(bucket_versioning);
|
||||
if let TransitionCandidateProbe::VersionedPresent(version_id) = &probe {
|
||||
validate_remote_version_id(version_id)?;
|
||||
}
|
||||
Ok(probe)
|
||||
}
|
||||
|
||||
fn advance_version_markers(
|
||||
key_marker: &mut String,
|
||||
version_id_marker: &mut String,
|
||||
versions: &ListVersionsResult,
|
||||
) -> Result<(), std::io::Error> {
|
||||
let next_markers = (&versions.next_key_marker, &versions.next_version_id_marker);
|
||||
if next_markers == (&*key_marker, &*version_id_marker) {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidData,
|
||||
"ListObjectVersions pagination markers did not advance",
|
||||
));
|
||||
}
|
||||
key_marker.clone_from(&versions.next_key_marker);
|
||||
version_id_marker.clone_from(&versions.next_version_id_marker);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct TransitionCandidateVersions {
|
||||
version_id: Option<String>,
|
||||
ambiguous: bool,
|
||||
}
|
||||
|
||||
impl TransitionCandidateVersions {
|
||||
fn extend(&mut self, remote_object: &str, versions: &ListVersionsResult) {
|
||||
for version in versions.versions.iter().filter(|version| version.key == remote_object) {
|
||||
if self.version_id.is_some() {
|
||||
self.ambiguous = true;
|
||||
return;
|
||||
}
|
||||
self.version_id = Some(version.version_id.clone());
|
||||
}
|
||||
) -> TransitionCandidateProbe {
|
||||
if versions.is_truncated {
|
||||
return TransitionCandidateProbe::Ambiguous;
|
||||
}
|
||||
|
||||
fn is_ambiguous(&self) -> bool {
|
||||
self.ambiguous
|
||||
if versions.delete_markers.iter().any(|marker| marker.key == remote_object) {
|
||||
return TransitionCandidateProbe::Ambiguous;
|
||||
}
|
||||
|
||||
fn classify(self, bucket_versioning: RemoteBucketVersioning) -> TransitionCandidateProbe {
|
||||
if self.ambiguous {
|
||||
return TransitionCandidateProbe::Ambiguous;
|
||||
}
|
||||
let Some(version_id) = self.version_id else {
|
||||
return TransitionCandidateProbe::Missing;
|
||||
};
|
||||
let mut exact_versions = versions.versions.iter().filter(|version| version.key == remote_object);
|
||||
let Some(version) = exact_versions.next() else {
|
||||
return TransitionCandidateProbe::Missing;
|
||||
};
|
||||
if exact_versions.next().is_some() {
|
||||
return TransitionCandidateProbe::Ambiguous;
|
||||
}
|
||||
|
||||
match bucket_versioning {
|
||||
RemoteBucketVersioning::Disabled => TransitionCandidateProbe::UnversionedPresent,
|
||||
RemoteBucketVersioning::Suspended if version_id == "null" => TransitionCandidateProbe::VersionedPresent(version_id),
|
||||
RemoteBucketVersioning::Suspended | RemoteBucketVersioning::Enabled if !version_id.is_empty() => {
|
||||
TransitionCandidateProbe::VersionedPresent(version_id)
|
||||
}
|
||||
RemoteBucketVersioning::Suspended | RemoteBucketVersioning::Enabled => TransitionCandidateProbe::Ambiguous,
|
||||
match bucket_versioning {
|
||||
RemoteBucketVersioning::Disabled => TransitionCandidateProbe::UnversionedPresent,
|
||||
RemoteBucketVersioning::Suspended if version.version_id == "null" => {
|
||||
TransitionCandidateProbe::VersionedPresent(version.version_id.clone())
|
||||
}
|
||||
RemoteBucketVersioning::Suspended | RemoteBucketVersioning::Enabled if !version.version_id.is_empty() => {
|
||||
TransitionCandidateProbe::VersionedPresent(version.version_id.clone())
|
||||
}
|
||||
RemoteBucketVersioning::Suspended | RemoteBucketVersioning::Enabled => TransitionCandidateProbe::Ambiguous,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -423,161 +275,72 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
fn classify_pages(bucket_versioning: RemoteBucketVersioning, pages: &[ListVersionsResult]) -> TransitionCandidateProbe {
|
||||
let mut candidates = TransitionCandidateVersions::default();
|
||||
for page in pages {
|
||||
candidates.extend("archive/object", page);
|
||||
}
|
||||
candidates.classify(bucket_versioning)
|
||||
}
|
||||
|
||||
fn candidate_identity() -> TransitionCandidateIdentity {
|
||||
TransitionCandidateIdentity {
|
||||
transaction_id: uuid::Uuid::parse_str("aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa").unwrap(),
|
||||
destination_id: [0x5a; 32],
|
||||
}
|
||||
}
|
||||
|
||||
fn candidate_metadata(identity: TransitionCandidateIdentity) -> HashMap<String, String> {
|
||||
let mut metadata = HashMap::new();
|
||||
rustfs_utils::http::metadata_compat::insert_str(
|
||||
&mut metadata,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TRANSACTION_ID,
|
||||
identity.transaction_id.to_string(),
|
||||
);
|
||||
rustfs_utils::http::metadata_compat::insert_str(
|
||||
&mut metadata,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
|
||||
rustfs_utils::crypto::hex(identity.destination_id),
|
||||
);
|
||||
metadata
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transition_candidate_identity_requires_exact_compatible_metadata() {
|
||||
let identity = candidate_identity();
|
||||
let metadata = candidate_metadata(identity);
|
||||
assert!(transition_candidate_metadata_matches(&metadata, identity).unwrap());
|
||||
|
||||
let mut adjacent = metadata;
|
||||
rustfs_utils::http::metadata_compat::insert_str(
|
||||
&mut adjacent,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TRANSACTION_ID,
|
||||
uuid::Uuid::parse_str("bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb")
|
||||
.unwrap()
|
||||
.to_string(),
|
||||
);
|
||||
assert!(!transition_candidate_metadata_matches(&adjacent, identity).unwrap());
|
||||
|
||||
assert!(!transition_candidate_metadata_matches(&HashMap::new(), identity).unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transition_candidate_identity_rejects_conflicting_compatibility_keys() {
|
||||
let identity = candidate_identity();
|
||||
let mut metadata = candidate_metadata(identity);
|
||||
metadata.insert(
|
||||
rustfs_utils::http::metadata_compat::internal_key_rustfs(
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TRANSACTION_ID,
|
||||
),
|
||||
uuid::Uuid::new_v4().to_string(),
|
||||
);
|
||||
assert!(transition_candidate_metadata_matches(&metadata, identity).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transition_candidate_probe_classifier_is_fail_closed() {
|
||||
assert_eq!(
|
||||
classify_pages(RemoteBucketVersioning::Disabled, &[list_versions(&[], &[], false)],),
|
||||
classify_transition_candidate_versions(
|
||||
"archive/object",
|
||||
RemoteBucketVersioning::Disabled,
|
||||
&list_versions(&[], &[], false),
|
||||
),
|
||||
TransitionCandidateProbe::Missing
|
||||
);
|
||||
assert_eq!(
|
||||
classify_pages(RemoteBucketVersioning::Disabled, &[list_versions(&[("archive/object", "")], &[], false)],),
|
||||
classify_transition_candidate_versions(
|
||||
"archive/object",
|
||||
RemoteBucketVersioning::Disabled,
|
||||
&list_versions(&[("archive/object", "")], &[], false),
|
||||
),
|
||||
TransitionCandidateProbe::UnversionedPresent
|
||||
);
|
||||
assert_eq!(
|
||||
classify_pages(
|
||||
classify_transition_candidate_versions(
|
||||
"archive/object",
|
||||
RemoteBucketVersioning::Enabled,
|
||||
&[list_versions(&[("archive/object", "version-a")], &[], false)],
|
||||
&list_versions(&[("archive/object", "version-a")], &[], false),
|
||||
),
|
||||
TransitionCandidateProbe::VersionedPresent("version-a".to_string())
|
||||
);
|
||||
assert_eq!(
|
||||
classify_pages(
|
||||
classify_transition_candidate_versions(
|
||||
"archive/object",
|
||||
RemoteBucketVersioning::Suspended,
|
||||
&[list_versions(&[("archive/object", "null")], &[], false)],
|
||||
&list_versions(&[("archive/object", "null")], &[], false),
|
||||
),
|
||||
TransitionCandidateProbe::VersionedPresent("null".to_string())
|
||||
);
|
||||
assert_eq!(
|
||||
classify_pages(RemoteBucketVersioning::Enabled, &[list_versions(&[("archive/object", "")], &[], false)],),
|
||||
TransitionCandidateProbe::Ambiguous
|
||||
);
|
||||
assert_eq!(
|
||||
classify_pages(
|
||||
classify_transition_candidate_versions(
|
||||
"archive/object",
|
||||
RemoteBucketVersioning::Enabled,
|
||||
&[list_versions(
|
||||
&[("archive/object", "version-a"), ("archive/object", "version-b")],
|
||||
&[],
|
||||
false,
|
||||
)],
|
||||
&list_versions(&[("archive/object", "")], &[], false),
|
||||
),
|
||||
TransitionCandidateProbe::Ambiguous
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transition_candidate_probe_reconciles_all_pages_and_ignores_delete_markers() {
|
||||
assert_eq!(
|
||||
classify_pages(
|
||||
classify_transition_candidate_versions(
|
||||
"archive/object",
|
||||
RemoteBucketVersioning::Enabled,
|
||||
&[
|
||||
list_versions(&[], &[("archive/object", "marker-a")], true),
|
||||
list_versions(&[("archive/object", "version-a"), ("archive/object-adjacent", "unrelated"),], &[], false,),
|
||||
],
|
||||
),
|
||||
TransitionCandidateProbe::VersionedPresent("version-a".to_string())
|
||||
);
|
||||
assert_eq!(
|
||||
classify_pages(
|
||||
RemoteBucketVersioning::Enabled,
|
||||
&[
|
||||
list_versions(&[("archive/object", "version-a")], &[], true),
|
||||
list_versions(&[("archive/object", "version-b")], &[], false),
|
||||
],
|
||||
&list_versions(&[("archive/object", "version-a"), ("archive/object", "version-b")], &[], false),
|
||||
),
|
||||
TransitionCandidateProbe::Ambiguous
|
||||
);
|
||||
assert_eq!(
|
||||
classify_transition_candidate_versions(
|
||||
"archive/object",
|
||||
RemoteBucketVersioning::Enabled,
|
||||
&list_versions(&[("archive/object", "version-a")], &[("archive/object", "marker-a")], false),
|
||||
),
|
||||
TransitionCandidateProbe::Ambiguous
|
||||
);
|
||||
assert_eq!(
|
||||
classify_transition_candidate_versions(
|
||||
"archive/object",
|
||||
RemoteBucketVersioning::Enabled,
|
||||
&list_versions(&[("archive/object", "version-a")], &[], true),
|
||||
),
|
||||
TransitionCandidateProbe::Ambiguous
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transition_candidate_pagination_advances_both_markers() {
|
||||
let mut key_marker = "old-key".to_string();
|
||||
let mut version_id_marker = "old-version".to_string();
|
||||
let page = ListVersionsResult {
|
||||
next_key_marker: "next-key".to_string(),
|
||||
next_version_id_marker: "next-version".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
advance_version_markers(&mut key_marker, &mut version_id_marker, &page)
|
||||
.expect("new ListObjectVersions markers should advance pagination");
|
||||
assert_eq!(key_marker, "next-key");
|
||||
assert_eq!(version_id_marker, "next-version");
|
||||
|
||||
let err = advance_version_markers(&mut key_marker, &mut version_id_marker, &page)
|
||||
.expect_err("repeated ListObjectVersions markers must fail closed");
|
||||
assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transition_candidate_probe_rejects_untrusted_version_ids() {
|
||||
let mut candidates = TransitionCandidateVersions::default();
|
||||
candidates.extend("archive/object", &list_versions(&[("archive/object", "version\ninjection")], &[], false));
|
||||
|
||||
let err = classify_transition_candidates(candidates, RemoteBucketVersioning::Enabled)
|
||||
.expect_err("control characters in listed version IDs must fail closed");
|
||||
assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -634,7 +397,12 @@ impl WarmBackend for WarmBackendS3 {
|
||||
|
||||
async fn probe_transition_candidate(&self, object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
|
||||
let bucket_versioning = self.remote_bucket_versioning().await?;
|
||||
self.probe_transition_candidate_versions(object, bucket_versioning).await
|
||||
let versions = self.list_transition_candidate_versions(object).await?;
|
||||
Ok(classify_transition_candidate_versions(
|
||||
&self.get_dest(object),
|
||||
bucket_versioning,
|
||||
&versions,
|
||||
))
|
||||
}
|
||||
|
||||
async fn in_use(&self) -> Result<bool, std::io::Error> {
|
||||
@@ -646,16 +414,3 @@ impl WarmBackend for WarmBackendS3 {
|
||||
Ok(result.common_prefixes.len() > 0 || result.contents.len() > 0)
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl TransitionCandidateReconciler for WarmBackendS3 {
|
||||
async fn probe_transition_candidate_for(
|
||||
&self,
|
||||
object: &str,
|
||||
identity: TransitionCandidateIdentity,
|
||||
) -> Result<TransitionCandidateProbe, std::io::Error> {
|
||||
let bucket_versioning = self.remote_bucket_versioning().await?;
|
||||
self.probe_transition_candidate_identity(object, identity, bucket_versioning)
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
@@ -110,10 +110,7 @@ use crate::{
|
||||
object_api::{GetObjectReader, ObjectInfo, PutObjReader},
|
||||
// event::name::EventName,
|
||||
services::event_notification::{EventArgs, send_event},
|
||||
store::init_format::{
|
||||
formats_match_reference_slots, get_format_erasure_in_quorum, load_format_erasure, load_format_erasure_all,
|
||||
save_format_file,
|
||||
},
|
||||
store::init_format::{get_format_erasure_in_quorum, load_format_erasure, load_format_erasure_all, save_format_file},
|
||||
};
|
||||
use bytes::Bytes;
|
||||
use bytesize::ByteSize;
|
||||
@@ -5852,27 +5849,23 @@ mod tests {
|
||||
crate::disk::DataDirDeleteStatus::Deleted
|
||||
);
|
||||
release_slow_candidate.notify_one();
|
||||
timeout(Duration::from_secs(1), async {
|
||||
loop {
|
||||
match disk2
|
||||
.delete_data_dir(
|
||||
bucket,
|
||||
data_dir,
|
||||
DeleteOptions {
|
||||
recursive: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("a token acquired after the deadline must be released")
|
||||
{
|
||||
crate::disk::DataDirDeleteStatus::Deleted => return,
|
||||
crate::disk::DataDirDeleteStatus::Deferred => tokio::time::sleep(Duration::from_millis(1)).await,
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("late snapshot lease cleanup must finish within the bounded wait");
|
||||
for _ in 0..10 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
assert_eq!(
|
||||
disk2
|
||||
.delete_data_dir(
|
||||
bucket,
|
||||
data_dir,
|
||||
DeleteOptions {
|
||||
recursive: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("a token acquired after the deadline must be released"),
|
||||
crate::disk::DataDirDeleteStatus::Deleted
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
|
||||
@@ -1373,24 +1373,11 @@ impl crate::storage_api_contracts::heal::HealOperations for SetDisks {
|
||||
async fn heal_format(&self, dry_run: bool) -> Result<(HealResultItem, Option<Error>)> {
|
||||
let disks = self.disks.read().await.clone();
|
||||
let (formats, errs) = load_format_erasure_all(&disks, true).await;
|
||||
if errs.iter().any(|err| {
|
||||
matches!(
|
||||
err,
|
||||
Some(DiskError::InconsistentDisk | DiskError::CorruptedFormat | DiskError::CorruptedBackend)
|
||||
)
|
||||
}) {
|
||||
return Ok((HealResultItem::default(), Some(StorageError::CorruptedFormat)));
|
||||
}
|
||||
let slot_offset = self
|
||||
.set_index
|
||||
.checked_mul(self.set_drive_count)
|
||||
.ok_or_else(|| Error::other("erasure set slot offset overflow"))?;
|
||||
let ref_format = match get_format_erasure_in_quorum(&formats, slot_offset) {
|
||||
Ok(format) if format.shared_identity() == self.format.shared_identity() => format,
|
||||
Ok(_) => return Ok((HealResultItem::default(), Some(StorageError::CorruptedFormat))),
|
||||
let ref_format = match get_format_erasure_in_quorum(&formats) {
|
||||
Ok(format) => format,
|
||||
Err(err) => {
|
||||
let can_use_cached_layout = count_errs(&errs, &DiskError::UnformattedDisk) > 0
|
||||
&& formats_match_reference_slots(&formats, &self.format, slot_offset)
|
||||
&& formats.iter().flatten().all(|format| self.format.check_other(format).is_ok())
|
||||
&& errs
|
||||
.iter()
|
||||
.all(|err| err.is_none() || matches!(err, Some(DiskError::UnformattedDisk)));
|
||||
@@ -1401,9 +1388,6 @@ impl crate::storage_api_contracts::heal::HealOperations for SetDisks {
|
||||
}
|
||||
}
|
||||
};
|
||||
if !formats_match_reference_slots(&formats, &ref_format, slot_offset) {
|
||||
return Ok((HealResultItem::default(), Some(StorageError::CorruptedFormat)));
|
||||
}
|
||||
|
||||
let endpoints = crate::layout::endpoints::Endpoints::from(self.set_endpoints.clone());
|
||||
let before_drives = crate::layout::set_heal::formats_to_drives_info(&endpoints, &formats, &errs);
|
||||
@@ -1559,16 +1543,11 @@ mod heal_result_report_tests {
|
||||
use crate::disk::error::DiskError;
|
||||
use crate::disk::format::FormatV3;
|
||||
use crate::disk::{DiskAPI as _, DiskOption, DiskStore, RUSTFS_META_TMP_BUCKET, ReadOptions, new_disk};
|
||||
use crate::error::Error;
|
||||
use crate::object_api::{ObjectOptions, PutObjReader};
|
||||
use crate::set_disk::ops::object::hermetic_set_disks_support::hermetic_set_disks_isolated;
|
||||
use crate::storage_api_contracts::bucket::{BucketOperations as _, MakeBucketOptions};
|
||||
use crate::storage_api_contracts::heal::HealOperations as _;
|
||||
use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _};
|
||||
use crate::{
|
||||
config::storageclass,
|
||||
store::init_format::{load_format_erasure, save_format_file},
|
||||
};
|
||||
use crate::{config::storageclass, store::init_format::save_format_file};
|
||||
use rustfs_common::heal_channel::{DriveState, HealOpts, HealScanMode};
|
||||
use rustfs_filemeta::{BLOCK_SIZE_V2, FileInfo, ObjectPartInfo, TRANSITION_COMPLETE};
|
||||
use std::sync::Arc;
|
||||
@@ -1884,44 +1863,6 @@ mod heal_result_report_tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn format_heal_cached_layout_rejects_a_disk_from_another_slot() {
|
||||
let mut _temp_dirs = Vec::new();
|
||||
let mut endpoints = Vec::new();
|
||||
let mut disks = Vec::new();
|
||||
for disk_index in 0..3 {
|
||||
let (temp_dir, mut endpoint, disk) = real_disk().await;
|
||||
endpoint.set_pool_index(0);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(disk_index);
|
||||
_temp_dirs.push(temp_dir);
|
||||
endpoints.push(endpoint);
|
||||
disks.push(Some(disk));
|
||||
}
|
||||
let set = set_disks_with(disks.clone(), endpoints, 1).await;
|
||||
let mut wrong_slot = set.format.clone();
|
||||
wrong_slot.erasure.this = set.format.erasure.sets[0][1];
|
||||
save_format_file(&disks[0], &Some(wrong_slot))
|
||||
.await
|
||||
.expect("wrong-slot format fixture should be saved");
|
||||
let mut correct_slot = set.format.clone();
|
||||
correct_slot.erasure.this = set.format.erasure.sets[0][2];
|
||||
save_format_file(&disks[2], &Some(correct_slot))
|
||||
.await
|
||||
.expect("correct format fixture should be saved");
|
||||
|
||||
let (_, heal_err) = set
|
||||
.heal_format(false)
|
||||
.await
|
||||
.expect("format heal should report the quorum failure in its result");
|
||||
|
||||
assert!(matches!(heal_err, Some(Error::CorruptedFormat)));
|
||||
let unformatted = load_format_erasure(disks[1].as_ref().expect("second disk should be online"), true)
|
||||
.await
|
||||
.expect_err("a rejected fallback must not format the missing slot");
|
||||
assert_eq!(unformatted, DiskError::UnformattedDisk);
|
||||
}
|
||||
|
||||
// Regression for #955: an offline disk must contribute exactly one drive
|
||||
// record. Before the fix the offline branch fell through and pushed a second
|
||||
// (Corrupt) record for the same disk, so `before/after.drives` grew to
|
||||
|
||||
@@ -343,23 +343,20 @@ impl SetDisks {
|
||||
}
|
||||
};
|
||||
|
||||
// Claiming a misplaced drive into `self.disks` would let two slots or
|
||||
// sets manage the same drive and degrade together (backlog#799 B19).
|
||||
if set_idx != self.set_index || self.set_endpoints.get(disk_idx) != Some(ep) {
|
||||
// The drive's format may place it in a different erasure set than this
|
||||
// one. Claiming a misplaced drive into `self.disks` would let two sets
|
||||
// manage the same drive and degrade together, so reject it here
|
||||
// (backlog#799 B19).
|
||||
if set_idx != self.set_index {
|
||||
warn!(
|
||||
endpoint = %ep,
|
||||
format_set_index = set_idx,
|
||||
format_disk_index = disk_idx,
|
||||
endpoint_pool_index = ep.pool_idx,
|
||||
endpoint_set_index = ep.set_idx,
|
||||
endpoint_disk_index = ep.disk_idx,
|
||||
expected_pool_index = self.pool_index,
|
||||
expected_set_index = self.set_index,
|
||||
"renew_disk rejected a drive whose endpoint and format do not identify the same topology slot"
|
||||
"renew_disk: drive {:?} belongs to set {} but is being renewed on set {}; skipping",
|
||||
ep, set_idx, self.set_index
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
// Check that the endpoint matches
|
||||
|
||||
let _ = new_disk.set_disk_id(Some(fm.erasure.this)).await;
|
||||
new_disk.enable_health_check();
|
||||
|
||||
@@ -718,108 +715,6 @@ mod tests {
|
||||
drop(temp_dirs);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn renew_disk_rejects_a_format_from_another_slot_or_cluster() {
|
||||
let disk_count = 3;
|
||||
let format = FormatV3::new(1, disk_count);
|
||||
let mut temp_dirs = Vec::with_capacity(disk_count);
|
||||
let mut endpoints = Vec::with_capacity(disk_count);
|
||||
let mut fixture_disks = Vec::with_capacity(disk_count);
|
||||
|
||||
for disk_idx in 0..disk_count {
|
||||
let (temp_dir, endpoint, disk) = make_formatted_local_disk(disk_idx, &format).await;
|
||||
temp_dirs.push(temp_dir);
|
||||
endpoints.push(endpoint);
|
||||
fixture_disks.push(disk);
|
||||
}
|
||||
|
||||
let set_disks = SetDisks::new(
|
||||
"test-owner".to_string(),
|
||||
Arc::new(RwLock::new(vec![Some(fixture_disks[0].clone()), None, None])),
|
||||
disk_count,
|
||||
disk_count / 2,
|
||||
0,
|
||||
0,
|
||||
endpoints.clone(),
|
||||
format.clone(),
|
||||
Vec::new(),
|
||||
)
|
||||
.await;
|
||||
|
||||
let mut other_cluster_format = format.clone();
|
||||
other_cluster_format.id = Uuid::new_v4();
|
||||
other_cluster_format.erasure.this = format.erasure.sets[0][2];
|
||||
save_format_file(&Some(fixture_disks[2].clone()), &Some(other_cluster_format))
|
||||
.await
|
||||
.expect("other-cluster format should be written for the rejection test");
|
||||
|
||||
set_disks.renew_disk(&endpoints[2]).await;
|
||||
|
||||
let disks = set_disks.get_disks_internal().await;
|
||||
assert_eq!(
|
||||
disks[0]
|
||||
.as_ref()
|
||||
.expect("the canonical first slot must remain attached")
|
||||
.endpoint(),
|
||||
endpoints[0]
|
||||
);
|
||||
assert!(
|
||||
disks[2].is_none(),
|
||||
"a disk from another deployment must remain detached even when its slot UUID matches"
|
||||
);
|
||||
|
||||
let mut correct_format = format.clone();
|
||||
correct_format.erasure.this = format.erasure.sets[0][2];
|
||||
let replacement_disk = new_disk(
|
||||
&endpoints[2],
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("third endpoint should reopen after other-cluster rejection");
|
||||
save_format_file(&Some(replacement_disk), &Some(correct_format))
|
||||
.await
|
||||
.expect("correct slot format should be restored");
|
||||
|
||||
set_disks.renew_disk(&endpoints[2]).await;
|
||||
|
||||
let disks = set_disks.get_disks_internal().await;
|
||||
assert_eq!(
|
||||
disks[0]
|
||||
.as_ref()
|
||||
.expect("the canonical first slot must remain attached")
|
||||
.endpoint(),
|
||||
endpoints[0]
|
||||
);
|
||||
assert_eq!(disks[2].as_ref().expect("the restored third slot should attach").endpoint(), endpoints[2]);
|
||||
|
||||
let third_disk = disks[2].clone();
|
||||
let mut wrong_slot_format = format.clone();
|
||||
wrong_slot_format.erasure.this = format.erasure.sets[0][0];
|
||||
save_format_file(&third_disk, &Some(wrong_slot_format))
|
||||
.await
|
||||
.expect("wrong-slot format should be written for the rejection test");
|
||||
set_disks.disks.write().await[2] = None;
|
||||
|
||||
let mut misplaced_endpoint = endpoints[2].clone();
|
||||
misplaced_endpoint.set_disk_index(0);
|
||||
set_disks.renew_disk(&misplaced_endpoint).await;
|
||||
|
||||
let disks = set_disks.get_disks_internal().await;
|
||||
assert_eq!(
|
||||
disks[0]
|
||||
.as_ref()
|
||||
.expect("the canonical first slot must remain attached")
|
||||
.endpoint(),
|
||||
endpoints[0]
|
||||
);
|
||||
assert!(disks[2].is_none(), "a disk claiming another endpoint's slot must remain detached");
|
||||
|
||||
drop(temp_dirs);
|
||||
}
|
||||
|
||||
// SetDisks split P0 (#816): the borrow handle must mirror the core state and
|
||||
// the List operation family must run identically through it.
|
||||
#[tokio::test]
|
||||
|
||||
@@ -3379,98 +3379,85 @@ mod tests {
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn complete_holds_object_then_upload_lock_through_commit() {
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(crate::object_api::ENV_RUSTFS_ENCRYPTED_RANGE_SEEK, Some("true")),
|
||||
(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("60")),
|
||||
],
|
||||
async {
|
||||
let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
|
||||
let signaling = Arc::new(SignalingLockClient::new(Arc::new(LocalClient::with_manager(manager))));
|
||||
let lockers: Vec<Arc<dyn LockClient>> = vec![signaling.clone()];
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks_with_lockers(4, 0, 2, lockers).await;
|
||||
let bucket = "multipart-layout-lock-order-bucket";
|
||||
let object = "object";
|
||||
make_bucket_on_all(&disk_stores, bucket).await;
|
||||
let create_opts = ObjectOptions {
|
||||
user_defined: HashMap::from([(SSEC_ALGORITHM_HEADER.to_string(), "AES256".to_string())]),
|
||||
..Default::default()
|
||||
};
|
||||
let (upload_id, parts) =
|
||||
stage_upload_with_create_opts(&set_disks, bucket, object, &[0x43; 4096], &create_opts).await;
|
||||
let upload_id_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id);
|
||||
let upload_resource = rustfs_lock::ObjectKey::new(RUSTFS_META_MULTIPART_BUCKET, upload_id_path);
|
||||
let object_resource = rustfs_lock::ObjectKey::new(bucket, object);
|
||||
signaling.set_target(upload_resource.clone());
|
||||
signaling.clear_observed();
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
|
||||
let barrier = MultipartCommitBarrier::install(bucket, object, MultipartCommitPause::AfterRename);
|
||||
temp_env::async_with_vars([(crate::object_api::ENV_RUSTFS_ENCRYPTED_RANGE_SEEK, Some("true"))], async {
|
||||
let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
|
||||
let signaling = Arc::new(SignalingLockClient::new(Arc::new(LocalClient::with_manager(manager))));
|
||||
let lockers: Vec<Arc<dyn LockClient>> = vec![signaling.clone()];
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks_with_lockers(4, 0, 2, lockers).await;
|
||||
let bucket = "multipart-layout-lock-order-bucket";
|
||||
let object = "object";
|
||||
make_bucket_on_all(&disk_stores, bucket).await;
|
||||
let create_opts = ObjectOptions {
|
||||
user_defined: HashMap::from([(SSEC_ALGORITHM_HEADER.to_string(), "AES256".to_string())]),
|
||||
..Default::default()
|
||||
};
|
||||
let (upload_id, parts) = stage_upload_with_create_opts(&set_disks, bucket, object, &[0x43; 4096], &create_opts).await;
|
||||
let upload_id_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id);
|
||||
let upload_resource = rustfs_lock::ObjectKey::new(RUSTFS_META_MULTIPART_BUCKET, upload_id_path);
|
||||
let object_resource = rustfs_lock::ObjectKey::new(bucket, object);
|
||||
signaling.set_target(upload_resource.clone());
|
||||
signaling.clear_observed();
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
|
||||
let barrier = MultipartCommitBarrier::install(bucket, object, MultipartCommitPause::AfterRename);
|
||||
|
||||
let complete_store = set_disks.clone();
|
||||
let complete_upload_id = upload_id.clone();
|
||||
let complete = tokio::spawn(async move {
|
||||
complete_store
|
||||
.complete_multipart_upload(bucket, object, &complete_upload_id, parts, &ObjectOptions::default())
|
||||
.await
|
||||
});
|
||||
barrier.wait_until_paused().await;
|
||||
|
||||
let observed = signaling.observed();
|
||||
let object_position = observed
|
||||
.iter()
|
||||
.position(|resource| resource == &object_resource)
|
||||
.expect("completion should acquire the object lock");
|
||||
let upload_position = observed
|
||||
.iter()
|
||||
.position(|resource| resource == &upload_resource)
|
||||
.expect("completion should acquire the upload lock");
|
||||
assert!(object_position < upload_position, "completion must acquire object before upload");
|
||||
|
||||
let abort_store = set_disks.clone();
|
||||
let abort_upload_id = upload_id.clone();
|
||||
let abort = tokio::spawn(async move {
|
||||
abort_store
|
||||
.abort_multipart_upload(bucket, object, &abort_upload_id, &ObjectOptions::default())
|
||||
.await
|
||||
});
|
||||
signaling.wait_for_attempts(2).await;
|
||||
tokio::task::yield_now().await;
|
||||
assert!(!abort.is_finished(), "abort must wait until completion releases the upload lock");
|
||||
|
||||
let list_store = set_disks.clone();
|
||||
let list_upload_id = upload_id.clone();
|
||||
let list = tokio::spawn(async move {
|
||||
list_store
|
||||
.list_object_parts(bucket, object, &list_upload_id, None, MAX_PARTS_COUNT, &ObjectOptions::default())
|
||||
.await
|
||||
});
|
||||
signaling.wait_for_attempts(3).await;
|
||||
tokio::task::yield_now().await;
|
||||
assert!(!list.is_finished(), "ListParts must wait until completion releases the upload lock");
|
||||
|
||||
barrier.release();
|
||||
complete
|
||||
let complete_store = set_disks.clone();
|
||||
let complete_upload_id = upload_id.clone();
|
||||
let complete = tokio::spawn(async move {
|
||||
complete_store
|
||||
.complete_multipart_upload(bucket, object, &complete_upload_id, parts, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("completion task should not panic")
|
||||
.expect("completion should commit after the barrier is released");
|
||||
let abort_err = abort
|
||||
});
|
||||
barrier.wait_until_paused().await;
|
||||
|
||||
let observed = signaling.observed();
|
||||
let object_position = observed
|
||||
.iter()
|
||||
.position(|resource| resource == &object_resource)
|
||||
.expect("completion should acquire the object lock");
|
||||
let upload_position = observed
|
||||
.iter()
|
||||
.position(|resource| resource == &upload_resource)
|
||||
.expect("completion should acquire the upload lock");
|
||||
assert!(object_position < upload_position, "completion must acquire object before upload");
|
||||
|
||||
let abort_store = set_disks.clone();
|
||||
let abort_upload_id = upload_id.clone();
|
||||
let abort = tokio::spawn(async move {
|
||||
abort_store
|
||||
.abort_multipart_upload(bucket, object, &abort_upload_id, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("abort task should not panic")
|
||||
.expect_err("the committed upload should no longer exist when abort acquires the lock");
|
||||
assert!(
|
||||
matches!(abort_err, StorageError::InvalidUploadID(..)),
|
||||
"abort should return InvalidUploadID after completion, got {abort_err:?}"
|
||||
);
|
||||
let list_err = list
|
||||
});
|
||||
signaling.wait_for_attempts(2).await;
|
||||
tokio::task::yield_now().await;
|
||||
assert!(!abort.is_finished(), "abort must wait until completion releases the upload lock");
|
||||
|
||||
let list_store = set_disks.clone();
|
||||
let list_upload_id = upload_id.clone();
|
||||
let list = tokio::spawn(async move {
|
||||
list_store
|
||||
.list_object_parts(bucket, object, &list_upload_id, None, MAX_PARTS_COUNT, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("ListParts task should not panic")
|
||||
.expect_err("the committed upload should no longer exist when ListParts acquires the lock");
|
||||
assert!(
|
||||
matches!(list_err, StorageError::InvalidUploadID(..)),
|
||||
"ListParts should return InvalidUploadID after completion, got {list_err:?}"
|
||||
);
|
||||
},
|
||||
)
|
||||
});
|
||||
signaling.wait_for_attempts(3).await;
|
||||
tokio::task::yield_now().await;
|
||||
assert!(!list.is_finished(), "ListParts must wait until completion releases the upload lock");
|
||||
|
||||
barrier.release();
|
||||
complete
|
||||
.await
|
||||
.expect("completion task should not panic")
|
||||
.expect("completion should commit after the barrier is released");
|
||||
let abort_err = abort
|
||||
.await
|
||||
.expect("abort task should not panic")
|
||||
.expect_err("the committed upload should no longer exist when abort acquires the lock");
|
||||
assert!(matches!(abort_err, StorageError::InvalidUploadID(..)));
|
||||
let list_err = list
|
||||
.await
|
||||
.expect("ListParts task should not panic")
|
||||
.expect_err("the committed upload should no longer exist when ListParts acquires the lock");
|
||||
assert!(matches!(list_err, StorageError::InvalidUploadID(..)));
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
|
||||
@@ -2320,25 +2320,13 @@ async fn pause_transition_commit(bucket: &str, object: &str, pause: TransitionCo
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn persisted_transition_version(
|
||||
remote_version: &str,
|
||||
) -> std::io::Result<(Option<String>, rustfs_filemeta::TransitionVersionState)> {
|
||||
persisted_transition_version_with_gate(remote_version, remote_version_state_writer_enabled())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn remote_version_state_writer_enabled() -> bool {
|
||||
remote_version_state_writer_fleet_proof().is_some()
|
||||
}
|
||||
|
||||
fn remote_version_state_writer_fleet_proof() -> Option<crate::services::notification_sys::RemoteVersionStateFleetProofToken> {
|
||||
remote_version_state_writer_requested()
|
||||
.then(crate::services::notification_sys::acquire_remote_version_state_fleet_proof)
|
||||
.flatten()
|
||||
}
|
||||
|
||||
fn remote_version_state_writer_requested() -> bool {
|
||||
remote_version_state_writer_enabled_for(
|
||||
rustfs_utils::get_env_bool(
|
||||
rustfs_config::ENV_TIER_REMOTE_VERSION_STATE_WRITE,
|
||||
@@ -2348,25 +2336,11 @@ fn remote_version_state_writer_requested() -> bool {
|
||||
rustfs_config::ENV_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED,
|
||||
rustfs_config::DEFAULT_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED,
|
||||
),
|
||||
true,
|
||||
)
|
||||
}
|
||||
|
||||
fn remote_version_state_writer_fleet_proof_matches(
|
||||
proof: &crate::services::notification_sys::RemoteVersionStateFleetProofToken,
|
||||
) -> bool {
|
||||
remote_version_state_writer_fleet_proof_matches_for(
|
||||
remote_version_state_writer_requested(),
|
||||
crate::services::notification_sys::remote_version_state_fleet_proof_matches(proof),
|
||||
)
|
||||
}
|
||||
|
||||
fn remote_version_state_writer_fleet_proof_matches_for(requested: bool, fleet_proof_matches: bool) -> bool {
|
||||
requested && fleet_proof_matches
|
||||
}
|
||||
|
||||
fn remote_version_state_writer_enabled_for(requested: bool, fleet_confirmed: bool, fleet_proof_valid: bool) -> bool {
|
||||
requested && fleet_confirmed && fleet_proof_valid
|
||||
fn remote_version_state_writer_enabled_for(requested: bool, fleet_confirmed: bool) -> bool {
|
||||
requested && fleet_confirmed
|
||||
}
|
||||
|
||||
fn persisted_transition_version_with_gate(
|
||||
@@ -2385,7 +2359,7 @@ fn persisted_transition_version_with_gate(
|
||||
Ok(_) => Ok((Some(remote_version.to_string()), rustfs_filemeta::TransitionVersionState::Exact)),
|
||||
Err(_) if !remote_version_state_writer_enabled => Err(std::io::Error::new(
|
||||
std::io::ErrorKind::Unsupported,
|
||||
"opaque remote tier versions require the operator-attested live fleet capability gate",
|
||||
"opaque remote tier versions require the operator-attested fleet gate",
|
||||
)),
|
||||
Err(_) if remote_version == "null" => {
|
||||
Ok((Some(remote_version.to_string()), rustfs_filemeta::TransitionVersionState::SuspendedNull))
|
||||
@@ -2631,7 +2605,7 @@ mod transition_upload_completion_tests {
|
||||
mod transition_version_id_tests {
|
||||
use super::{
|
||||
TransitionUploadCandidate, persisted_transition_version, persisted_transition_version_with_gate,
|
||||
remote_version_state_writer_enabled_for, remote_version_state_writer_fleet_proof_matches_for,
|
||||
remote_version_state_writer_enabled_for,
|
||||
};
|
||||
use rustfs_filemeta::TransitionVersionState;
|
||||
use uuid::Uuid;
|
||||
@@ -2674,35 +2648,15 @@ mod transition_version_id_tests {
|
||||
|
||||
#[test]
|
||||
fn remote_version_state_writer_requires_request_and_fleet_confirmation() {
|
||||
for (case, requested, fleet_confirmed, fleet_proof_valid, expected) in [
|
||||
("old defaults", false, false, false, false),
|
||||
("missing fleet confirmation", true, false, true, false),
|
||||
("missing local opt-in", false, true, true, false),
|
||||
("missing fleet proof", true, true, false, false),
|
||||
("explicitly unconfirmed fleet", true, false, true, false),
|
||||
("rolled-back writer", false, true, true, false),
|
||||
("fully upgraded fleet", true, true, true, true),
|
||||
for (case, requested, fleet_confirmed, expected) in [
|
||||
("old defaults", false, false, false),
|
||||
("missing fleet confirmation", true, false, false),
|
||||
("missing local opt-in", false, true, false),
|
||||
("explicitly unconfirmed fleet", true, false, false),
|
||||
("rolled-back writer", false, true, false),
|
||||
("fully upgraded fleet", true, true, true),
|
||||
] {
|
||||
assert_eq!(
|
||||
remote_version_state_writer_enabled_for(requested, fleet_confirmed, fleet_proof_valid),
|
||||
expected,
|
||||
"{case}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_version_state_commit_rechecks_operator_gate_and_live_proof() {
|
||||
for (case, requested, fleet_proof_matches, expected) in [
|
||||
("operator gate closed", false, true, false),
|
||||
("fleet proof changed", true, false, false),
|
||||
("current authorization", true, true, true),
|
||||
] {
|
||||
assert_eq!(
|
||||
remote_version_state_writer_fleet_proof_matches_for(requested, fleet_proof_matches),
|
||||
expected,
|
||||
"{case}"
|
||||
);
|
||||
assert_eq!(remote_version_state_writer_enabled_for(requested, fleet_confirmed), expected, "{case}");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3887,16 +3841,6 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
let dest_obj = transaction.remote_object.clone();
|
||||
let mut transition_meta = (*oi.user_defined).clone();
|
||||
transition_meta.insert("name".to_string(), object.to_string());
|
||||
rustfs_utils::http::metadata_compat::insert_str(
|
||||
&mut transition_meta,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TRANSACTION_ID,
|
||||
transaction.transaction_id.to_string(),
|
||||
);
|
||||
rustfs_utils::http::metadata_compat::insert_str(
|
||||
&mut transition_meta,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
|
||||
rustfs_utils::crypto::hex(transaction.backend_fingerprint),
|
||||
);
|
||||
|
||||
if let Some(content_type) = oi.content_type.as_ref().filter(|value| !value.is_empty()) {
|
||||
transition_meta.insert(CONTENT_TYPE.to_ascii_lowercase(), content_type.clone());
|
||||
@@ -4007,24 +3951,20 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
delete_transition_transaction_after_remote_cleanup(transaction_api.as_ref(), transaction_id, bucket, object).await;
|
||||
return Err(err.into());
|
||||
}
|
||||
let fleet_proof = remote_version_state_writer_fleet_proof();
|
||||
let remote_version_requires_fleet_proof =
|
||||
!candidate.remote_version().is_empty() && Uuid::parse_str(candidate.remote_version()).is_err();
|
||||
let (transition_version_id, transition_version_state) =
|
||||
match persisted_transition_version_with_gate(candidate.remote_version(), fleet_proof.is_some()) {
|
||||
Ok(version) => version,
|
||||
Err(err) => {
|
||||
let cleanup_api = transition_cleanup_store(&self.ctx).await;
|
||||
if let Err(cleanup_err) = upload_cleanup.cleanup_rejected_upload(cleanup_api).await {
|
||||
return Err(StorageError::Io(std::io::Error::other(format!(
|
||||
"{err}; rejected remote upload cleanup failed: {cleanup_err}"
|
||||
))));
|
||||
}
|
||||
delete_transition_transaction_after_remote_cleanup(transaction_api.as_ref(), transaction_id, bucket, object)
|
||||
.await;
|
||||
return Err(err.into());
|
||||
let (transition_version_id, transition_version_state) = match persisted_transition_version(candidate.remote_version()) {
|
||||
Ok(version) => version,
|
||||
Err(err) => {
|
||||
let cleanup_api = transition_cleanup_store(&self.ctx).await;
|
||||
if let Err(cleanup_err) = upload_cleanup.cleanup_rejected_upload(cleanup_api).await {
|
||||
return Err(StorageError::Io(std::io::Error::other(format!(
|
||||
"{err}; rejected remote upload cleanup failed: {cleanup_err}"
|
||||
))));
|
||||
}
|
||||
};
|
||||
delete_transition_transaction_after_remote_cleanup(transaction_api.as_ref(), transaction_id, bucket, object)
|
||||
.await;
|
||||
return Err(err.into());
|
||||
}
|
||||
};
|
||||
if let Err(err) = advance_and_save_transition_transaction(
|
||||
transaction_api.as_ref(),
|
||||
&mut transaction,
|
||||
@@ -4140,21 +4080,6 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
}
|
||||
#[cfg(test)]
|
||||
pause_transition_commit(bucket, object, TransitionCommitPause::AfterLeaseValidation).await;
|
||||
// This check is the fleet-proof lease linearization point. Revocation
|
||||
// blocks later commits; an already-authorized local quorum commit is
|
||||
// allowed to finish without holding a synchronous lock across I/O.
|
||||
if remote_version_requires_fleet_proof
|
||||
&& !fleet_proof
|
||||
.as_ref()
|
||||
.is_some_and(remote_version_state_writer_fleet_proof_matches)
|
||||
{
|
||||
drop(transition_lock_guard);
|
||||
if upload_cleanup.cleanup().await.is_ok() {
|
||||
delete_transition_transaction_after_remote_cleanup(transaction_api.as_ref(), transaction_id, bucket, object)
|
||||
.await;
|
||||
}
|
||||
return Err(Error::other("remote version state fleet capability changed during transition"));
|
||||
}
|
||||
if let Err(err) = advance_and_save_transition_transaction(
|
||||
transaction_api.as_ref(),
|
||||
&mut transaction,
|
||||
|
||||
@@ -310,13 +310,12 @@ impl ECStore {
|
||||
meta.set_created(opts.created_at);
|
||||
|
||||
if opts.lock_enabled {
|
||||
meta.object_lock_config_xml =
|
||||
crate::bucket::utils::serialize::<ObjectLockConfiguration>(&ENABLED_OBJECT_LOCK_CONFIG)?;
|
||||
meta.versioning_config_xml = crate::bucket::utils::serialize::<VersioningConfiguration>(&ENABLED_VERSIONING_CONFIG)?;
|
||||
meta.object_lock_config_xml = crate::bucket::utils::serialize::<ObjectLockConfiguration>(&enableObjcetLockConfig)?;
|
||||
meta.versioning_config_xml = crate::bucket::utils::serialize::<VersioningConfiguration>(&enableVersioningConfig)?;
|
||||
}
|
||||
|
||||
if opts.versioning_enabled {
|
||||
meta.versioning_config_xml = crate::bucket::utils::serialize::<VersioningConfiguration>(&ENABLED_VERSIONING_CONFIG)?;
|
||||
meta.versioning_config_xml = crate::bucket::utils::serialize::<VersioningConfiguration>(&enableVersioningConfig)?;
|
||||
}
|
||||
|
||||
await_bucket_namespace_operation(
|
||||
@@ -569,7 +568,6 @@ mod tests {
|
||||
};
|
||||
use crate::bucket::metadata::table_bucket_catalog_metadata_prefix;
|
||||
use crate::bucket::metadata_sys;
|
||||
use crate::cluster::rpc::peer_s3_client::install_delete_bucket_empty_scan_barrier;
|
||||
use crate::disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET};
|
||||
use crate::error::StorageError;
|
||||
use crate::object_api::{ObjectOptions, PutObjReader};
|
||||
@@ -591,7 +589,6 @@ mod tests {
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::time::{Duration, SystemTime};
|
||||
use time::OffsetDateTime;
|
||||
use tokio::io::AsyncReadExt;
|
||||
use tokio::sync::{Notify, OnceCell};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use uuid::Uuid;
|
||||
@@ -1051,7 +1048,7 @@ mod tests {
|
||||
.await
|
||||
.expect("bucket should be created");
|
||||
for disk_path in &disk_paths {
|
||||
tokio::fs::create_dir_all(disk_path.join(&bucket).join("empty-directory/nested/leaf"))
|
||||
tokio::fs::create_dir_all(disk_path.join(&bucket).join("empty-directory"))
|
||||
.await
|
||||
.expect("empty directory remnant should be created");
|
||||
}
|
||||
@@ -1260,55 +1257,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn bucket_delete_preserves_put_committed_after_empty_scan() {
|
||||
let (_, ecstore) = setup_bucket_delete_test_env().await;
|
||||
let bucket = format!("bucket-delete-empty-scan-race-{}", Uuid::new_v4().simple());
|
||||
let object = "committed-after-empty-scan";
|
||||
let payload = b"object committed after DeleteBucket empty scan".to_vec();
|
||||
|
||||
ecstore
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket should be created");
|
||||
|
||||
let barrier = install_delete_bucket_empty_scan_barrier();
|
||||
let delete_store = ecstore.clone();
|
||||
let delete_bucket = bucket.clone();
|
||||
let delete = tokio::spawn(async move {
|
||||
delete_store
|
||||
.delete_bucket(&delete_bucket, &DeleteBucketOptions::default())
|
||||
.await
|
||||
});
|
||||
barrier.wait_until_paused().await;
|
||||
|
||||
let mut put_reader = PutObjReader::from_vec(payload.clone());
|
||||
ecstore
|
||||
.put_object(&bucket, object, &mut put_reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("PUT should commit while DeleteBucket is paused after its empty scan");
|
||||
|
||||
barrier.release();
|
||||
let err = delete
|
||||
.await
|
||||
.expect("DeleteBucket task should join")
|
||||
.expect_err("DeleteBucket must reject a PUT committed after its empty scan");
|
||||
assert!(matches!(err, StorageError::BucketNotEmpty(name) if name == bucket));
|
||||
|
||||
let mut reader = ecstore
|
||||
.get_object_reader(&bucket, object, None, http::HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("committed object should remain readable after DeleteBucket fails");
|
||||
let mut restored = Vec::new();
|
||||
reader
|
||||
.stream
|
||||
.read_to_end(&mut restored)
|
||||
.await
|
||||
.expect("object body should remain readable");
|
||||
assert_eq!(restored, payload);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn bucket_recreation_does_not_publish_unverified_usage() {
|
||||
|
||||
@@ -30,7 +30,6 @@ impl ECStore {
|
||||
};
|
||||
|
||||
let mut count_no_heal = 0;
|
||||
let mut first_error = None;
|
||||
for pool in self.pools.iter() {
|
||||
let (mut result, err) = pool.heal_format(dry_run).await?;
|
||||
if let Some(err) = err {
|
||||
@@ -38,8 +37,8 @@ impl ECStore {
|
||||
StorageError::NoHealRequired => {
|
||||
count_no_heal += 1;
|
||||
}
|
||||
err => {
|
||||
first_error.get_or_insert(err);
|
||||
_ => {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -48,9 +47,6 @@ impl ECStore {
|
||||
r.before.drives.append(&mut result.before.drives);
|
||||
r.after.drives.append(&mut result.after.drives);
|
||||
}
|
||||
if let Some(err) = first_error {
|
||||
return Ok((r, Some(err)));
|
||||
}
|
||||
if count_no_heal == self.pools.len() {
|
||||
info!(
|
||||
event = EVENT_HEAL_FORMAT_COMPLETED,
|
||||
@@ -169,134 +165,3 @@ impl ECStore {
|
||||
Err(StorageError::NotImplemented)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::disk::{DiskOption, format::FormatV3, new_disk};
|
||||
use crate::layout::endpoints::{Endpoints, PoolEndpoints};
|
||||
use crate::store::init_format::{load_format_erasure, save_format_file};
|
||||
|
||||
#[tokio::test]
|
||||
async fn handle_heal_format_continues_after_a_pool_error() {
|
||||
let canonical_format = FormatV3::new(1, 3);
|
||||
let mut foreign_format = canonical_format.clone();
|
||||
foreign_format.id = Uuid::new_v4();
|
||||
let mut temp_dirs = Vec::new();
|
||||
let mut endpoints = Vec::new();
|
||||
let mut disks = Vec::new();
|
||||
|
||||
for disk_index in 0..3 {
|
||||
let temp_dir = tempfile::tempdir().expect("temporary disk root should be created");
|
||||
let mut endpoint = Endpoint::try_from(temp_dir.path().to_str().expect("temporary path should be UTF-8"))
|
||||
.expect("temporary endpoint should parse");
|
||||
endpoint.set_pool_index(0);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(disk_index);
|
||||
let disk = new_disk(
|
||||
&endpoint,
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("temporary disk should open");
|
||||
let mut disk_format = foreign_format.clone();
|
||||
disk_format.erasure.this = foreign_format.erasure.sets[0][disk_index];
|
||||
save_format_file(&Some(disk.clone()), &Some(disk_format))
|
||||
.await
|
||||
.expect("foreign format should be written");
|
||||
temp_dirs.push(temp_dir);
|
||||
endpoints.push(endpoint);
|
||||
disks.push(Some(disk));
|
||||
}
|
||||
|
||||
let pool_endpoints = PoolEndpoints {
|
||||
legacy: false,
|
||||
set_count: 1,
|
||||
drives_per_set: 3,
|
||||
endpoints: Endpoints::from(endpoints),
|
||||
cmd_line: "foreign-format-majority-test".to_string(),
|
||||
platform: "test".to_string(),
|
||||
};
|
||||
let pool = Sets::new(disks, &pool_endpoints, &canonical_format, 0, 1)
|
||||
.await
|
||||
.expect("test pool should build around the cached canonical format");
|
||||
|
||||
let mut recoverable_format = FormatV3::new(1, 3);
|
||||
recoverable_format.id = canonical_format.id;
|
||||
let mut recoverable_temp_dirs = Vec::new();
|
||||
let mut recoverable_endpoints = Vec::new();
|
||||
let mut recoverable_disks = Vec::new();
|
||||
let mut unformatted_disk = None;
|
||||
for disk_index in 0..3 {
|
||||
let temp_dir = tempfile::tempdir().expect("temporary disk root should be created");
|
||||
let mut endpoint = Endpoint::try_from(temp_dir.path().to_str().expect("temporary path should be UTF-8"))
|
||||
.expect("temporary endpoint should parse");
|
||||
endpoint.set_pool_index(1);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(disk_index);
|
||||
let disk = new_disk(
|
||||
&endpoint,
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("temporary disk should open");
|
||||
if disk_index < 2 {
|
||||
let mut disk_format = recoverable_format.clone();
|
||||
disk_format.erasure.this = recoverable_format.erasure.sets[0][disk_index];
|
||||
save_format_file(&Some(disk.clone()), &Some(disk_format))
|
||||
.await
|
||||
.expect("recoverable format should be written");
|
||||
} else {
|
||||
unformatted_disk = Some(disk.clone());
|
||||
}
|
||||
recoverable_temp_dirs.push(temp_dir);
|
||||
recoverable_endpoints.push(endpoint);
|
||||
recoverable_disks.push(Some(disk));
|
||||
}
|
||||
let recoverable_pool_endpoints = PoolEndpoints {
|
||||
legacy: false,
|
||||
set_count: 1,
|
||||
drives_per_set: 3,
|
||||
endpoints: Endpoints::from(recoverable_endpoints),
|
||||
cmd_line: "recoverable-format-test".to_string(),
|
||||
platform: "test".to_string(),
|
||||
};
|
||||
let recoverable_pool = Sets::new(recoverable_disks, &recoverable_pool_endpoints, &recoverable_format, 1, 1)
|
||||
.await
|
||||
.expect("recoverable test pool should build");
|
||||
|
||||
let endpoint_pools = EndpointServerPools::from(vec![pool_endpoints.clone(), recoverable_pool_endpoints.clone()]);
|
||||
let store = ECStore {
|
||||
id: canonical_format.id,
|
||||
disk_map: HashMap::new(),
|
||||
pools: vec![pool, recoverable_pool],
|
||||
peer_sys: S3PeerSys::new(&endpoint_pools),
|
||||
pool_meta: RwLock::new(PoolMeta::default()),
|
||||
rebalance_meta: RwLock::new(None),
|
||||
decommission_cancelers: RwLock::new(Vec::new()),
|
||||
start_gate: Mutex::new(()),
|
||||
pool_meta_save_gate: Mutex::new(()),
|
||||
ctx: crate::runtime::instance::bootstrap_ctx(),
|
||||
};
|
||||
|
||||
let (result, err) = store
|
||||
.handle_heal_format(false)
|
||||
.await
|
||||
.expect("format heal should return the typed pool error");
|
||||
assert!(
|
||||
matches!(err, Some(StorageError::CorruptedFormat)),
|
||||
"foreign format majority must not be downgraded to a successful heal: {err:?}"
|
||||
);
|
||||
assert_eq!(result.disk_count, 3, "the recoverable pool should still be inspected");
|
||||
let healed = load_format_erasure(&unformatted_disk.expect("the unformatted disk handle should be retained"), true)
|
||||
.await
|
||||
.expect("the later pool should be healed despite the first pool error");
|
||||
assert_eq!(healed.erasure.this, recoverable_format.erasure.sets[0][2]);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -101,10 +101,6 @@ fn should_retry_local_decommission_resume(err: &Error, attempt: usize) -> bool {
|
||||
matches!(err, Error::ConfigNotFound) && attempt < LOCAL_DECOMMISSION_RESUME_MAX_CONFIG_RETRIES
|
||||
}
|
||||
|
||||
fn should_retry_format_load(err: &Error) -> bool {
|
||||
!matches!(err, Error::CorruptedFormat)
|
||||
}
|
||||
|
||||
fn should_auto_start_rebalance_after_init(decommission_running: bool, rebalance_meta_loaded: bool) -> bool {
|
||||
rebalance_meta_loaded && !decommission_running
|
||||
}
|
||||
@@ -298,7 +294,7 @@ impl ECStore {
|
||||
// periodic monitoring until format loading succeeds. Startup RPC
|
||||
// failures can still spawn recovery probes for peers that come up
|
||||
// after this node.
|
||||
let (mut disks, errs) = init_format::init_disks(
|
||||
let (disks, errs) = init_format::init_disks(
|
||||
&pool_eps.endpoints,
|
||||
&DiskOption {
|
||||
cleanup: true,
|
||||
@@ -315,7 +311,7 @@ impl ECStore {
|
||||
loop {
|
||||
match init_format::connect_load_init_formats(
|
||||
pool_first_is_local,
|
||||
&mut disks,
|
||||
&disks,
|
||||
pool_eps.set_count,
|
||||
pool_eps.drives_per_set,
|
||||
deployment_id,
|
||||
@@ -323,7 +319,6 @@ impl ECStore {
|
||||
.await
|
||||
{
|
||||
Ok(fm) => break Ok(fm),
|
||||
Err(e) if !should_retry_format_load(&e) => break Err(e),
|
||||
// Wrap the final error if we are giving up
|
||||
Err(e) if times >= 10 => {
|
||||
break Err(Error::other(format!("store init failed to load formats after {times} retries: {e}")));
|
||||
@@ -556,7 +551,7 @@ mod tests {
|
||||
LOCAL_DECOMMISSION_RESUME_MAX_CONFIG_RETRIES, load_pool_meta_for_startup, pool_first_endpoint_is_local,
|
||||
pool_meta_has_active_decommission, preflight_startup_rpc_secret_with, resolve_startup_pool_defaults_with,
|
||||
resolve_store_init_stage_result, save_validated_pool_meta_for_startup, should_auto_start_rebalance_after_init,
|
||||
should_retry_format_load, should_retry_local_decommission_resume, wait_for_local_decommission_resume_delay,
|
||||
should_retry_local_decommission_resume, wait_for_local_decommission_resume_delay,
|
||||
};
|
||||
#[cfg(feature = "test-util")]
|
||||
use crate::{
|
||||
@@ -567,12 +562,10 @@ mod tests {
|
||||
},
|
||||
tier_sweeper::Jentry,
|
||||
transition_transaction::{
|
||||
TRANSITION_TRANSACTION_RECORD_PREFIX, TransitionCleanupDecision, TransitionCleanupProof, TransitionOperatorError,
|
||||
TransitionOperatorProbe, TransitionRemoteVersion, TransitionSourceIdentity, TransitionSourceVersionMode,
|
||||
TransitionTransaction, TransitionTransactionInit, TransitionTransactionState,
|
||||
delete_transition_candidate_for_operator, finalize_missing_transition_transaction_for_operator,
|
||||
inspect_transition_transaction_for_operator, load_transition_transaction_record,
|
||||
recover_transition_transaction_records, save_transition_transaction_record,
|
||||
TRANSITION_TRANSACTION_RECORD_PREFIX, TransitionCleanupDecision, TransitionCleanupProof, TransitionRemoteVersion,
|
||||
TransitionSourceIdentity, TransitionSourceVersionMode, TransitionTransaction, TransitionTransactionInit,
|
||||
TransitionTransactionState, load_transition_transaction_record, recover_transition_transaction_records,
|
||||
save_transition_transaction_record,
|
||||
},
|
||||
},
|
||||
client::transition_api::ReaderImpl,
|
||||
@@ -582,7 +575,6 @@ mod tests {
|
||||
services::tier::{
|
||||
test_util::{MockWarmBackend, MockWarmOp, TransitionCleanupStoreBarrier, register_mock_tier},
|
||||
tier::{TIER_CONFIG_FILE, TierConfigMgr},
|
||||
tier_config::{TierConfig, TierType, TierWasabi},
|
||||
tier_mutation_intent::{
|
||||
TIER_MUTATION_INTENT_RECORD_PREFIX, TierMutationIntent, TierMutationIntentKind, TierMutationIntentState,
|
||||
TierMutationIntentTarget, advance_tier_mutation_intent_record_idempotent, delete_tier_mutation_intent_record,
|
||||
@@ -778,13 +770,6 @@ mod tests {
|
||||
assert!(!should_retry_local_decommission_resume(&StorageError::SlowDown, 0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_should_retry_format_load_rejects_permanent_corruption() {
|
||||
assert!(!should_retry_format_load(&StorageError::CorruptedFormat));
|
||||
assert!(should_retry_format_load(&StorageError::ErasureReadQuorum));
|
||||
assert!(should_retry_format_load(&StorageError::FirstDiskWait));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_should_auto_start_rebalance_after_init_allows_loaded_rebalance_without_decommission() {
|
||||
assert!(should_auto_start_rebalance_after_init(false, true));
|
||||
@@ -1178,37 +1163,6 @@ mod tests {
|
||||
.len()
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
async fn register_transition_reconcile_test_tier(
|
||||
handle: &Arc<tokio::sync::RwLock<TierConfigMgr>>,
|
||||
tier_name: &str,
|
||||
) -> MockWarmBackend {
|
||||
let backend = MockWarmBackend::new();
|
||||
let mut manager = handle.write().await;
|
||||
manager.tiers.insert(
|
||||
tier_name.to_string(),
|
||||
TierConfig {
|
||||
version: "v1".to_string(),
|
||||
tier_type: TierType::Wasabi,
|
||||
name: tier_name.to_string(),
|
||||
wasabi: Some(TierWasabi {
|
||||
name: tier_name.to_string(),
|
||||
endpoint: "https://s3.wasabisys.com".to_string(),
|
||||
access_key: "test-access-key".to_string(),
|
||||
secret_key: "test-secret-key".to_string(),
|
||||
bucket: "mock-tier".to_string(),
|
||||
prefix: format!("mock/{}/", uuid::Uuid::new_v4()),
|
||||
region: "us-east-1".to_string(),
|
||||
}),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
manager
|
||||
.install_test_driver(tier_name, Box::new(backend.clone()))
|
||||
.expect("mock fallback tier driver should install");
|
||||
backend
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
async fn wait_for_tier_delete_journal_recovery(
|
||||
store: Arc<crate::store::ECStore>,
|
||||
@@ -2784,7 +2738,7 @@ mod tests {
|
||||
.await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
|
||||
let backend = register_transition_reconcile_test_tier(&ctx.tier_config_mgr(), tier_name).await;
|
||||
let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
|
||||
let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
|
||||
.await
|
||||
.expect("tier lease should resolve")
|
||||
@@ -2855,157 +2809,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn operator_reconcile_deletes_exact_candidate_before_finalizing_record() {
|
||||
let temp_dir = tempfile::tempdir().expect("create temp store dir");
|
||||
let (ctx, store, _shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transition-operator-reconcile", &[4])).await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
|
||||
let tier_name = "TXOPERATOR";
|
||||
let backend = register_transition_reconcile_test_tier(&ctx.tier_config_mgr(), tier_name).await;
|
||||
let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
|
||||
.await
|
||||
.expect("tier lease should resolve")
|
||||
.backend_identity();
|
||||
let mut transaction = TransitionTransaction::new(TransitionTransactionInit {
|
||||
deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
|
||||
transaction_id: uuid::Uuid::new_v4(),
|
||||
owner_epoch: uuid::Uuid::new_v4(),
|
||||
write_id: uuid::Uuid::new_v4(),
|
||||
source: TransitionSourceIdentity {
|
||||
bucket: "source-bucket".to_string(),
|
||||
object: "source-object".to_string(),
|
||||
version_id: None,
|
||||
data_dir: uuid::Uuid::new_v4(),
|
||||
mod_time_unix_nanos: 1_770_000_000_000_000_000,
|
||||
size: 42,
|
||||
etag: "source-etag".to_string(),
|
||||
version_mode: TransitionSourceVersionMode::Unversioned,
|
||||
},
|
||||
tier_name: tier_name.to_string(),
|
||||
backend_fingerprint: backend_identity,
|
||||
not_after_unix_nanos: 1,
|
||||
})
|
||||
.expect("transaction should build");
|
||||
transaction
|
||||
.advance(transaction.fence(), TransitionTransactionState::UploadOutcomeUnknown, None)
|
||||
.expect("transaction should enter unknown upload outcome state");
|
||||
|
||||
let remote_version = uuid::Uuid::new_v4().to_string();
|
||||
backend.set_put_remote_version(Some(remote_version.clone())).await;
|
||||
let candidate = bytes::Bytes::from_static(b"operator-confirmed transition candidate");
|
||||
backend
|
||||
.put(
|
||||
&transaction.remote_object,
|
||||
ReaderImpl::Body(candidate.clone()),
|
||||
i64::try_from(candidate.len()).expect("test candidate length should fit i64"),
|
||||
)
|
||||
.await
|
||||
.expect("mock backend should accept candidate");
|
||||
save_transition_transaction_record(store.clone(), &transaction)
|
||||
.await
|
||||
.expect("transaction record should persist");
|
||||
|
||||
let status = inspect_transition_transaction_for_operator(store.clone(), transaction.transaction_id)
|
||||
.await
|
||||
.expect("operator inspection should probe the candidate");
|
||||
assert_eq!(status.probe, TransitionOperatorProbe::VersionedPresent(remote_version.clone()));
|
||||
|
||||
let wrong_version = uuid::Uuid::new_v4().to_string();
|
||||
let err = delete_transition_candidate_for_operator(store.clone(), transaction.transaction_id, &wrong_version)
|
||||
.await
|
||||
.expect_err("a mismatched exact version must fail before deleting a candidate");
|
||||
assert!(matches!(
|
||||
err,
|
||||
TransitionOperatorError::CandidateVersionMismatch {
|
||||
expected,
|
||||
actual: TransitionOperatorProbe::VersionedPresent(ref observed),
|
||||
} if expected == wrong_version && observed == &remote_version
|
||||
));
|
||||
assert!(backend.contains(&transaction.remote_object).await);
|
||||
assert_eq!(backend.exact_remove_count(), 0);
|
||||
load_transition_transaction_record(store.clone(), transaction.transaction_id)
|
||||
.await
|
||||
.expect("an incorrect exact version must retain the transaction journal");
|
||||
|
||||
let result = delete_transition_candidate_for_operator(store.clone(), transaction.transaction_id, &remote_version)
|
||||
.await
|
||||
.expect("operator-confirmed exact candidate should be deleted");
|
||||
assert_eq!(result.status.probe, TransitionOperatorProbe::Missing);
|
||||
assert!(result.journal_observed_after_delete);
|
||||
assert_eq!(backend.exact_remove_count(), 1);
|
||||
assert_eq!(backend.remove_versions().await, vec![(transaction.remote_object.clone(), remote_version)]);
|
||||
load_transition_transaction_record(store.clone(), transaction.transaction_id)
|
||||
.await
|
||||
.expect("candidate deletion must retain the transaction journal");
|
||||
|
||||
finalize_missing_transition_transaction_for_operator(store.clone(), transaction.transaction_id)
|
||||
.await
|
||||
.expect("a separately confirmed missing candidate should permit finalization");
|
||||
assert!(matches!(
|
||||
load_transition_transaction_record(store, transaction.transaction_id).await,
|
||||
Err(Error::ConfigNotFound)
|
||||
));
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn operator_finalize_retains_record_without_missing_proof() {
|
||||
let temp_dir = tempfile::tempdir().expect("create temp store dir");
|
||||
let (ctx, store, _shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transition-operator-fail-closed", &[4])).await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
|
||||
let tier_name = "TXOPERATORFAIL";
|
||||
let backend = register_transition_reconcile_test_tier(&ctx.tier_config_mgr(), tier_name).await;
|
||||
let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
|
||||
.await
|
||||
.expect("tier lease should resolve")
|
||||
.backend_identity();
|
||||
let mut transaction = TransitionTransaction::new(TransitionTransactionInit {
|
||||
deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
|
||||
transaction_id: uuid::Uuid::new_v4(),
|
||||
owner_epoch: uuid::Uuid::new_v4(),
|
||||
write_id: uuid::Uuid::new_v4(),
|
||||
source: TransitionSourceIdentity {
|
||||
bucket: "source-bucket".to_string(),
|
||||
object: "source-object".to_string(),
|
||||
version_id: None,
|
||||
data_dir: uuid::Uuid::new_v4(),
|
||||
mod_time_unix_nanos: 1_770_000_000_000_000_000,
|
||||
size: 42,
|
||||
etag: "source-etag".to_string(),
|
||||
version_mode: TransitionSourceVersionMode::Unversioned,
|
||||
},
|
||||
tier_name: tier_name.to_string(),
|
||||
backend_fingerprint: backend_identity,
|
||||
not_after_unix_nanos: 1,
|
||||
})
|
||||
.expect("transaction should build");
|
||||
transaction
|
||||
.advance(transaction.fence(), TransitionTransactionState::UploadOutcomeUnknown, None)
|
||||
.expect("transaction should enter unknown upload outcome state");
|
||||
save_transition_transaction_record(store.clone(), &transaction)
|
||||
.await
|
||||
.expect("transaction record should persist");
|
||||
backend
|
||||
.set_transition_candidate_probe_override(Some(TransitionCandidateProbe::Unsupported))
|
||||
.await;
|
||||
|
||||
assert!(matches!(
|
||||
finalize_missing_transition_transaction_for_operator(store.clone(), transaction.transaction_id).await,
|
||||
Err(TransitionOperatorError::CandidateNotMissing(TransitionOperatorProbe::Unsupported))
|
||||
));
|
||||
load_transition_transaction_record(store, transaction.transaction_id)
|
||||
.await
|
||||
.expect("an unsupported probe must retain the transaction journal");
|
||||
assert_eq!(backend.remove_count().await, 0);
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
@@ -3016,7 +2819,7 @@ mod tests {
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
|
||||
let tier_name = "TXRESPONSELOSS";
|
||||
let backend = register_transition_reconcile_test_tier(&ctx.tier_config_mgr(), tier_name).await;
|
||||
let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
|
||||
let bucket = "transition-response-loss-bucket";
|
||||
let object = "source.bin";
|
||||
store
|
||||
|
||||
+137
-1074
File diff suppressed because it is too large
Load Diff
@@ -6895,7 +6895,6 @@ mod test {
|
||||
)
|
||||
.await
|
||||
.expect("local disk should be created");
|
||||
disk.make_volume("bucket").await.expect("test bucket should be created");
|
||||
let mut fi = FileInfo::new("object", 1, 1);
|
||||
fi.volume = "bucket".to_owned();
|
||||
fi.name = "object".to_owned();
|
||||
|
||||
@@ -428,11 +428,11 @@ impl ECStore {
|
||||
}
|
||||
|
||||
lazy_static! {
|
||||
static ref ENABLED_OBJECT_LOCK_CONFIG: ObjectLockConfiguration = ObjectLockConfiguration {
|
||||
static ref enableObjcetLockConfig: ObjectLockConfiguration = ObjectLockConfiguration {
|
||||
object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
|
||||
..Default::default()
|
||||
};
|
||||
static ref ENABLED_VERSIONING_CONFIG: VersioningConfiguration = VersioningConfiguration {
|
||||
static ref enableVersioningConfig: VersioningConfiguration = VersioningConfiguration {
|
||||
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
|
||||
..Default::default()
|
||||
};
|
||||
@@ -989,7 +989,7 @@ mod tests {
|
||||
|
||||
init_local_disks(endpoint_pools.clone()).await.expect("init local disks");
|
||||
|
||||
let (mut disks, errs) = init_disks(
|
||||
let (disks, errs) = init_disks(
|
||||
&endpoint_pools.as_ref().first().expect("pool endpoints").endpoints,
|
||||
&DiskOption {
|
||||
cleanup: true,
|
||||
@@ -999,7 +999,7 @@ mod tests {
|
||||
.await;
|
||||
|
||||
assert!(errs.iter().all(|err| err.is_none()), "disk init should succeed: {errs:?}");
|
||||
connect_load_init_formats(true, &mut disks, 1, 4, None)
|
||||
connect_load_init_formats(true, &disks, 1, 4, None)
|
||||
.await
|
||||
.expect("initialize format metadata");
|
||||
|
||||
|
||||
@@ -1,43 +0,0 @@
|
||||
//! Test-only guard against tracing's process-global callsite-interest cache.
|
||||
//!
|
||||
//! Any test that installs its own subscriber and then asserts on span or event
|
||||
//! context is asserting on process-global state. See
|
||||
//! [`pin_callsite_interest_for_test`] for what goes wrong and why holding its
|
||||
//! guard fixes it.
|
||||
|
||||
/// Stop a sibling test thread from poisoning tracing's process-global callsite
|
||||
/// interest cache while this test asserts on span or event context.
|
||||
///
|
||||
/// `tracing` caches every callsite's `Interest` in **process-global** state, and
|
||||
/// the first thread to reach a callsite fixes that value. While at most one
|
||||
/// dispatcher is registered, tracing-core takes a fast path
|
||||
/// (`Dispatchers::rebuilder` -> `Rebuilder::JustOne`) that derives a
|
||||
/// newly-registered callsite's interest from whichever subscriber is current
|
||||
/// *on the registering thread*, and registration happens exactly once (guarded
|
||||
/// by a CAS in `DefaultCallsite::register`).
|
||||
///
|
||||
/// In a multi-threaded `cargo test` binary a sibling test therefore routinely
|
||||
/// reaches a production callsite first, from a thread with no subscriber
|
||||
/// installed: the interest is derived from that thread's `NoSubscriber` and
|
||||
/// cached as `Interest::never()` for the whole process. From then on the
|
||||
/// callsite is dead for *every* later caller, including a test that installed
|
||||
/// its own subscriber — an `info_span!` silently evaluates to `Span::none()`, and
|
||||
/// a `warn!`/`info!` event never fires at all.
|
||||
///
|
||||
/// Registering a second, inert dispatcher closes both halves of that race:
|
||||
///
|
||||
/// * constructing it rebuilds every *already-registered* callsite's interest
|
||||
/// against the live dispatcher set — which includes the caller's subscriber —
|
||||
/// repairing whatever a sibling may already have poisoned; and
|
||||
/// * holding it alive keeps tracing-core off the single-dispatcher fast path, so
|
||||
/// a callsite registered *later* by any thread is resolved against that live
|
||||
/// set instead of the registering thread's `NoSubscriber`.
|
||||
///
|
||||
/// Call this **after** installing the test's subscriber, and hold the returned
|
||||
/// guard for the rest of the test. Only the `cargo test` fallback needs it:
|
||||
/// nextest runs each test in its own process, where there is no sibling thread
|
||||
/// to lose the race to (see `docs/testing/README.md`).
|
||||
#[must_use = "callsite interest is only pinned while the returned guard is held"]
|
||||
pub(crate) fn pin_callsite_interest_for_test() -> tracing::Dispatch {
|
||||
tracing::Dispatch::new(tracing_core::subscriber::NoSubscriber::default())
|
||||
}
|
||||
@@ -556,7 +556,7 @@ where
|
||||
Ok(now)
|
||||
}
|
||||
|
||||
pub async fn list_policies(&self, bucket_name: &str) -> Result<HashMap<String, Policy>> {
|
||||
pub async fn list_polices(&self, bucket_name: &str) -> Result<HashMap<String, Policy>> {
|
||||
let mut m = HashMap::new();
|
||||
|
||||
self.api.load_policy_docs(&mut m).await?;
|
||||
@@ -588,15 +588,6 @@ where
|
||||
Ok(filtered)
|
||||
}
|
||||
|
||||
/// Backward-compatible misspelling retained until the next breaking release.
|
||||
#[deprecated(
|
||||
since = "1.0.0",
|
||||
note = "use list_policies instead; this alias will be removed in the next breaking release"
|
||||
)]
|
||||
pub async fn list_polices(&self, bucket_name: &str) -> Result<HashMap<String, Policy>> {
|
||||
self.list_policies(bucket_name).await
|
||||
}
|
||||
|
||||
pub async fn merge_policies(&self, name: &str) -> (String, Policy) {
|
||||
let mut policies = Vec::new();
|
||||
let mut to_merge = Vec::new();
|
||||
@@ -2193,7 +2184,7 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get_default_policies() -> HashMap<String, PolicyDoc> {
|
||||
pub fn get_default_policyes() -> HashMap<String, PolicyDoc> {
|
||||
let default_policies = &DEFAULT_POLICIES;
|
||||
default_policies
|
||||
.iter()
|
||||
@@ -2210,15 +2201,6 @@ pub fn get_default_policies() -> HashMap<String, PolicyDoc> {
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Backward-compatible misspelling retained until the next breaking release.
|
||||
#[deprecated(
|
||||
since = "1.0.0",
|
||||
note = "use get_default_policies instead; this alias will be removed in the next breaking release"
|
||||
)]
|
||||
pub fn get_default_policyes() -> HashMap<String, PolicyDoc> {
|
||||
get_default_policies()
|
||||
}
|
||||
|
||||
fn set_default_canned_policies(policies: &mut HashMap<String, PolicyDoc>) {
|
||||
let default_policies = &DEFAULT_POLICIES;
|
||||
for (k, v) in default_policies.iter() {
|
||||
@@ -2918,7 +2900,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_get_default_policies() {
|
||||
let policies = get_default_policies();
|
||||
let policies = get_default_policyes();
|
||||
|
||||
// Should contain some default policies
|
||||
assert!(!policies.is_empty());
|
||||
@@ -2931,12 +2913,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[allow(deprecated)]
|
||||
fn deprecated_get_default_policyes_matches_current_api() {
|
||||
assert_eq!(get_default_policyes().len(), get_default_policies().len());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_get_token_signing_key() {
|
||||
// This function returns the global action credential's secret key
|
||||
|
||||
@@ -22,7 +22,7 @@ use crate::{
|
||||
cache::{Cache, CacheEntity},
|
||||
error::{is_err_no_such_policy, is_err_no_such_user},
|
||||
keyring,
|
||||
manager::{extract_jwt_claims, extract_jwt_claims_allow_missing_exp, get_default_policies},
|
||||
manager::{extract_jwt_claims, extract_jwt_claims_allow_missing_exp, get_default_policyes},
|
||||
root_credentials,
|
||||
};
|
||||
use futures::future::join_all;
|
||||
@@ -1127,7 +1127,7 @@ impl Store for ObjectStore {
|
||||
let cache_snapshot = cache.snapshot();
|
||||
let listed_config_items = self.list_all_iamconfig_items().await?;
|
||||
|
||||
let mut policy_docs_cache = CacheEntity::new(get_default_policies());
|
||||
let mut policy_docs_cache = CacheEntity::new(get_default_policyes());
|
||||
|
||||
if let Some(policies_list) = listed_config_items.get(POLICIES_LIST_KEY) {
|
||||
// Load in fixed-size chunks so each policy is fetched exactly once.
|
||||
|
||||
+5
-20
@@ -18,7 +18,7 @@ use crate::error::is_err_no_such_temp_account;
|
||||
use crate::error::{Error, Result};
|
||||
use crate::federation::OIDC_VIRTUAL_PARENT_CLAIM;
|
||||
use crate::manager::extract_jwt_claims;
|
||||
use crate::manager::get_default_policies;
|
||||
use crate::manager::get_default_policyes;
|
||||
use crate::manager::{IamCache, IamSyncMetricsSnapshot};
|
||||
use crate::store::GroupInfo;
|
||||
use crate::store::MappedPolicy;
|
||||
@@ -249,7 +249,7 @@ impl<T: Store> IamSys<T> {
|
||||
}
|
||||
|
||||
pub async fn delete_policy(&self, name: &str, notify: bool) -> Result<()> {
|
||||
for k in get_default_policies().keys() {
|
||||
for k in get_default_policyes().keys() {
|
||||
if k == name {
|
||||
return Err(Error::other("system policy can not be deleted"));
|
||||
}
|
||||
@@ -291,17 +291,8 @@ impl<T: Store> IamSys<T> {
|
||||
self.store.api.load_mapped_policies(user_type, is_group, m).await
|
||||
}
|
||||
|
||||
pub async fn list_policies(&self, bucket_name: &str) -> Result<HashMap<String, Policy>> {
|
||||
self.store.list_policies(bucket_name).await
|
||||
}
|
||||
|
||||
/// Backward-compatible misspelling retained until the next breaking release.
|
||||
#[deprecated(
|
||||
since = "1.0.0",
|
||||
note = "use list_policies instead; this alias will be removed in the next breaking release"
|
||||
)]
|
||||
pub async fn list_polices(&self, bucket_name: &str) -> Result<HashMap<String, Policy>> {
|
||||
self.list_policies(bucket_name).await
|
||||
self.store.list_polices(bucket_name).await
|
||||
}
|
||||
|
||||
pub async fn list_policy_docs(&self, bucket_name: &str) -> Result<HashMap<String, PolicyDoc>> {
|
||||
@@ -1692,17 +1683,11 @@ mod tests {
|
||||
use super::*;
|
||||
use crate::cache::{Cache, CacheEntity};
|
||||
use crate::error::Error;
|
||||
use crate::manager::get_default_policies;
|
||||
use crate::manager::get_default_policyes;
|
||||
use crate::store::{GroupInfo, MappedPolicy, Store, UserType};
|
||||
use rustfs_credentials::{Credentials, init_global_action_credentials};
|
||||
use rustfs_policy::auth::{UserIdentity, get_new_credentials_with_metadata};
|
||||
use rustfs_policy::policy::Args;
|
||||
|
||||
#[test]
|
||||
#[allow(deprecated)]
|
||||
fn deprecated_list_polices_api_is_available() {
|
||||
let _ = IamSys::<StsTestMockStore>::list_polices;
|
||||
}
|
||||
use rustfs_policy::policy::action::{Action, AdminAction, S3Action};
|
||||
use rustfs_policy::policy::policy_uses_existing_object_tag_conditions;
|
||||
use serde_json::Value;
|
||||
@@ -1940,7 +1925,7 @@ mod tests {
|
||||
}
|
||||
|
||||
async fn load_all(&self, cache: &Cache) -> Result<()> {
|
||||
let mut policy_docs = get_default_policies();
|
||||
let mut policy_docs = get_default_policyes();
|
||||
let custom_claim_policy =
|
||||
Policy::parse_config(CUSTOM_STS_CLAIM_POLICY_JSON.as_bytes()).expect("custom STS claim policy should parse");
|
||||
policy_docs.insert(CUSTOM_STS_CLAIM_POLICY.to_string(), PolicyDoc::new(custom_claim_policy));
|
||||
|
||||
@@ -67,7 +67,6 @@ const INTERNODE_MSGPACK_JSON_FALLBACK_TOTAL: &str = "rustfs_system_network_inter
|
||||
const INTERNODE_MSGPACK_JSON_DECODE_ERROR_TOTAL: &str = "rustfs_system_network_internode_msgpack_json_decode_error_total";
|
||||
const INTERNODE_SIGNATURE_V1_FALLBACK_TOTAL: &str = "rustfs_system_network_internode_signature_v1_fallback_total";
|
||||
const INTERNODE_BODY_DIGEST_FALLBACK_TOTAL: &str = "rustfs_system_network_internode_body_digest_fallback_total";
|
||||
const INTERNODE_REPLAY_SCOPE_FALLBACK_TOTAL: &str = "rustfs_system_network_internode_replay_scope_fallback_total";
|
||||
const INTERNODE_REPLAY_CACHE_OVERFLOW_TOTAL: &str = "rustfs_system_network_internode_replay_cache_overflow_total";
|
||||
const ERASURE_WRITE_QUORUM_FAILURES_TOTAL: &str = "rustfs_system_storage_erasure_write_quorum_failures_total";
|
||||
|
||||
@@ -160,7 +159,6 @@ pub struct InternodeMetricsSnapshot {
|
||||
pub operation_write_shutdown_errors_total: u64,
|
||||
pub signature_v1_fallback_total: u64,
|
||||
pub body_digest_fallback_total: u64,
|
||||
pub replay_scope_fallback_total: u64,
|
||||
pub replay_cache_overflow_total: u64,
|
||||
}
|
||||
|
||||
@@ -182,7 +180,6 @@ pub struct InternodeMetrics {
|
||||
msgpack_json_decode_error_total: AtomicU64,
|
||||
signature_v1_fallback_total: AtomicU64,
|
||||
body_digest_fallback_total: AtomicU64,
|
||||
replay_scope_fallback_total: AtomicU64,
|
||||
replay_cache_overflow_total: AtomicU64,
|
||||
}
|
||||
|
||||
@@ -434,13 +431,6 @@ impl InternodeMetrics {
|
||||
counter!(INTERNODE_BODY_DIGEST_FALLBACK_TOTAL).increment(1);
|
||||
}
|
||||
|
||||
/// Count an accepted v1/v2 request that does not carry the replay-scoped signature. This is
|
||||
/// the convergence signal for `RUSTFS_INTERNODE_RPC_REPLAY_SCOPE_STRICT`.
|
||||
pub fn record_replay_scope_fallback(&self) {
|
||||
self.replay_scope_fallback_total.fetch_add(1, Ordering::Relaxed);
|
||||
counter!(INTERNODE_REPLAY_SCOPE_FALLBACK_TOTAL).increment(1);
|
||||
}
|
||||
|
||||
/// Count a body-bound internode RPC rejected because the replay-protection nonce cache was
|
||||
/// full. Overflow fails closed, so a sustained non-zero rate means
|
||||
/// `RUSTFS_INTERNODE_RPC_REPLAY_CACHE_CAPACITY` is undersized for this node's peak legitimate
|
||||
@@ -498,7 +488,6 @@ impl InternodeMetrics {
|
||||
operation_write_shutdown_errors_total: self.operation_write_shutdown_errors_total.load(Ordering::Relaxed),
|
||||
signature_v1_fallback_total: self.signature_v1_fallback_total.load(Ordering::Relaxed),
|
||||
body_digest_fallback_total: self.body_digest_fallback_total.load(Ordering::Relaxed),
|
||||
replay_scope_fallback_total: self.replay_scope_fallback_total.load(Ordering::Relaxed),
|
||||
replay_cache_overflow_total: self.replay_cache_overflow_total.load(Ordering::Relaxed),
|
||||
}
|
||||
}
|
||||
@@ -521,7 +510,6 @@ impl InternodeMetrics {
|
||||
self.msgpack_json_decode_error_total.store(0, Ordering::Relaxed);
|
||||
self.signature_v1_fallback_total.store(0, Ordering::Relaxed);
|
||||
self.body_digest_fallback_total.store(0, Ordering::Relaxed);
|
||||
self.replay_scope_fallback_total.store(0, Ordering::Relaxed);
|
||||
self.replay_cache_overflow_total.store(0, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
@@ -899,19 +887,6 @@ mod tests {
|
||||
assert_eq!(metrics.snapshot().signature_v1_fallback_total, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replay_scope_fallback_counter_updates_snapshot_and_resets() {
|
||||
let metrics = InternodeMetrics::default();
|
||||
assert_eq!(metrics.snapshot().replay_scope_fallback_total, 0);
|
||||
|
||||
metrics.record_replay_scope_fallback();
|
||||
metrics.record_replay_scope_fallback();
|
||||
assert_eq!(metrics.snapshot().replay_scope_fallback_total, 2);
|
||||
|
||||
metrics.reset_for_test();
|
||||
assert_eq!(metrics.snapshot().replay_scope_fallback_total, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cluster_peer_flips_offline_after_threshold_and_back_online() {
|
||||
// Unique addr keeps this independent of the process-global registry / other tests.
|
||||
|
||||
@@ -84,19 +84,20 @@ impl SwiftRouter {
|
||||
Self { enabled, url_prefix }
|
||||
}
|
||||
|
||||
/// Return whether a URI matches the Swift route shape without allocating
|
||||
/// decoded route components.
|
||||
pub fn matches(&self, uri: &Uri) -> bool {
|
||||
let Some(path) = self.route_path(uri) else {
|
||||
return false;
|
||||
};
|
||||
let mut segments = path.trim_start_matches('/').split('/');
|
||||
segments.next() == Some("v1") && segments.next().is_some_and(Self::is_valid_account)
|
||||
}
|
||||
|
||||
/// Parse a URI and return a SwiftRoute if it matches Swift URL pattern
|
||||
pub fn route(&self, uri: &Uri, method: Method) -> Option<SwiftRoute> {
|
||||
let path = self.route_path(uri)?;
|
||||
if !self.enabled {
|
||||
return None;
|
||||
}
|
||||
|
||||
let path = uri.path();
|
||||
|
||||
// Strip optional prefix
|
||||
let path = if let Some(prefix) = &self.url_prefix {
|
||||
path.strip_prefix(&format!("/{}/", prefix))?
|
||||
} else {
|
||||
path
|
||||
};
|
||||
|
||||
// Split path into segments - preserve empty segments to maintain object key fidelity
|
||||
// Swift allows trailing slashes and consecutive slashes in object names (e.g., "dir/" or "a//b")
|
||||
@@ -174,17 +175,6 @@ impl SwiftRouter {
|
||||
fn is_valid_account(account: &str) -> bool {
|
||||
ACCOUNT_PATTERN.is_match(account)
|
||||
}
|
||||
|
||||
fn route_path<'a>(&self, uri: &'a Uri) -> Option<&'a str> {
|
||||
if !self.enabled {
|
||||
return None;
|
||||
}
|
||||
let path = uri.path();
|
||||
let Some(prefix) = &self.url_prefix else {
|
||||
return Some(path);
|
||||
};
|
||||
path.strip_prefix('/')?.strip_prefix(prefix)?.strip_prefix('/')
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -291,42 +281,6 @@ mod tests {
|
||||
assert_eq!(route, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_matches_agrees_with_route_without_decoding_components() {
|
||||
let router = SwiftRouter::new(true, None);
|
||||
for path in [
|
||||
"/v1/AUTH_project",
|
||||
"/v1/AUTH_project/",
|
||||
"/v1/AUTH_project/container",
|
||||
"/v1/AUTH_project/container/a%20long/object",
|
||||
"//v1/AUTH_project/container/object",
|
||||
"/v1/not-a-swift-account/object",
|
||||
"/v1/AUTH_/object",
|
||||
"/bucket/object",
|
||||
] {
|
||||
let uri = path.parse().expect("Swift route test URI");
|
||||
assert_eq!(
|
||||
router.matches(&uri),
|
||||
router.route(&uri, Method::GET).is_some(),
|
||||
"classification must match full routing for {path}"
|
||||
);
|
||||
}
|
||||
|
||||
let prefixed_router = SwiftRouter::new(true, Some("swift".to_string()));
|
||||
for path in [
|
||||
"/swift/v1/AUTH_project/container",
|
||||
"/swiftish/v1/AUTH_project/container",
|
||||
"/v1/AUTH_project/container",
|
||||
] {
|
||||
let uri = path.parse().expect("prefixed Swift route test URI");
|
||||
assert_eq!(
|
||||
prefixed_router.matches(&uri),
|
||||
prefixed_router.route(&uri, Method::GET).is_some(),
|
||||
"prefixed classification must match full routing for {path}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_project_id_extraction() {
|
||||
let route = SwiftRoute::Account {
|
||||
|
||||
@@ -171,7 +171,6 @@ pub const HEAL_CONTROL_RPC_MAX_MESSAGE_SIZE: usize = heal_control::RESULT_MAX_SI
|
||||
pub const HEAL_CONTROL_PROTOCOL_VERSION: u32 = 2;
|
||||
pub const DYNAMIC_CONFIG_PROTOCOL_VERSION: u32 = 1;
|
||||
pub const HEAL_CONTROL_CAPABILITY_PROBE_PREFIX: &[u8] = b"rustfs-heal-control-capability-v2\0";
|
||||
pub const REMOTE_VERSION_STATE_CAPABILITY_PROBE_PREFIX: &[u8] = b"rustfs-tier-remote-version-state-capability-v1\0";
|
||||
pub const TIER_MUTATION_RPC_MAX_PREPARE_PAYLOAD_SIZE: usize = 64 * 1024;
|
||||
pub const TIER_MUTATION_RPC_MAX_COMMIT_PAYLOAD_SIZE: usize = 1024;
|
||||
pub const TIER_MUTATION_RPC_MAX_MESSAGE_SIZE: usize = TIER_MUTATION_RPC_MAX_PREPARE_PAYLOAD_SIZE + 4096;
|
||||
@@ -198,49 +197,6 @@ pub fn is_heal_control_capability_probe(command: &[u8]) -> bool {
|
||||
command.len() == HEAL_CONTROL_CAPABILITY_PROBE_PREFIX.len() + 16 && command.starts_with(HEAL_CONTROL_CAPABILITY_PROBE_PREFIX)
|
||||
}
|
||||
|
||||
pub fn remote_version_state_capability_probe(nonce: &[u8; 16]) -> Vec<u8> {
|
||||
let mut probe = Vec::with_capacity(REMOTE_VERSION_STATE_CAPABILITY_PROBE_PREFIX.len() + nonce.len());
|
||||
probe.extend_from_slice(REMOTE_VERSION_STATE_CAPABILITY_PROBE_PREFIX);
|
||||
probe.extend_from_slice(nonce);
|
||||
probe
|
||||
}
|
||||
|
||||
pub fn is_remote_version_state_capability_probe(command: &[u8]) -> bool {
|
||||
command.len() == REMOTE_VERSION_STATE_CAPABILITY_PROBE_PREFIX.len() + 16
|
||||
&& command.starts_with(REMOTE_VERSION_STATE_CAPABILITY_PROBE_PREFIX)
|
||||
}
|
||||
|
||||
pub fn encode_remote_version_state_capability(
|
||||
topology_member: &str,
|
||||
process_epoch: &[u8; 16],
|
||||
) -> Result<Vec<u8>, std::num::TryFromIntError> {
|
||||
let topology_member = topology_member.as_bytes();
|
||||
let mut result = Vec::with_capacity(8 + topology_member.len() + process_epoch.len());
|
||||
result.extend_from_slice(&u64::try_from(topology_member.len())?.to_be_bytes());
|
||||
result.extend_from_slice(topology_member);
|
||||
result.extend_from_slice(process_epoch);
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
pub fn decode_remote_version_state_capability(result: &[u8]) -> Result<(&str, &[u8; 16]), &'static str> {
|
||||
let member_len = result
|
||||
.get(..8)
|
||||
.and_then(|value| value.try_into().ok())
|
||||
.map(u64::from_be_bytes)
|
||||
.ok_or("remote version state capability is truncated")?;
|
||||
let member_len = usize::try_from(member_len).map_err(|_| "remote version state member length cannot be represented")?;
|
||||
let member_end = 8_usize
|
||||
.checked_add(member_len)
|
||||
.ok_or("remote version state member length overflow")?;
|
||||
let topology_member = std::str::from_utf8(result.get(8..member_end).ok_or("remote version state member is truncated")?)
|
||||
.map_err(|_| "remote version state member is not UTF-8")?;
|
||||
let process_epoch = result
|
||||
.get(member_end..)
|
||||
.and_then(|value| value.try_into().ok())
|
||||
.ok_or("remote version state process epoch has an invalid length")?;
|
||||
Ok((topology_member, process_epoch))
|
||||
}
|
||||
|
||||
/// Builds the stable byte representation authenticated for a heal-control request.
|
||||
///
|
||||
/// This deliberately does not reuse protobuf encoding: mixed-version peers may
|
||||
@@ -1700,12 +1656,10 @@ mod scanner_activity_tests {
|
||||
#[cfg(test)]
|
||||
mod heal_control_tests {
|
||||
use super::{
|
||||
HEAL_CONTROL_CAPABILITY_PROBE_PREFIX, HEAL_CONTROL_PROTOCOL_VERSION, REMOTE_VERSION_STATE_CAPABILITY_PROBE_PREFIX,
|
||||
canonical_heal_control_capability_ack, canonical_heal_control_request_body, canonical_heal_control_response_body,
|
||||
decode_remote_version_state_capability, encode_remote_version_state_capability, heal_control_capability_probe,
|
||||
HEAL_CONTROL_CAPABILITY_PROBE_PREFIX, HEAL_CONTROL_PROTOCOL_VERSION, canonical_heal_control_capability_ack,
|
||||
canonical_heal_control_request_body, canonical_heal_control_response_body, heal_control_capability_probe,
|
||||
heal_control_coordinator_epoch, heal_control_execution_timeout, heal_control_execution_timeout_for,
|
||||
internode_rpc_timeout, is_heal_control_capability_probe, is_remote_version_state_capability_probe,
|
||||
normalize_internode_rpc_timeout, remote_version_state_capability_probe,
|
||||
internode_rpc_timeout, is_heal_control_capability_probe, normalize_internode_rpc_timeout,
|
||||
};
|
||||
use crate::heal_control;
|
||||
use std::time::Duration;
|
||||
@@ -1762,29 +1716,6 @@ mod heal_control_tests {
|
||||
assert!(!is_heal_control_capability_probe(HEAL_CONTROL_CAPABILITY_PROBE_PREFIX));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_version_state_capability_probe_requires_exact_nonce() {
|
||||
let probe = remote_version_state_capability_probe(&[7; 16]);
|
||||
assert!(is_remote_version_state_capability_probe(&probe));
|
||||
assert!(!is_remote_version_state_capability_probe(REMOTE_VERSION_STATE_CAPABILITY_PROBE_PREFIX));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_version_state_capability_binds_member_and_process_epoch() {
|
||||
let encoded =
|
||||
encode_remote_version_state_capability("node-a:9000", &[7; 16]).expect("small capability response should encode");
|
||||
assert_eq!(
|
||||
decode_remote_version_state_capability(&encoded).expect("capability response should decode"),
|
||||
("node-a:9000", &[7; 16])
|
||||
);
|
||||
assert!(decode_remote_version_state_capability(&encoded[..encoded.len() - 1]).is_err());
|
||||
|
||||
let mut invalid_utf8 =
|
||||
encode_remote_version_state_capability("node-a", &[7; 16]).expect("small capability response should encode");
|
||||
invalid_utf8[8] = 0xff;
|
||||
assert!(decode_remote_version_state_capability(&invalid_utf8).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn canonical_response_binds_request_and_result() {
|
||||
let baseline = canonical_heal_control_response_body(2, "abcdef", b"query", b"result").unwrap();
|
||||
|
||||
@@ -18,9 +18,7 @@ use crate::rule::ReplicationRuleExt as _;
|
||||
use s3s::dto::DeleteMarkerReplicationStatus;
|
||||
use s3s::dto::DeleteReplicationStatus;
|
||||
use s3s::dto::Destination;
|
||||
use s3s::dto::{
|
||||
ExistingObjectReplicationStatus, ReplicationConfiguration, ReplicationRule, ReplicationRuleStatus, ReplicationRules,
|
||||
};
|
||||
use s3s::dto::{ExistingObjectReplicationStatus, ReplicationConfiguration, ReplicationRuleStatus, ReplicationRules};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashSet;
|
||||
use uuid::Uuid;
|
||||
@@ -47,86 +45,6 @@ pub trait ReplicationConfigurationExt {
|
||||
fn filter_target_arns(&self, obj: &ObjectOpts) -> Vec<String>;
|
||||
}
|
||||
|
||||
pub fn delete_replication_target_arns(config: &ReplicationConfiguration, object_name: &str, replica: bool) -> HashSet<String> {
|
||||
let role = config.role.trim();
|
||||
if !role.is_empty() && active_replication_rule_destination_arns(config).len() > 1 {
|
||||
return HashSet::new();
|
||||
}
|
||||
|
||||
let mut targets = HashSet::new();
|
||||
let mut targets_with_unknown_tags = HashSet::new();
|
||||
for rule in &config.rules {
|
||||
if rule.status == ReplicationRuleStatus::from_static(ReplicationRuleStatus::DISABLED)
|
||||
|| !object_name.starts_with(rule.prefix())
|
||||
{
|
||||
continue;
|
||||
}
|
||||
let arn = if role.is_empty() {
|
||||
rule.destination.bucket.trim()
|
||||
} else {
|
||||
role
|
||||
};
|
||||
if arn.is_empty() {
|
||||
continue;
|
||||
}
|
||||
targets.insert(arn.to_string());
|
||||
if rule.filter.as_ref().is_some_and(|filter| {
|
||||
filter.tag.is_some()
|
||||
|| filter
|
||||
.and
|
||||
.as_ref()
|
||||
.and_then(|and| and.tags.as_ref())
|
||||
.is_some_and(|tags| !tags.is_empty())
|
||||
}) {
|
||||
targets_with_unknown_tags.insert(arn.to_string());
|
||||
}
|
||||
}
|
||||
|
||||
targets
|
||||
.into_iter()
|
||||
.filter(|arn| !targets_with_unknown_tags.contains(arn))
|
||||
.filter(|arn| {
|
||||
config.replicate(&ObjectOpts {
|
||||
name: object_name.to_string(),
|
||||
target_arn: arn.clone(),
|
||||
version_id: Some(Uuid::nil()),
|
||||
delete_marker: true,
|
||||
op_type: ReplicationType::Delete,
|
||||
replica,
|
||||
..Default::default()
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn rule_replicates(rule: &ReplicationRule, obj: &ObjectOpts) -> bool {
|
||||
if let Some(status) = &rule.existing_object_replication
|
||||
&& obj.existing_object
|
||||
&& status.status == ExistingObjectReplicationStatus::from_static(ExistingObjectReplicationStatus::DISABLED)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if obj.op_type == ReplicationType::Delete {
|
||||
if !rule.metadata_replicate(obj) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if obj.version_id.is_some() {
|
||||
return rule
|
||||
.delete_replication
|
||||
.clone()
|
||||
.is_some_and(|d| d.status == DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED));
|
||||
}
|
||||
|
||||
return rule.delete_marker_replication.clone().is_some_and(|d| {
|
||||
d.status == Some(DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::ENABLED))
|
||||
});
|
||||
}
|
||||
|
||||
rule.metadata_replicate(obj)
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum ReplicationTargetValidationError {
|
||||
RoleWithMultipleDestinations,
|
||||
@@ -286,7 +204,37 @@ impl ReplicationConfigurationExt for ReplicationConfiguration {
|
||||
continue;
|
||||
}
|
||||
|
||||
return rule_replicates(rule, obj);
|
||||
if let Some(status) = &rule.existing_object_replication
|
||||
&& obj.existing_object
|
||||
&& status.status == ExistingObjectReplicationStatus::from_static(ExistingObjectReplicationStatus::DISABLED)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if obj.op_type == ReplicationType::Delete {
|
||||
if !rule.metadata_replicate(obj) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if obj.version_id.is_some() {
|
||||
if obj.delete_marker {
|
||||
return rule.delete_marker_replication.clone().is_some_and(|d| {
|
||||
d.status == Some(DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::ENABLED))
|
||||
});
|
||||
}
|
||||
return rule
|
||||
.delete_replication
|
||||
.clone()
|
||||
.is_some_and(|d| d.status == DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED));
|
||||
} else {
|
||||
return rule.delete_marker_replication.clone().is_some_and(|d| {
|
||||
d.status == Some(DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::ENABLED))
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Regular object/metadata replication
|
||||
return rule.metadata_replicate(obj);
|
||||
}
|
||||
false
|
||||
}
|
||||
@@ -357,10 +305,7 @@ impl ReplicationConfigurationExt for ReplicationConfiguration {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use s3s::dto::{
|
||||
DeleteMarkerReplication, DeleteReplication, Destination, ExistingObjectReplication, ReplicationRule,
|
||||
ReplicationRuleFilter, Tag,
|
||||
};
|
||||
use s3s::dto::{DeleteMarkerReplication, Destination, ExistingObjectReplication, ReplicationRule};
|
||||
|
||||
fn replication_rule(id: &str, arn: &str) -> ReplicationRule {
|
||||
ReplicationRule {
|
||||
@@ -382,37 +327,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_replication_target_arns_uses_highest_priority_matching_rule() {
|
||||
let arn = "arn:target:a";
|
||||
let mut lower_priority = replication_rule("lower", arn);
|
||||
lower_priority.priority = Some(1);
|
||||
lower_priority.delete_replication = Some(DeleteReplication {
|
||||
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED),
|
||||
});
|
||||
let mut higher_priority = replication_rule("higher", arn);
|
||||
higher_priority.priority = Some(2);
|
||||
higher_priority.delete_replication = Some(DeleteReplication {
|
||||
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::DISABLED),
|
||||
});
|
||||
let mut config = ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![lower_priority, higher_priority],
|
||||
};
|
||||
|
||||
let targets = delete_replication_target_arns(&config, "object", false);
|
||||
|
||||
assert!(targets.is_empty(), "the higher-priority disabled rule must suppress the target");
|
||||
|
||||
config.rules[0].delete_replication = Some(DeleteReplication {
|
||||
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::DISABLED),
|
||||
});
|
||||
config.rules[1].delete_replication = Some(DeleteReplication {
|
||||
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED),
|
||||
});
|
||||
assert_eq!(delete_replication_target_arns(&config, "object", false), HashSet::from([arn.to_string()]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn filter_target_arns_uses_role_when_role_is_present() {
|
||||
let config = ReplicationConfiguration {
|
||||
@@ -423,151 +337,15 @@ mod tests {
|
||||
],
|
||||
};
|
||||
|
||||
let opts = ObjectOpts {
|
||||
let arns = config.filter_target_arns(&ObjectOpts {
|
||||
name: "object".to_string(),
|
||||
op_type: ReplicationType::Object,
|
||||
..Default::default()
|
||||
};
|
||||
let arns = config.filter_target_arns(&opts);
|
||||
});
|
||||
|
||||
assert_eq!(arns, vec!["arn:legacy:target".to_string()]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_replication_target_arns_uses_role_when_role_is_present() {
|
||||
let mut rule = replication_rule("rule", "arn:target:a");
|
||||
rule.delete_replication = Some(DeleteReplication {
|
||||
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED),
|
||||
});
|
||||
let config = ReplicationConfiguration {
|
||||
role: " arn:legacy:target ".to_string(),
|
||||
rules: vec![rule],
|
||||
};
|
||||
|
||||
assert_eq!(
|
||||
delete_replication_target_arns(&config, "object", false),
|
||||
HashSet::from(["arn:legacy:target".to_string()])
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_replication_target_arns_ignores_disjoint_prefix_rules() {
|
||||
let arn = "arn:target:a";
|
||||
let mut matching = replication_rule("matching", arn);
|
||||
matching.prefix = None;
|
||||
matching.filter = Some(ReplicationRuleFilter {
|
||||
prefix: Some("logs/".to_string()),
|
||||
..Default::default()
|
||||
});
|
||||
matching.delete_replication = Some(DeleteReplication {
|
||||
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::DISABLED),
|
||||
});
|
||||
let mut unrelated = replication_rule("unrelated", arn);
|
||||
unrelated.prefix = None;
|
||||
unrelated.filter = Some(ReplicationRuleFilter {
|
||||
prefix: Some("archive/".to_string()),
|
||||
..Default::default()
|
||||
});
|
||||
unrelated.priority = Some(2);
|
||||
unrelated.delete_replication = Some(DeleteReplication {
|
||||
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED),
|
||||
});
|
||||
let mut config = ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![matching, unrelated],
|
||||
};
|
||||
|
||||
assert!(delete_replication_target_arns(&config, "logs/object", false).is_empty());
|
||||
|
||||
config.rules[0].delete_replication = Some(DeleteReplication {
|
||||
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED),
|
||||
});
|
||||
config.rules[1].delete_replication = Some(DeleteReplication {
|
||||
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::DISABLED),
|
||||
});
|
||||
assert_eq!(
|
||||
delete_replication_target_arns(&config, "logs/object", false),
|
||||
HashSet::from([arn.to_string()])
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_replication_target_arns_fails_closed_for_unknown_tag_rules() {
|
||||
let arn = "arn:target:a";
|
||||
let mut known = replication_rule("known", arn);
|
||||
known.priority = Some(2);
|
||||
known.delete_replication = Some(DeleteReplication {
|
||||
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED),
|
||||
});
|
||||
let mut unknown = replication_rule("unknown", arn);
|
||||
unknown.priority = Some(1);
|
||||
unknown.prefix = None;
|
||||
unknown.filter = Some(ReplicationRuleFilter {
|
||||
tag: Some(Tag {
|
||||
key: Some("env".to_string()),
|
||||
value: Some("prod".to_string()),
|
||||
}),
|
||||
..Default::default()
|
||||
});
|
||||
unknown.delete_replication = Some(DeleteReplication {
|
||||
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED),
|
||||
});
|
||||
let config = ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![known, unknown],
|
||||
};
|
||||
|
||||
assert!(delete_replication_target_arns(&config, "object", false).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_replication_target_arns_reuses_full_destination_rule_order() {
|
||||
let arn = "arn:target:a";
|
||||
let mut first = replication_rule("first", arn);
|
||||
first.priority = Some(1);
|
||||
first.destination.account = Some("account-a".to_string());
|
||||
first.delete_replication = Some(DeleteReplication {
|
||||
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::DISABLED),
|
||||
});
|
||||
let mut second = replication_rule("second", arn);
|
||||
second.priority = Some(2);
|
||||
second.destination.account = Some("account-b".to_string());
|
||||
second.delete_replication = Some(DeleteReplication {
|
||||
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED),
|
||||
});
|
||||
let config = ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![first, second],
|
||||
};
|
||||
let opts = ObjectOpts {
|
||||
name: "object".to_string(),
|
||||
target_arn: arn.to_string(),
|
||||
version_id: Some(Uuid::new_v4()),
|
||||
delete_marker: true,
|
||||
op_type: ReplicationType::Delete,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert!(!config.replicate(&opts));
|
||||
assert!(delete_replication_target_arns(&config, "object", false).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_replication_target_arns_rejects_role_with_multiple_destinations() {
|
||||
let mut first = replication_rule("first", "arn:target:a");
|
||||
first.delete_replication = Some(DeleteReplication {
|
||||
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED),
|
||||
});
|
||||
let mut second = replication_rule("second", "arn:target:b");
|
||||
second.delete_replication = first.delete_replication.clone();
|
||||
let config = ReplicationConfiguration {
|
||||
role: "arn:legacy:target".to_string(),
|
||||
rules: vec![first, second],
|
||||
};
|
||||
|
||||
assert!(delete_replication_target_arns(&config, "object", false).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn filter_target_arns_falls_back_to_role_when_destination_is_empty() {
|
||||
let config = ReplicationConfiguration {
|
||||
@@ -827,42 +605,4 @@ mod tests {
|
||||
"highest-priority rule disables delete-marker replication, so the delete marker must not replicate"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_marker_version_purge_requires_delete_replication() {
|
||||
let arn = "arn:rustfs:replication:us-east-1:target:bucket";
|
||||
let mut config = ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![delete_marker_rule("delete-markers-only", arn, "", 1, true)],
|
||||
};
|
||||
let opts = ObjectOpts {
|
||||
name: "object.txt".to_string(),
|
||||
op_type: ReplicationType::Delete,
|
||||
delete_marker: true,
|
||||
version_id: Some(Uuid::new_v4()),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert!(
|
||||
!config.replicate(&opts),
|
||||
"permanently deleting a delete-marker version must not use the delete-marker replication setting"
|
||||
);
|
||||
|
||||
config.rules[0].delete_replication = Some(DeleteReplication {
|
||||
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::DISABLED),
|
||||
});
|
||||
assert!(
|
||||
!config.replicate(&opts),
|
||||
"an explicitly disabled permanent-delete setting must not replicate"
|
||||
);
|
||||
|
||||
config.rules[0].delete_replication = Some(DeleteReplication {
|
||||
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED),
|
||||
});
|
||||
|
||||
assert!(
|
||||
config.replicate(&opts),
|
||||
"permanently deleting a delete-marker version should replicate when delete replication is enabled"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,7 +15,7 @@
|
||||
use std::any::Any;
|
||||
|
||||
use crate::storage_api::DeletedObject;
|
||||
use crate::{MrfOpKind, MrfReplicateEntry, ReplicationStatusType, ReplicationType, ReplicationWorkerOperation};
|
||||
use crate::{MrfOpKind, MrfReplicateEntry, ReplicationType, ReplicationWorkerOperation};
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct DeletedObjectReplicationInfo {
|
||||
@@ -42,11 +42,6 @@ impl ReplicationWorkerOperation for DeletedObjectReplicationInfo {
|
||||
op: MrfOpKind::Delete,
|
||||
delete_marker_version_id: self.delete_object.delete_marker_version_id,
|
||||
delete_marker: self.delete_object.delete_marker,
|
||||
replica: self
|
||||
.delete_object
|
||||
.replication_state
|
||||
.as_ref()
|
||||
.is_some_and(|state| state.replica_status == ReplicationStatusType::Replica),
|
||||
// Persist the original delete-marker mtime as Unix nanoseconds so replay after a
|
||||
// restart stamps the replica with the source timestamp rather than the replay time
|
||||
// (backlog#867). None when unknown; replay then falls back to the current time.
|
||||
@@ -90,22 +85,14 @@ pub fn is_retryable_delete_replication_head_error(is_not_found: bool, code: Opti
|
||||
!(is_not_found || matches!(code, Some("MethodNotAllowed" | "405")))
|
||||
}
|
||||
|
||||
pub fn version_purge_target_missing(is_not_found: bool, code: Option<&str>, raw_status: Option<u16>) -> bool {
|
||||
if matches!(code, Some("MethodNotAllowed" | "405")) || raw_status == Some(405) {
|
||||
return false;
|
||||
}
|
||||
|
||||
is_not_found || matches!(code, Some("NoSuchVersion" | "NoSuchKey" | "NotFound" | "404")) || raw_status == Some(404)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{
|
||||
DeletedObjectReplicationInfo, is_retryable_delete_replication_head_error, is_version_delete_replication,
|
||||
should_retry_delete_marker_purge, version_purge_target_missing,
|
||||
should_retry_delete_marker_purge,
|
||||
};
|
||||
use crate::storage_api::DeletedObject;
|
||||
use crate::{MrfOpKind, ReplicationState, ReplicationStatusType, ReplicationType, ReplicationWorkerOperation};
|
||||
use crate::{MrfOpKind, ReplicationType, ReplicationWorkerOperation};
|
||||
use uuid::Uuid;
|
||||
|
||||
#[test]
|
||||
@@ -122,10 +109,6 @@ mod tests {
|
||||
delete_marker_version_id: Some(delete_marker_version_id),
|
||||
delete_marker: true,
|
||||
delete_marker_mtime: Some(mtime),
|
||||
replication_state: Some(ReplicationState {
|
||||
replica_status: ReplicationStatusType::Replica,
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
@@ -139,7 +122,6 @@ mod tests {
|
||||
assert_eq!(entry.delete_marker_version_id, Some(delete_marker_version_id));
|
||||
assert_eq!(entry.op, MrfOpKind::Delete);
|
||||
assert!(entry.delete_marker);
|
||||
assert!(entry.replica);
|
||||
// The original mtime must be persisted (as Unix nanos) so replay keeps the source
|
||||
// timestamp instead of stamping the replica with the replay time (backlog#867).
|
||||
assert_eq!(
|
||||
@@ -230,14 +212,4 @@ mod tests {
|
||||
assert!(!is_retryable_delete_replication_head_error(true, Some("NoSuchKey")));
|
||||
assert!(is_retryable_delete_replication_head_error(false, Some("AccessDenied")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn version_purge_target_missing_requires_not_found() {
|
||||
assert!(version_purge_target_missing(true, Some("NoSuchVersion"), Some(404)));
|
||||
assert!(version_purge_target_missing(false, Some("NoSuchVersion"), None));
|
||||
assert!(version_purge_target_missing(false, None, Some(404)));
|
||||
assert!(!version_purge_target_missing(false, None, None));
|
||||
assert!(!version_purge_target_missing(true, Some("MethodNotAllowed"), Some(405)));
|
||||
assert!(!version_purge_target_missing(true, Some("405"), None));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -587,11 +587,6 @@ pub struct MrfReplicateEntry {
|
||||
#[serde(rename = "deleteMarker", default)]
|
||||
pub delete_marker: bool,
|
||||
|
||||
// For delete entries: whether the operation originated from a replica.
|
||||
// Old files lack this field and therefore default to a local-source delete.
|
||||
#[serde(rename = "replica", default)]
|
||||
pub replica: bool,
|
||||
|
||||
// For delete entries: the original delete-marker mtime, persisted as Unix nanoseconds so
|
||||
// replay stamps replicas with the source timestamp instead of the replay time. Old files
|
||||
// lack this key; default=None means "unknown", and replay falls back to the current time
|
||||
@@ -803,7 +798,6 @@ impl ReplicationWorkerOperation for ReplicateObjectInfo {
|
||||
op: MrfOpKind::Object,
|
||||
delete_marker_version_id: None,
|
||||
delete_marker: false,
|
||||
replica: false,
|
||||
delete_marker_mtime: None,
|
||||
}
|
||||
}
|
||||
@@ -858,7 +852,6 @@ impl ReplicateObjectInfo {
|
||||
op: MrfOpKind::Object,
|
||||
delete_marker_version_id: None,
|
||||
delete_marker: false,
|
||||
replica: false,
|
||||
delete_marker_mtime: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -30,12 +30,11 @@ pub mod tagging;
|
||||
|
||||
pub use config::{
|
||||
ObjectOpts, ReplicationConfigurationExt, ReplicationTargetValidationError, active_replication_rule_destination_arns,
|
||||
delete_replication_target_arns, replication_target_arns, should_remove_replication_target,
|
||||
validate_replication_config_target_arns,
|
||||
replication_target_arns, should_remove_replication_target, validate_replication_config_target_arns,
|
||||
};
|
||||
pub use delete::{
|
||||
DeletedObjectReplicationInfo, is_retryable_delete_replication_head_error, is_version_delete_replication,
|
||||
should_retry_delete_marker_purge, version_purge_target_missing,
|
||||
should_retry_delete_marker_purge,
|
||||
};
|
||||
pub use filemeta::{
|
||||
REPLICATE_EXISTING, REPLICATE_EXISTING_DELETE, REPLICATE_HEAL, REPLICATE_HEAL_DELETE, REPLICATE_INCOMING,
|
||||
@@ -55,11 +54,10 @@ pub use object::{
|
||||
replication_etags_match, target_is_newer_than_source_null_version,
|
||||
};
|
||||
pub use operation::{
|
||||
MustReplicateOptions, ReplicationDeleteParts, ReplicationDeleteScheduleInput, ReplicationDeleteSource,
|
||||
ReplicationDeleteStateSource, ReplicationResyncTargetObject, delete_replication_missing_source_decision,
|
||||
delete_replication_object_opts, delete_replication_parts, delete_replication_state_from_config,
|
||||
delete_replication_version_id, heal_uses_delete_replication_path, is_ssec_encrypted, resync_target_for_object,
|
||||
should_schedule_delete_replication, should_use_existing_delete_replication_info,
|
||||
MustReplicateOptions, ReplicationDeleteScheduleInput, ReplicationDeleteSource, ReplicationDeleteStateSource,
|
||||
ReplicationResyncTargetObject, delete_replication_missing_source_decision, delete_replication_object_opts,
|
||||
delete_replication_state_from_config, delete_replication_version_id, heal_uses_delete_replication_path, is_ssec_encrypted,
|
||||
resync_target_for_object, should_schedule_delete_replication, should_use_existing_delete_replication_info,
|
||||
should_use_existing_delete_replication_source,
|
||||
};
|
||||
pub use queue::{
|
||||
|
||||
@@ -13,7 +13,6 @@
|
||||
// limitations under the License.
|
||||
|
||||
use byteorder::{ByteOrder, LittleEndian};
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::{Error, Result};
|
||||
|
||||
@@ -22,31 +21,6 @@ pub use crate::filemeta::{MrfOpKind, MrfReplicateEntry};
|
||||
pub const MRF_META_FORMAT: u16 = 1;
|
||||
pub const MRF_META_VERSION: u16 = 1;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct MrfDeleteParts {
|
||||
pub version_id: Option<Uuid>,
|
||||
pub delete_marker_version_id: Option<Uuid>,
|
||||
pub delete_marker: bool,
|
||||
}
|
||||
|
||||
impl MrfReplicateEntry {
|
||||
pub fn delete_parts_for_replay(&self) -> Option<MrfDeleteParts> {
|
||||
match (self.version_id, self.delete_marker_version_id) {
|
||||
(Some(version_id), None) => Some(MrfDeleteParts {
|
||||
version_id: Some(version_id),
|
||||
delete_marker_version_id: None,
|
||||
delete_marker: false,
|
||||
}),
|
||||
(None, Some(delete_marker_version_id)) => Some(MrfDeleteParts {
|
||||
version_id: None,
|
||||
delete_marker_version_id: Some(delete_marker_version_id),
|
||||
delete_marker: self.delete_marker,
|
||||
}),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn encode_mrf_file(entries: &[MrfReplicateEntry]) -> Result<Vec<u8>> {
|
||||
let payload = rmp_serde::to_vec_named(entries).map_err(|e| Error::Other(e.to_string()))?;
|
||||
let mut data = Vec::with_capacity(4 + payload.len());
|
||||
@@ -80,6 +54,7 @@ pub fn decode_mrf_file(data: &[u8]) -> Result<Vec<MrfReplicateEntry>> {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use uuid::Uuid;
|
||||
|
||||
#[test]
|
||||
fn mrf_file_round_trips_object_and_delete_entries() {
|
||||
@@ -95,7 +70,6 @@ mod tests {
|
||||
op: MrfOpKind::Object,
|
||||
delete_marker_version_id: None,
|
||||
delete_marker: false,
|
||||
replica: false,
|
||||
delete_marker_mtime: None,
|
||||
},
|
||||
MrfReplicateEntry {
|
||||
@@ -107,7 +81,6 @@ mod tests {
|
||||
op: MrfOpKind::Delete,
|
||||
delete_marker_version_id: Some(del_vid),
|
||||
delete_marker: true,
|
||||
replica: true,
|
||||
delete_marker_mtime: Some(1_705_312_200_123_456_789),
|
||||
},
|
||||
];
|
||||
@@ -122,7 +95,6 @@ mod tests {
|
||||
assert_eq!(decoded[1].delete_marker_version_id, Some(del_vid));
|
||||
assert_eq!(decoded[1].op, MrfOpKind::Delete);
|
||||
assert!(decoded[1].delete_marker);
|
||||
assert!(decoded[1].replica);
|
||||
assert_eq!(
|
||||
decoded[1].delete_marker_mtime,
|
||||
Some(1_705_312_200_123_456_789),
|
||||
@@ -130,46 +102,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn legacy_version_purge_replay_clears_delete_marker_creation() {
|
||||
let entry = MrfReplicateEntry {
|
||||
bucket: "bucket".to_string(),
|
||||
object: "object".to_string(),
|
||||
version_id: Some(Uuid::new_v4()),
|
||||
retry_count: 0,
|
||||
size: 0,
|
||||
delete_marker: true,
|
||||
op: MrfOpKind::Delete,
|
||||
delete_marker_version_id: None,
|
||||
replica: false,
|
||||
delete_marker_mtime: None,
|
||||
};
|
||||
let encoded = encode_mrf_file(&[entry]).expect("legacy MRF entry should encode");
|
||||
let decoded = decode_mrf_file(&encoded).expect("legacy MRF entry should decode");
|
||||
|
||||
assert_eq!(decoded[0].delete_parts_for_replay().map(|parts| parts.delete_marker), Some(false));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_delete_id_shapes_fail_closed_on_replay() {
|
||||
for (version_id, delete_marker_version_id) in [(None, None), (Some(Uuid::new_v4()), Some(Uuid::new_v4()))] {
|
||||
let entry = MrfReplicateEntry {
|
||||
bucket: "bucket".to_string(),
|
||||
object: "object".to_string(),
|
||||
version_id,
|
||||
retry_count: 0,
|
||||
size: 0,
|
||||
op: MrfOpKind::Delete,
|
||||
delete_marker_version_id,
|
||||
delete_marker: false,
|
||||
replica: false,
|
||||
delete_marker_mtime: None,
|
||||
};
|
||||
|
||||
assert_eq!(entry.delete_parts_for_replay(), None);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mrf_legacy_file_without_op_decodes_as_object() {
|
||||
let mut payload = Vec::new();
|
||||
@@ -197,7 +129,6 @@ mod tests {
|
||||
assert_eq!(decoded[0].retry_count, 2);
|
||||
assert_eq!(decoded[0].size, 100);
|
||||
assert_eq!(decoded[0].op, MrfOpKind::Object);
|
||||
assert!(!decoded[0].replica);
|
||||
// Old files lack the deleteMarkerMtime key; it must default to None so replay keeps the
|
||||
// pre-#867 fallback to the current time.
|
||||
assert_eq!(decoded[0].delete_marker_mtime, None);
|
||||
|
||||
@@ -151,11 +151,6 @@ pub fn delete_replication_state_from_config(
|
||||
|
||||
let pending_status = decision.pending_status();
|
||||
let mut state = ReplicationState {
|
||||
replica_status: if source.replica {
|
||||
ReplicationStatusType::Replica
|
||||
} else {
|
||||
ReplicationStatusType::Empty
|
||||
},
|
||||
replicate_decision_str: decision.to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
@@ -179,31 +174,11 @@ pub struct ReplicationDeleteScheduleInput<'a> {
|
||||
pub deleted_delete_marker_version: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct ReplicationDeleteParts {
|
||||
pub delete_marker: bool,
|
||||
pub version_id: Option<Uuid>,
|
||||
pub delete_marker_version_id: Option<Uuid>,
|
||||
}
|
||||
|
||||
pub fn delete_replication_parts(
|
||||
source_delete_marker: bool,
|
||||
source_version_id: Option<Uuid>,
|
||||
version_purge: bool,
|
||||
) -> Option<ReplicationDeleteParts> {
|
||||
if version_purge {
|
||||
return source_version_id.map(|version_id| ReplicationDeleteParts {
|
||||
delete_marker: false,
|
||||
version_id: Some(version_id),
|
||||
delete_marker_version_id: None,
|
||||
});
|
||||
}
|
||||
|
||||
Some(ReplicationDeleteParts {
|
||||
delete_marker: source_delete_marker,
|
||||
version_id: None,
|
||||
delete_marker_version_id: source_version_id,
|
||||
})
|
||||
fn delete_version_purge_source_status(status: &ReplicationStatusType) -> bool {
|
||||
status == &ReplicationStatusType::Replica
|
||||
|| status == &ReplicationStatusType::Pending
|
||||
|| status == &ReplicationStatusType::Completed
|
||||
|| status == &ReplicationStatusType::Failed
|
||||
}
|
||||
|
||||
pub fn should_schedule_delete_replication(input: ReplicationDeleteScheduleInput<'_>) -> bool {
|
||||
@@ -211,11 +186,14 @@ pub fn should_schedule_delete_replication(input: ReplicationDeleteScheduleInput<
|
||||
return false;
|
||||
}
|
||||
|
||||
if input.version_id_requested {
|
||||
return input.source_version_purge_status == &VersionPurgeStatusType::Pending;
|
||||
if input.version_id_requested && !input.deleted_delete_marker_version && !input.source_delete_marker {
|
||||
return delete_version_purge_source_status(input.source_replication_status);
|
||||
}
|
||||
|
||||
input.source_replication_status == &ReplicationStatusType::Pending
|
||||
input.source_replication_status == &ReplicationStatusType::Replica
|
||||
|| input.source_replication_status == &ReplicationStatusType::Pending
|
||||
|| input.source_version_purge_status == &VersionPurgeStatusType::Pending
|
||||
|| (input.deleted_delete_marker_version && input.source_replication_status == &ReplicationStatusType::Completed)
|
||||
}
|
||||
|
||||
pub fn delete_replication_version_id(
|
||||
@@ -334,20 +312,19 @@ pub fn resync_target_for_object(
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{
|
||||
MustReplicateOptions, ReplicationDeleteParts, ReplicationDeleteScheduleInput, ReplicationDeleteSource,
|
||||
ReplicationDeleteStateSource, ReplicationResyncTargetObject, delete_replication_missing_source_decision,
|
||||
delete_replication_object_opts, delete_replication_parts, delete_replication_state_from_config,
|
||||
delete_replication_version_id, heal_uses_delete_replication_path, is_ssec_encrypted, resync_target_for_object,
|
||||
should_schedule_delete_replication, should_use_existing_delete_replication_info,
|
||||
should_use_existing_delete_replication_source,
|
||||
MustReplicateOptions, ReplicationDeleteScheduleInput, ReplicationDeleteSource, ReplicationDeleteStateSource,
|
||||
ReplicationResyncTargetObject, delete_replication_missing_source_decision, delete_replication_object_opts,
|
||||
delete_replication_state_from_config, delete_replication_version_id, heal_uses_delete_replication_path,
|
||||
is_ssec_encrypted, resync_target_for_object, should_schedule_delete_replication,
|
||||
should_use_existing_delete_replication_info, should_use_existing_delete_replication_source,
|
||||
};
|
||||
use crate::http::{AMZ_BUCKET_REPLICATION_STATUS, SSEC_ALGORITHM_HEADER};
|
||||
use crate::storage_api::ObjectToDelete;
|
||||
use crate::{ReplicationStatusType, ReplicationType, VersionPurgeStatusType, target_reset_header};
|
||||
use s3s::dto::{
|
||||
DeleteMarkerReplication, DeleteMarkerReplicationStatus, DeleteReplication, DeleteReplicationStatus, Destination,
|
||||
ExistingObjectReplication, ExistingObjectReplicationStatus, ReplicaModifications, ReplicaModificationsStatus,
|
||||
ReplicationConfiguration, ReplicationRule, ReplicationRuleStatus, SourceSelectionCriteria,
|
||||
DeleteMarkerReplication, DeleteMarkerReplicationStatus, Destination, ExistingObjectReplication,
|
||||
ExistingObjectReplicationStatus, ReplicaModifications, ReplicaModificationsStatus, ReplicationConfiguration,
|
||||
ReplicationRule, ReplicationRuleStatus, SourceSelectionCriteria,
|
||||
};
|
||||
use std::collections::HashMap;
|
||||
use time::{Duration, OffsetDateTime};
|
||||
@@ -497,7 +474,6 @@ mod tests {
|
||||
.expect("replica delete marker should be forwarded to downstream targets");
|
||||
let pending = format!("{arn}=PENDING;");
|
||||
|
||||
assert_eq!(state.replica_status, ReplicationStatusType::Replica);
|
||||
assert_eq!(state.replication_status_internal.as_deref(), Some(pending.as_str()));
|
||||
assert_eq!(state.replicate_decision_str, format!("{arn}=true;false;{arn};"));
|
||||
assert!(state.targets.contains_key(arn));
|
||||
@@ -524,13 +500,9 @@ mod tests {
|
||||
#[test]
|
||||
fn delete_replication_state_tracks_delete_marker_version_purges() {
|
||||
let arn = "arn:aws:s3:::target-bucket";
|
||||
let mut rule = delete_replication_rule(arn, false);
|
||||
rule.delete_replication = Some(DeleteReplication {
|
||||
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED),
|
||||
});
|
||||
let config = ReplicationConfiguration {
|
||||
role: arn.to_string(),
|
||||
rules: vec![rule],
|
||||
rules: vec![delete_replication_rule(arn, false)],
|
||||
};
|
||||
let source = ReplicationDeleteStateSource {
|
||||
name: "test/object.txt".to_string(),
|
||||
@@ -541,10 +513,9 @@ mod tests {
|
||||
};
|
||||
|
||||
let state = delete_replication_state_from_config(&config, &source)
|
||||
.expect("delete-marker version purge should honor delete replication rules");
|
||||
.expect("delete-marker version purge should honor delete-marker replication rules");
|
||||
let pending = format!("{arn}=PENDING;");
|
||||
|
||||
assert_eq!(state.replica_status, ReplicationStatusType::Empty);
|
||||
assert_eq!(state.version_purge_status_internal.as_deref(), Some(pending.as_str()));
|
||||
assert_eq!(state.replicate_decision_str, format!("{arn}=true;false;{arn};"));
|
||||
assert!(state.purge_targets.contains_key(arn));
|
||||
@@ -563,13 +534,13 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_replication_schedule_uses_pending_state_from_current_delete_rules() {
|
||||
fn delete_replication_schedule_keeps_marker_and_version_purges() {
|
||||
assert!(should_schedule_delete_replication(ReplicationDeleteScheduleInput {
|
||||
replication_request: false,
|
||||
version_id_requested: true,
|
||||
source_delete_marker: true,
|
||||
source_replication_status: &ReplicationStatusType::Empty,
|
||||
source_version_purge_status: &VersionPurgeStatusType::Pending,
|
||||
source_replication_status: &ReplicationStatusType::Completed,
|
||||
source_version_purge_status: &VersionPurgeStatusType::Empty,
|
||||
deleted_delete_marker_version: true,
|
||||
}));
|
||||
assert!(should_schedule_delete_replication(ReplicationDeleteScheduleInput {
|
||||
@@ -577,54 +548,19 @@ mod tests {
|
||||
version_id_requested: true,
|
||||
source_delete_marker: false,
|
||||
source_replication_status: &ReplicationStatusType::Completed,
|
||||
source_version_purge_status: &VersionPurgeStatusType::Pending,
|
||||
source_version_purge_status: &VersionPurgeStatusType::Empty,
|
||||
deleted_delete_marker_version: false,
|
||||
}));
|
||||
assert!(should_schedule_delete_replication(ReplicationDeleteScheduleInput {
|
||||
replication_request: false,
|
||||
version_id_requested: false,
|
||||
source_delete_marker: false,
|
||||
source_replication_status: &ReplicationStatusType::Pending,
|
||||
source_version_purge_status: &VersionPurgeStatusType::Empty,
|
||||
source_replication_status: &ReplicationStatusType::Empty,
|
||||
source_version_purge_status: &VersionPurgeStatusType::Pending,
|
||||
deleted_delete_marker_version: false,
|
||||
}));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_replication_schedule_skips_non_pending_states() {
|
||||
for replication_status in [
|
||||
ReplicationStatusType::Empty,
|
||||
ReplicationStatusType::Replica,
|
||||
ReplicationStatusType::Completed,
|
||||
ReplicationStatusType::CompletedLegacy,
|
||||
ReplicationStatusType::Failed,
|
||||
] {
|
||||
assert!(!should_schedule_delete_replication(ReplicationDeleteScheduleInput {
|
||||
replication_request: false,
|
||||
version_id_requested: false,
|
||||
source_delete_marker: false,
|
||||
source_replication_status: &replication_status,
|
||||
source_version_purge_status: &VersionPurgeStatusType::Empty,
|
||||
deleted_delete_marker_version: false,
|
||||
}));
|
||||
}
|
||||
|
||||
for version_purge_status in [
|
||||
VersionPurgeStatusType::Empty,
|
||||
VersionPurgeStatusType::Complete,
|
||||
VersionPurgeStatusType::Failed,
|
||||
] {
|
||||
assert!(!should_schedule_delete_replication(ReplicationDeleteScheduleInput {
|
||||
replication_request: false,
|
||||
version_id_requested: true,
|
||||
source_delete_marker: true,
|
||||
source_replication_status: &ReplicationStatusType::Completed,
|
||||
source_version_purge_status: &version_purge_status,
|
||||
deleted_delete_marker_version: true,
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_replication_version_id_splits_marker_creation_and_purge() {
|
||||
let version_id = Uuid::new_v4();
|
||||
@@ -633,32 +569,6 @@ mod tests {
|
||||
assert_eq!(delete_replication_version_id(true, Some(version_id), true), Some(version_id));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_replication_parts_fail_closed_without_purge_version() {
|
||||
assert_eq!(delete_replication_parts(true, None, true), None);
|
||||
|
||||
for version_id in [Uuid::nil(), Uuid::new_v4()] {
|
||||
assert_eq!(
|
||||
delete_replication_parts(true, Some(version_id), true),
|
||||
Some(ReplicationDeleteParts {
|
||||
delete_marker: false,
|
||||
version_id: Some(version_id),
|
||||
delete_marker_version_id: None,
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
let marker_version_id = Uuid::new_v4();
|
||||
assert_eq!(
|
||||
delete_replication_parts(true, Some(marker_version_id), false),
|
||||
Some(ReplicationDeleteParts {
|
||||
delete_marker: true,
|
||||
version_id: None,
|
||||
delete_marker_version_id: Some(marker_version_id),
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_replication_source_selection_prefers_existing_marker_source_only_for_replica_requests() {
|
||||
assert!(should_use_existing_delete_replication_source(true, true, true));
|
||||
|
||||
@@ -17,8 +17,8 @@ use std::any::Any;
|
||||
use crate::storage_api::DeletedObject;
|
||||
use crate::{
|
||||
DeletedObjectReplicationInfo, MrfReplicateEntry, REPLICATE_EXISTING, REPLICATE_HEAL, REPLICATE_HEAL_DELETE,
|
||||
ReplicateObjectInfo, ReplicationDeleteParts, ReplicationStatusType, ReplicationType, ReplicationWorkerOperation,
|
||||
ResyncDecision, VersionPurgeStatusType, delete_replication_parts,
|
||||
ReplicateObjectInfo, ReplicationStatusType, ReplicationType, ReplicationWorkerOperation, ResyncDecision,
|
||||
VersionPurgeStatusType,
|
||||
};
|
||||
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||
@@ -109,11 +109,7 @@ pub fn replication_heal_queue_action(roi: &mut ReplicateObjectInfo) -> Replicati
|
||||
}
|
||||
|
||||
if roi.delete_marker || !roi.version_purge_status.is_empty() {
|
||||
let Some(parts) = delete_replication_parts(roi.delete_marker, roi.version_id, !roi.version_purge_status.is_empty())
|
||||
else {
|
||||
return ReplicationHealQueueAction::Skip;
|
||||
};
|
||||
let delete_info = heal_deleted_object_replication_info(roi, parts);
|
||||
let delete_info = heal_deleted_object_replication_info(roi);
|
||||
|
||||
if is_pending_or_failed_object_heal(roi) || is_pending_or_failed_version_purge(roi) {
|
||||
return ReplicationHealQueueAction::QueueDelete(delete_info);
|
||||
@@ -148,18 +144,21 @@ pub fn replication_heal_queue_action(roi: &mut ReplicateObjectInfo) -> Replicati
|
||||
ReplicationHealQueueAction::Skip
|
||||
}
|
||||
|
||||
fn heal_deleted_object_replication_info(
|
||||
roi: &ReplicateObjectInfo,
|
||||
parts: ReplicationDeleteParts,
|
||||
) -> DeletedObjectReplicationInfo {
|
||||
fn heal_deleted_object_replication_info(roi: &ReplicateObjectInfo) -> DeletedObjectReplicationInfo {
|
||||
let (version_id, delete_marker_version_id) = if roi.version_purge_status.is_empty() {
|
||||
(None, roi.version_id)
|
||||
} else {
|
||||
(roi.version_id, None)
|
||||
};
|
||||
|
||||
DeletedObjectReplicationInfo {
|
||||
delete_object: DeletedObject {
|
||||
object_name: roi.name.clone(),
|
||||
delete_marker_version_id: parts.delete_marker_version_id,
|
||||
version_id: parts.version_id,
|
||||
delete_marker_version_id,
|
||||
version_id,
|
||||
replication_state: roi.replication_state.clone(),
|
||||
delete_marker_mtime: roi.mod_time,
|
||||
delete_marker: parts.delete_marker,
|
||||
delete_marker: roi.delete_marker,
|
||||
..Default::default()
|
||||
},
|
||||
bucket: roi.bucket.clone(),
|
||||
@@ -400,7 +399,6 @@ mod tests {
|
||||
let version_id = Uuid::new_v4();
|
||||
let mut roi = replicate_object_info(ReplicationStatusType::Completed);
|
||||
roi.version_id = Some(version_id);
|
||||
roi.delete_marker = true;
|
||||
roi.version_purge_status = VersionPurgeStatusType::Pending;
|
||||
|
||||
let action = replication_heal_queue_action(&mut roi);
|
||||
@@ -410,16 +408,6 @@ mod tests {
|
||||
};
|
||||
assert_eq!(delete_info.delete_object.version_id, Some(version_id));
|
||||
assert_eq!(delete_info.delete_object.delete_marker_version_id, None);
|
||||
assert!(!delete_info.delete_object.delete_marker);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn heal_queue_action_skips_version_purge_without_version_id() {
|
||||
let mut roi = replicate_object_info(ReplicationStatusType::Completed);
|
||||
roi.delete_marker = true;
|
||||
roi.version_purge_status = VersionPurgeStatusType::Pending;
|
||||
|
||||
assert!(matches!(replication_heal_queue_action(&mut roi), ReplicationHealQueueAction::Skip));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -25,9 +25,6 @@ keywords = ["s3-select", "api", "rustfs", "Minio", "object-store"]
|
||||
categories = ["web-programming", "development-tools", "asynchronous"]
|
||||
documentation = "https://docs.rs/rustfs-s3select-api/latest/rustfs_s3select_api/"
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[dependencies]
|
||||
metrics = { workspace = true }
|
||||
async-trait.workspace = true
|
||||
|
||||
@@ -25,9 +25,6 @@ keywords = ["s3-select", "query-engine", "rustfs", "Minio", "data-retrieval"]
|
||||
categories = ["web-programming", "development-tools", "data-structures"]
|
||||
documentation = "https://docs.rs/rustfs-s3select-query/latest/rustfs_s3select_query/"
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[dependencies]
|
||||
rustfs-s3select-api = { workspace = true }
|
||||
async-recursion = { workspace = true }
|
||||
|
||||
@@ -76,7 +76,7 @@ impl ContextProviderExtension for MetadataProvider {
|
||||
|
||||
let table_handle = self.build_table_handle()?;
|
||||
|
||||
Ok(Arc::new(TableSourceAdapter::try_new(table_ref, table_name, table_handle)?))
|
||||
Ok(Arc::new(TableSourceAdapter::try_new(table_ref.clone(), table_name, table_handle)?))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -48,7 +48,6 @@ pub const DATA_USAGE_ROOT: &str = SLASH_SEPARATOR;
|
||||
const DATA_USAGE_BLOOM_NAME: &str = ".bloomcycle.bin";
|
||||
|
||||
pub const DATA_USAGE_CACHE_NAME: &str = ".usage-cache.bin";
|
||||
pub(crate) const DATA_USAGE_CACHE_KEY_FORMAT: u16 = 1;
|
||||
const DATA_USAGE_CACHE_SAVE_RETRIES: u32 = 2;
|
||||
const DATA_USAGE_CACHE_BACKUP_SAVE_TIMEOUT_SECS_MAX: u64 = 5;
|
||||
const DATA_USAGE_CACHE_BACKUP_SAVE_RETRIES: u32 = 0;
|
||||
@@ -438,8 +437,6 @@ pub struct DataUsageCacheInfo {
|
||||
pub snapshot_complete: bool,
|
||||
#[serde(default)]
|
||||
pub scan_plan_digest: Option<DataUsageScanPlanDigest>,
|
||||
#[serde(default)]
|
||||
pub cache_key_format: u16,
|
||||
}
|
||||
|
||||
impl Serialize for DataUsageCacheInfo {
|
||||
@@ -449,7 +446,7 @@ impl Serialize for DataUsageCacheInfo {
|
||||
{
|
||||
// Keep this metadata map-encoded so older readers can ignore fields
|
||||
// appended by newer scanner versions during rolling upgrades.
|
||||
let mut state = serializer.serialize_map(Some(16))?;
|
||||
let mut state = serializer.serialize_map(Some(15))?;
|
||||
state.serialize_entry("name", &self.name)?;
|
||||
state.serialize_entry("next_cycle", &self.next_cycle)?;
|
||||
state.serialize_entry("leader_epoch", &self.leader_epoch)?;
|
||||
@@ -465,7 +462,6 @@ impl Serialize for DataUsageCacheInfo {
|
||||
state.serialize_entry("source", &self.source)?;
|
||||
state.serialize_entry("snapshot_complete", &self.snapshot_complete)?;
|
||||
state.serialize_entry("scan_plan_digest", &self.scan_plan_digest)?;
|
||||
state.serialize_entry("cache_key_format", &self.cache_key_format)?;
|
||||
state.end()
|
||||
}
|
||||
}
|
||||
@@ -504,34 +500,19 @@ impl DataUsageCache {
|
||||
|
||||
let source_matches = self.info.source == Some(source);
|
||||
let plan_matches = self.info.scan_plan_digest == Some(scan_plan_digest);
|
||||
let metadata_is_reusable = self.info.name == name
|
||||
let reusable = self.info.name == name
|
||||
&& self.info.leader_epoch == leader_epoch
|
||||
&& plan_matches
|
||||
&& (source_matches || (!require_source && self.info.source.is_none()))
|
||||
&& self.info.cache_key_format == DATA_USAGE_CACHE_KEY_FORMAT;
|
||||
let reusable = metadata_is_reusable
|
||||
&& (self.cache.is_empty()
|
||||
|| if name == DATA_USAGE_ROOT || self.info.snapshot_complete {
|
||||
self.checked_flatten_complete_scope(name).is_some()
|
||||
} else {
|
||||
self.checked_flatten(name).is_some()
|
||||
});
|
||||
&& (source_matches || (!require_source && self.info.source.is_none()));
|
||||
if !reusable {
|
||||
let pending_heals = if self.info.name == name {
|
||||
std::mem::take(&mut self.info.pending_heals)
|
||||
} else {
|
||||
Vec::new()
|
||||
};
|
||||
*self = Self::default();
|
||||
self.info.name = name.to_string();
|
||||
self.info.pending_heals = pending_heals;
|
||||
}
|
||||
|
||||
self.info.next_cycle = next_cycle;
|
||||
self.info.leader_epoch = leader_epoch;
|
||||
self.info.source = Some(source);
|
||||
self.info.scan_plan_digest = Some(scan_plan_digest);
|
||||
self.info.cache_key_format = DATA_USAGE_CACHE_KEY_FORMAT;
|
||||
self.info.snapshot_complete = false;
|
||||
if reusable {
|
||||
DataUsageCachePrepareOutcome::Reused
|
||||
@@ -595,60 +576,36 @@ impl DataUsageCache {
|
||||
}
|
||||
|
||||
pub(crate) fn checked_flatten(&self, path: &str) -> Option<DataUsageEntry> {
|
||||
self.checked_flatten_inner(path).map(|(entry, _)| entry)
|
||||
}
|
||||
|
||||
pub(crate) fn checked_flatten_complete(&self, path: &str) -> Option<DataUsageEntry> {
|
||||
self.checked_flatten_inner(path)
|
||||
.filter(|(_, visited)| *visited == self.cache.len())
|
||||
.map(|(entry, _)| entry)
|
||||
}
|
||||
|
||||
pub(crate) fn checked_flatten_complete_scope(&self, path: &str) -> Option<DataUsageEntry> {
|
||||
if path == DATA_USAGE_ROOT {
|
||||
return self.checked_flatten_complete(path);
|
||||
}
|
||||
let (entry, visited) = self.checked_flatten_inner(path)?;
|
||||
let root_parent_only = {
|
||||
let path_key = hash_path(path).key();
|
||||
self.cache
|
||||
.get(DATA_USAGE_ROOT)
|
||||
.is_some_and(|root| root_is_parent_only(root, &path_key))
|
||||
};
|
||||
let expected_entries = self.cache.len().saturating_sub(usize::from(root_parent_only));
|
||||
(visited == expected_entries).then_some(entry)
|
||||
}
|
||||
|
||||
fn checked_flatten_inner(&self, path: &str) -> Option<(DataUsageEntry, usize)> {
|
||||
let root_key = hash_path(path).key();
|
||||
let (root_key, root) = self.cache.get_key_value(&root_key)?;
|
||||
if root.compacted && !root.children.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let mut visited = HashSet::from([root_key.as_str()]);
|
||||
let mut pending = root.children.iter().map(|child| (child.as_str(), 1usize)).collect::<Vec<_>>();
|
||||
let root = self.cache.get(&root_key)?;
|
||||
let mut visited = HashSet::from([root_key]);
|
||||
let mut pending = root.children.iter().map(|child| (child.clone(), 1usize)).collect::<Vec<_>>();
|
||||
let mut flattened = DataUsageEntry::default();
|
||||
if !flattened.checked_merge(root) {
|
||||
let mut root_entry = root.clone();
|
||||
root_entry.children.clear();
|
||||
if !flattened.checked_merge(&root_entry) {
|
||||
return None;
|
||||
}
|
||||
flattened.compacted = root.compacted;
|
||||
|
||||
while let Some((key, depth)) = pending.pop() {
|
||||
if depth > MAX_DATA_USAGE_CACHE_DEPTH || !visited.insert(key) {
|
||||
if depth > MAX_DATA_USAGE_CACHE_DEPTH || !visited.insert(key.clone()) {
|
||||
return None;
|
||||
}
|
||||
let entry = self.cache.get(key)?;
|
||||
if (entry.compacted || depth == MAX_DATA_USAGE_CACHE_DEPTH) && !entry.children.is_empty() {
|
||||
let entry = self.cache.get(&key)?;
|
||||
if depth == MAX_DATA_USAGE_CACHE_DEPTH && !entry.children.is_empty() {
|
||||
return None;
|
||||
}
|
||||
pending.extend(entry.children.iter().map(|child| (child.as_str(), depth + 1)));
|
||||
pending.extend(entry.children.iter().map(|child| (child.clone(), depth + 1)));
|
||||
|
||||
if !flattened.checked_merge(entry) {
|
||||
let mut child_entry = entry.clone();
|
||||
child_entry.children.clear();
|
||||
if !flattened.checked_merge(&child_entry) {
|
||||
return None;
|
||||
}
|
||||
}
|
||||
|
||||
Some((flattened, visited.len()))
|
||||
Some(flattened)
|
||||
}
|
||||
|
||||
fn flatten_with_guard(&self, root: &DataUsageEntry, visited: &mut HashSet<String>, depth: usize) -> DataUsageEntry {
|
||||
@@ -1567,18 +1524,6 @@ fn mark_with_depth(duc: &DataUsageCache, entry: &DataUsageEntry, found: &mut Has
|
||||
}
|
||||
}
|
||||
|
||||
fn root_is_parent_only(root: &DataUsageEntry, child: &str) -> bool {
|
||||
root.children.len() == 1
|
||||
&& root.children.contains(child)
|
||||
&& root.size == 0
|
||||
&& root.objects == 0
|
||||
&& root.versions == 0
|
||||
&& root.delete_markers == 0
|
||||
&& root.replication_stats.is_none()
|
||||
&& !root.compacted
|
||||
&& root.failed_objects == 0
|
||||
}
|
||||
|
||||
/// Trait for storage-specific operations on DataUsageCache
|
||||
#[async_trait::async_trait]
|
||||
pub trait DataUsageCacheStorage {
|
||||
@@ -2342,7 +2287,6 @@ mod tests {
|
||||
assert!(decoded.source.is_none());
|
||||
assert!(!decoded.snapshot_complete);
|
||||
assert!(decoded.scan_plan_digest.is_none());
|
||||
assert_eq!(decoded.cache_key_format, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -2384,7 +2328,6 @@ mod tests {
|
||||
assert!(decoded.source.is_none());
|
||||
assert!(!decoded.snapshot_complete);
|
||||
assert!(decoded.scan_plan_digest.is_none());
|
||||
assert_eq!(decoded.cache_key_format, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -2420,7 +2363,6 @@ mod tests {
|
||||
source: Some(DataUsageCacheSource::new(1, 2)),
|
||||
snapshot_complete: true,
|
||||
scan_plan_digest: Some(TEST_PLAN_DIGEST),
|
||||
cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
@@ -2439,7 +2381,6 @@ mod tests {
|
||||
assert_eq!(current.info.source, Some(DataUsageCacheSource::new(1, 2)));
|
||||
assert!(current.info.snapshot_complete);
|
||||
assert_eq!(current.info.scan_plan_digest, Some(TEST_PLAN_DIGEST));
|
||||
assert_eq!(current.info.cache_key_format, DATA_USAGE_CACHE_KEY_FORMAT);
|
||||
assert_eq!(current.find("bucket").map(|entry| entry.objects), Some(3));
|
||||
|
||||
let decoded: OldDataUsageCache = rmp_serde::from_slice(&buf).expect("Old reader failed to deserialize new cache");
|
||||
@@ -2501,7 +2442,6 @@ mod tests {
|
||||
source: Some(source),
|
||||
snapshot_complete: false,
|
||||
scan_plan_digest: Some(TEST_PLAN_DIGEST),
|
||||
cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
@@ -2526,254 +2466,6 @@ mod tests {
|
||||
assert!(!cache.info.snapshot_complete);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn data_usage_cache_prepare_for_scan_preserves_pending_heal_only_progress() {
|
||||
let source = DataUsageCacheSource::new(1, 0);
|
||||
let pending_heal = PendingScannerHeal {
|
||||
kind: PendingScannerHealKind::Object,
|
||||
bucket: "bucket".to_string(),
|
||||
object: Some("prefix/object".to_string()),
|
||||
version_id: Some("version-a".to_string()),
|
||||
scan_mode: HealScanMode::Deep,
|
||||
first_seen: 1,
|
||||
last_attempt: 2,
|
||||
attempts: 3,
|
||||
last_admission_result: "full".to_string(),
|
||||
last_admission_reason: "queue_full".to_string(),
|
||||
};
|
||||
let mut cache = DataUsageCache {
|
||||
info: DataUsageCacheInfo {
|
||||
name: "bucket".to_string(),
|
||||
next_cycle: 7,
|
||||
source: Some(source),
|
||||
scan_plan_digest: Some(TEST_PLAN_DIGEST),
|
||||
cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
|
||||
pending_heals: vec![pending_heal.clone()],
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let outcome = cache.prepare_for_scan("bucket", 8, 0, source, TEST_PLAN_DIGEST, true);
|
||||
|
||||
assert_eq!(outcome, DataUsageCachePrepareOutcome::Reused);
|
||||
assert_eq!(cache.info.pending_heals, vec![pending_heal]);
|
||||
assert!(cache.cache.is_empty());
|
||||
assert!(!cache.info.snapshot_complete);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn data_usage_cache_prepare_for_scan_preserves_namespace_pending_heals_during_key_format_rebuild() {
|
||||
let source = DataUsageCacheSource::new(1, 0);
|
||||
let pending_heal = PendingScannerHeal {
|
||||
kind: PendingScannerHealKind::Object,
|
||||
bucket: "bucket".to_string(),
|
||||
object: Some("prefix/object".to_string()),
|
||||
version_id: Some("version-a".to_string()),
|
||||
scan_mode: HealScanMode::Deep,
|
||||
first_seen: 1,
|
||||
last_attempt: 2,
|
||||
attempts: 3,
|
||||
last_admission_result: "full".to_string(),
|
||||
last_admission_reason: "queue_full".to_string(),
|
||||
};
|
||||
let mut cache = DataUsageCache {
|
||||
info: DataUsageCacheInfo {
|
||||
name: DATA_USAGE_ROOT.to_string(),
|
||||
next_cycle: 7,
|
||||
source: Some(source),
|
||||
scan_plan_digest: Some(TEST_PLAN_DIGEST),
|
||||
pending_heals: vec![pending_heal.clone()],
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
cache.replace(DATA_USAGE_ROOT, "", DataUsageEntry::default());
|
||||
|
||||
let outcome = cache.prepare_for_scan(DATA_USAGE_ROOT, 8, 0, source, TEST_PLAN_DIGEST, true);
|
||||
|
||||
assert_eq!(outcome, DataUsageCachePrepareOutcome::Reset);
|
||||
assert_eq!(cache.info.pending_heals, vec![pending_heal]);
|
||||
assert!(cache.cache.is_empty());
|
||||
assert_eq!(cache.info.cache_key_format, DATA_USAGE_CACHE_KEY_FORMAT);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn data_usage_cache_prepare_for_scan_drops_pending_heals_from_a_different_scope() {
|
||||
let source = DataUsageCacheSource::new(1, 0);
|
||||
let mut cache = DataUsageCache {
|
||||
info: DataUsageCacheInfo {
|
||||
name: "old-bucket".to_string(),
|
||||
next_cycle: 7,
|
||||
source: Some(source),
|
||||
scan_plan_digest: Some(TEST_PLAN_DIGEST),
|
||||
pending_heals: vec![PendingScannerHeal {
|
||||
kind: PendingScannerHealKind::Object,
|
||||
bucket: "old-bucket".to_string(),
|
||||
object: Some("prefix/object".to_string()),
|
||||
version_id: None,
|
||||
scan_mode: HealScanMode::Normal,
|
||||
first_seen: 1,
|
||||
last_attempt: 2,
|
||||
attempts: 3,
|
||||
last_admission_result: "full".to_string(),
|
||||
last_admission_reason: "queue_full".to_string(),
|
||||
}],
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let outcome = cache.prepare_for_scan("new-bucket", 8, 0, source, TEST_PLAN_DIGEST, true);
|
||||
|
||||
assert_eq!(outcome, DataUsageCachePrepareOutcome::Reset);
|
||||
assert_eq!(cache.info.name, "new-bucket");
|
||||
assert!(cache.info.pending_heals.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn data_usage_cache_prepare_for_scan_resets_unknown_key_format() {
|
||||
let source = DataUsageCacheSource::new(1, 0);
|
||||
let mut cache = DataUsageCache {
|
||||
info: DataUsageCacheInfo {
|
||||
name: "bucket".to_string(),
|
||||
next_cycle: 7,
|
||||
source: Some(source),
|
||||
scan_plan_digest: Some(TEST_PLAN_DIGEST),
|
||||
cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT + 1,
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
cache.replace(
|
||||
"bucket",
|
||||
"",
|
||||
DataUsageEntry {
|
||||
objects: 3,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
|
||||
let outcome = cache.prepare_for_scan("bucket", 8, 0, source, TEST_PLAN_DIGEST, true);
|
||||
|
||||
assert_eq!(outcome, DataUsageCachePrepareOutcome::Reset);
|
||||
assert!(cache.cache.is_empty());
|
||||
assert_eq!(cache.info.cache_key_format, DATA_USAGE_CACHE_KEY_FORMAT);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn data_usage_cache_prepare_for_scan_resets_persisted_windows_key_mismatch() {
|
||||
let source = DataUsageCacheSource::new(1, 0);
|
||||
let root_key = hash_path("bucket").key();
|
||||
let mut cache = DataUsageCache {
|
||||
info: DataUsageCacheInfo {
|
||||
name: "bucket".to_string(),
|
||||
next_cycle: 7,
|
||||
source: Some(source),
|
||||
snapshot_complete: true,
|
||||
scan_plan_digest: Some(TEST_PLAN_DIGEST),
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
cache.cache.insert(
|
||||
root_key,
|
||||
DataUsageEntry {
|
||||
children: HashSet::from(["bucket/prefix".to_string()]),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
cache.cache.insert(
|
||||
"bucket\\prefix".to_string(),
|
||||
DataUsageEntry {
|
||||
objects: 3,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
|
||||
let encoded = cache.marshal_msg().expect("legacy Windows cache should serialize");
|
||||
let mut decoded = DataUsageCache::unmarshal(&encoded).expect("legacy Windows cache should deserialize");
|
||||
let outcome = decoded.prepare_for_scan("bucket", 8, 0, source, TEST_PLAN_DIGEST, true);
|
||||
|
||||
assert_eq!(outcome, DataUsageCachePrepareOutcome::Reset);
|
||||
assert!(decoded.cache.is_empty());
|
||||
assert_eq!(decoded.info.name, "bucket");
|
||||
assert_eq!(decoded.info.next_cycle, 8);
|
||||
assert_eq!(decoded.info.source, Some(source));
|
||||
assert!(!decoded.info.snapshot_complete);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn data_usage_cache_prepare_for_scan_resets_current_cache_with_dangling_child() {
|
||||
let source = DataUsageCacheSource::new(1, 0);
|
||||
let mut cache = DataUsageCache {
|
||||
info: DataUsageCacheInfo {
|
||||
name: "bucket".to_string(),
|
||||
next_cycle: 7,
|
||||
source: Some(source),
|
||||
scan_plan_digest: Some(TEST_PLAN_DIGEST),
|
||||
cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
cache.cache.insert(
|
||||
hash_path("bucket").key(),
|
||||
DataUsageEntry {
|
||||
children: HashSet::from([hash_path("bucket/missing").key()]),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
|
||||
let outcome = cache.prepare_for_scan("bucket", 8, 0, source, TEST_PLAN_DIGEST, true);
|
||||
|
||||
assert_eq!(outcome, DataUsageCachePrepareOutcome::Reset);
|
||||
assert!(cache.cache.is_empty());
|
||||
assert_eq!(cache.info.cache_key_format, DATA_USAGE_CACHE_KEY_FORMAT);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn data_usage_cache_prepare_for_scan_resets_complete_bucket_cache_with_detached_entry() {
|
||||
let source = DataUsageCacheSource::new(1, 0);
|
||||
let mut cache = DataUsageCache {
|
||||
info: DataUsageCacheInfo {
|
||||
name: "bucket".to_string(),
|
||||
next_cycle: 7,
|
||||
source: Some(source),
|
||||
snapshot_complete: true,
|
||||
scan_plan_digest: Some(TEST_PLAN_DIGEST),
|
||||
cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
cache.cache.insert(
|
||||
hash_path("bucket").key(),
|
||||
DataUsageEntry {
|
||||
objects: 1,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
cache.cache.insert(
|
||||
hash_path("bucket/detached").key(),
|
||||
DataUsageEntry {
|
||||
objects: 2,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
assert_eq!(cache.checked_flatten("bucket").map(|entry| entry.objects), Some(1));
|
||||
assert!(
|
||||
cache.checked_flatten_complete("bucket").is_none(),
|
||||
"complete bucket cache reuse must reject detached entries"
|
||||
);
|
||||
|
||||
let outcome = cache.prepare_for_scan("bucket", 8, 0, source, TEST_PLAN_DIGEST, true);
|
||||
|
||||
assert_eq!(outcome, DataUsageCachePrepareOutcome::Reset);
|
||||
assert!(cache.cache.is_empty());
|
||||
assert_eq!(cache.info.cache_key_format, DATA_USAGE_CACHE_KEY_FORMAT);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn data_usage_cache_prepare_for_scan_rejects_legacy_cache_without_a_bucket_plan() {
|
||||
let source = DataUsageCacheSource::new(0, 0);
|
||||
@@ -3144,98 +2836,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn checked_flatten_complete_rejects_detached_entries() {
|
||||
let mut cache = DataUsageCache::default();
|
||||
cache.cache.insert(
|
||||
hash_path("bucket").key(),
|
||||
DataUsageEntry {
|
||||
objects: 1,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
cache.cache.insert(
|
||||
hash_path("bucket/detached").key(),
|
||||
DataUsageEntry {
|
||||
objects: 2,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
cache.checked_flatten("bucket").map(|entry| entry.objects),
|
||||
Some(1),
|
||||
"subtree flattening may ignore entries outside the requested subtree"
|
||||
);
|
||||
assert!(
|
||||
cache.checked_flatten_complete("bucket").is_none(),
|
||||
"an authoritative cache root must reach every persisted entry"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn checked_flatten_rejects_compacted_entries_with_children() {
|
||||
let root_key = hash_path("bucket").key();
|
||||
let child_key = hash_path("bucket/prefix").key();
|
||||
let mut cache = DataUsageCache::default();
|
||||
cache.cache.insert(
|
||||
root_key,
|
||||
DataUsageEntry {
|
||||
children: HashSet::from([child_key.clone()]),
|
||||
compacted: true,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
cache.cache.insert(
|
||||
child_key,
|
||||
DataUsageEntry {
|
||||
objects: 1,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
|
||||
assert!(
|
||||
cache.checked_flatten_complete("bucket").is_none(),
|
||||
"a compacted entry cannot retain child links without double-counting"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn checked_flatten_rejects_compacted_descendants_with_children() {
|
||||
let root_key = hash_path("bucket").key();
|
||||
let child_key = hash_path("bucket/prefix").key();
|
||||
let grandchild_key = hash_path("bucket/prefix/object").key();
|
||||
let mut cache = DataUsageCache::default();
|
||||
cache.cache.insert(
|
||||
root_key,
|
||||
DataUsageEntry {
|
||||
children: HashSet::from([child_key.clone()]),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
cache.cache.insert(
|
||||
child_key,
|
||||
DataUsageEntry {
|
||||
objects: 1,
|
||||
children: HashSet::from([grandchild_key.clone()]),
|
||||
compacted: true,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
cache.cache.insert(
|
||||
grandchild_key,
|
||||
DataUsageEntry {
|
||||
objects: 1,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
|
||||
assert!(
|
||||
cache.checked_flatten("bucket").is_none(),
|
||||
"a compacted descendant cannot retain child links without double-counting"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn checked_flatten_accepts_depth_limit_and_rejects_deeper_tree() {
|
||||
let root_key = hash_path("bucket").key();
|
||||
|
||||
@@ -14,8 +14,8 @@
|
||||
|
||||
use crate::scanner_budget::{ScannerCycleBudget, ScannerCycleBudgetConfig};
|
||||
use crate::scanner_io::{
|
||||
DataUsageCacheScanState, ScannerDiskScanOutcome, ScannerIODisk, cache_root_entry_info, current_cache_root_or_prepare,
|
||||
scanner_cache_lock_resource, scanner_cache_lock_timeout, scanner_set_disk_inventory,
|
||||
ScannerDiskScanOutcome, ScannerIODisk, cache_root_entry_info, cache_snapshot_is_current, scanner_cache_lock_resource,
|
||||
scanner_cache_lock_timeout, scanner_set_disk_inventory,
|
||||
};
|
||||
use crate::storage_api::owner::NS_SCANNER_PROTOCOL_VERSION;
|
||||
use crate::storage_api::scan::NamespaceLocking as _;
|
||||
@@ -966,34 +966,32 @@ async fn scan_and_persist_local_bucket(
|
||||
let revisions = cache.load_with_revisions(set.clone(), &cache_name).await.map_err(|err| {
|
||||
RemoteScannerServerError::worker(format!("remote namespace scanner cache load or revision lookup failed: {err}"))
|
||||
})?;
|
||||
let scan_state = current_cache_root_or_prepare(&mut cache, &bucket, source, next_cycle, leader_epoch, scan_plan_digest, true);
|
||||
match scan_state {
|
||||
DataUsageCacheScanState::Current(usage) => {
|
||||
if guard.is_lock_lost() {
|
||||
return Err(RemoteScannerServerError::worker(
|
||||
"remote namespace scanner cache lock was lost before reusing the current snapshot",
|
||||
));
|
||||
}
|
||||
return Ok(RemoteScannerFrameResult::Complete(Box::new(RemoteScannerComplete {
|
||||
source,
|
||||
scan_plan_digest,
|
||||
usage: *usage,
|
||||
pending_maintenance_work: !cache.info.pending_heals.is_empty(),
|
||||
})));
|
||||
if cache_snapshot_is_current(&cache, &bucket, source, next_cycle, leader_epoch, scan_plan_digest) {
|
||||
if guard.is_lock_lost() {
|
||||
return Err(RemoteScannerServerError::worker(
|
||||
"remote namespace scanner cache lock was lost before reusing the current snapshot",
|
||||
));
|
||||
}
|
||||
DataUsageCacheScanState::Prepared { outcome, .. } => match outcome {
|
||||
DataUsageCachePrepareOutcome::RejectedNewerCycle => {
|
||||
return Ok(RemoteScannerFrameResult::CycleAhead {
|
||||
required_cycle: cache.info.next_cycle,
|
||||
});
|
||||
}
|
||||
DataUsageCachePrepareOutcome::RejectedNewerLeader => {
|
||||
return Err(RemoteScannerServerError::worker(
|
||||
"remote namespace scanner rejected work from an older leader epoch",
|
||||
));
|
||||
}
|
||||
DataUsageCachePrepareOutcome::Reused | DataUsageCachePrepareOutcome::Reset => {}
|
||||
},
|
||||
return Ok(RemoteScannerFrameResult::Complete(Box::new(RemoteScannerComplete {
|
||||
source,
|
||||
scan_plan_digest,
|
||||
usage: cache_root_entry_info(&cache)
|
||||
.map_err(|err| RemoteScannerServerError::worker(format!("remote namespace scanner cache is corrupt: {err}")))?,
|
||||
pending_maintenance_work: !cache.info.pending_heals.is_empty(),
|
||||
})));
|
||||
}
|
||||
match cache.prepare_for_scan(&bucket, next_cycle, leader_epoch, source, scan_plan_digest, true) {
|
||||
DataUsageCachePrepareOutcome::RejectedNewerCycle => {
|
||||
return Ok(RemoteScannerFrameResult::CycleAhead {
|
||||
required_cycle: cache.info.next_cycle,
|
||||
});
|
||||
}
|
||||
DataUsageCachePrepareOutcome::RejectedNewerLeader => {
|
||||
return Err(RemoteScannerServerError::worker(
|
||||
"remote namespace scanner rejected work from an older leader epoch",
|
||||
));
|
||||
}
|
||||
DataUsageCachePrepareOutcome::Reused | DataUsageCachePrepareOutcome::Reset => {}
|
||||
}
|
||||
cache.info.skip_healing = skip_healing;
|
||||
|
||||
@@ -1973,7 +1971,6 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn request_rejects_empty_truncated_oversized_and_wrong_version_payloads() {
|
||||
assert_eq!(NS_SCANNER_PROTOCOL_VERSION, 3);
|
||||
assert!(decode_remote_scanner_request(&[]).is_err());
|
||||
|
||||
let mut body = rmp_serde::to_vec_named(&test_request(Uuid::new_v4())).expect("request should encode");
|
||||
@@ -1983,7 +1980,7 @@ mod tests {
|
||||
let oversized = vec![0_u8; NS_SCANNER_MAX_REQUEST_BODY_SIZE + 1];
|
||||
assert!(decode_remote_scanner_request(&oversized).is_err());
|
||||
|
||||
for version in [2, 4] {
|
||||
for version in [NS_SCANNER_PROTOCOL_VERSION - 1, NS_SCANNER_PROTOCOL_VERSION + 1] {
|
||||
let mut wrong_version = test_request(Uuid::new_v4());
|
||||
wrong_version.version = version;
|
||||
let body = rmp_serde::to_vec_named(&wrong_version).expect("request should encode");
|
||||
@@ -2888,15 +2885,14 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn wrong_frame_version_and_sequence_are_rejected() {
|
||||
assert_eq!(NS_SCANNER_PROTOCOL_VERSION, 3);
|
||||
let request_id = Uuid::new_v4();
|
||||
let auth = FrameAuthenticator::for_test(request_id);
|
||||
let frame = RemoteScannerFrame::progress(RemoteScannerProgress::default());
|
||||
let payload = rmp_serde::to_vec_named(&frame).expect("frame should encode");
|
||||
|
||||
for (version, sequence, expected_error) in [
|
||||
(2, 0, "unsupported remote namespace scanner frame version"),
|
||||
(4, 0, "unsupported remote namespace scanner frame version"),
|
||||
(NS_SCANNER_PROTOCOL_VERSION - 1, 0, "unsupported remote namespace scanner frame version"),
|
||||
(NS_SCANNER_PROTOCOL_VERSION + 1, 0, "unsupported remote namespace scanner frame version"),
|
||||
(NS_SCANNER_PROTOCOL_VERSION, 1, "frame sequence is invalid"),
|
||||
] {
|
||||
let envelope = RemoteScannerFrameEnvelope {
|
||||
|
||||
@@ -1508,9 +1508,6 @@ impl FolderScanner {
|
||||
if entry.children.contains(&child) {
|
||||
continue;
|
||||
}
|
||||
if !self.old_cache.cache.contains_key(&child) {
|
||||
continue;
|
||||
}
|
||||
|
||||
let child_hash = DataUsageHash(child.clone());
|
||||
self.new_cache
|
||||
@@ -2323,6 +2320,7 @@ impl FolderScanner {
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
|
||||
let h = DataUsageHash(folder_item.name.clone());
|
||||
into.add_child(&h);
|
||||
self.record_scan_resume_hint(&folder_item.name);
|
||||
// We scanned a folder, optionally send update.
|
||||
@@ -2648,7 +2646,7 @@ impl FolderScanner {
|
||||
tokio::task::yield_now().await;
|
||||
} else {
|
||||
let mut dst = DataUsageEntry::default();
|
||||
let h = hash_path(&folder_item.name);
|
||||
let h = DataUsageHash(folder_item.name.clone());
|
||||
|
||||
// Use Box::pin for recursive async call
|
||||
let fut = Box::pin(self.scan_folder(ctx.clone(), folder_item.clone(), &mut dst));
|
||||
@@ -2913,7 +2911,7 @@ mod tests {
|
||||
use serial_test::serial;
|
||||
#[cfg(unix)]
|
||||
use std::os::unix::fs::{PermissionsExt, symlink};
|
||||
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
|
||||
use std::sync::atomic::AtomicBool;
|
||||
use temp_env::{with_var, with_var_unset};
|
||||
use uuid::Uuid;
|
||||
|
||||
@@ -3866,15 +3864,11 @@ mod tests {
|
||||
}
|
||||
|
||||
async fn write_test_object_metadata(root: &std::path::Path, bucket: &str, object: &str) {
|
||||
write_test_object_metadata_bytes(root, bucket, object, &metadata_for_object(bucket, object)).await;
|
||||
}
|
||||
|
||||
async fn write_test_object_metadata_bytes(root: &std::path::Path, bucket: &str, object: &str, metadata: &[u8]) {
|
||||
let object_dir = root.join(bucket).join(object);
|
||||
tokio::fs::create_dir_all(&object_dir)
|
||||
.await
|
||||
.expect("failed to create test object directory");
|
||||
tokio::fs::write(object_dir.join("xl.meta"), metadata)
|
||||
tokio::fs::write(object_dir.join("xl.meta"), metadata_for_object(bucket, object))
|
||||
.await
|
||||
.expect("failed to write test object metadata");
|
||||
}
|
||||
@@ -4171,28 +4165,27 @@ mod tests {
|
||||
async fn test_scan_folder_exits_when_abandoned_child_listing_finishes() {
|
||||
let (mut scanner, temp_dir) = build_test_scanner().await;
|
||||
let _guard = TestGuard::new(60, 100, &mut scanner, temp_dir.clone());
|
||||
let heal_starts = Arc::new(AtomicUsize::new(0));
|
||||
let heal_starts_clone = heal_starts.clone();
|
||||
let mut heal_rx =
|
||||
rustfs_common::heal_channel::init_heal_channel().expect("heal channel should initialize once for scanner tests");
|
||||
let _heal_responder = tokio::spawn(async move {
|
||||
while let Some(command) = heal_rx.recv().await {
|
||||
if let rustfs_common::heal_channel::HealChannelCommand::Start { response_tx, .. } = command {
|
||||
heal_starts_clone.fetch_add(1, Ordering::Relaxed);
|
||||
let _ = response_tx.send(Ok(HealAdmissionResult::Accepted));
|
||||
let _heal_responder = rustfs_common::heal_channel::init_heal_channel().ok().map(|mut heal_rx| {
|
||||
tokio::spawn(async move {
|
||||
while let Some(command) = heal_rx.recv().await {
|
||||
if let rustfs_common::heal_channel::HealChannelCommand::Start { response_tx, .. } = command {
|
||||
let _ = response_tx.send(Ok(HealAdmissionResult::Accepted));
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
});
|
||||
|
||||
let bucket = "src-archive";
|
||||
let object = "snapshots/37b3f20d941e2f5e6d99114d9bb2f3e67a8a2e5c9c4c5a1b0d6e7f8091a2b3c4";
|
||||
let metadata = metadata_for_object(bucket, object);
|
||||
write_test_object_metadata_bytes(&temp_dir, bucket, object, &metadata).await;
|
||||
tokio::fs::create_dir_all(temp_dir.join(bucket))
|
||||
.await
|
||||
.expect("failed to create bucket directory");
|
||||
|
||||
let mut disks = vec![scanner.local_disk.clone()];
|
||||
for disk_name in ["disk2", "disk3", "disk4"] {
|
||||
let disk_root = temp_dir.join(disk_name);
|
||||
write_test_object_metadata_bytes(&disk_root, bucket, object, &metadata).await;
|
||||
tokio::fs::create_dir_all(disk_root.join(bucket))
|
||||
.await
|
||||
.expect("failed to create extra disk bucket directory");
|
||||
let endpoint =
|
||||
Endpoint::try_from(disk_root.to_string_lossy().as_ref()).expect("failed to create extra disk endpoint");
|
||||
let disk = new_disk(
|
||||
@@ -4211,7 +4204,7 @@ mod tests {
|
||||
scanner.disks = disks;
|
||||
scanner.disks_quorum = 2;
|
||||
scanner.old_cache.replace(
|
||||
&format!("{bucket}/{object}"),
|
||||
"src-archive/snapshots/37b3f20d941e2f5e6d99114d9bb2f3e67a8a2e5c9c4c5a1b0d6e7f8091a2b3c4",
|
||||
bucket,
|
||||
DataUsageEntry {
|
||||
objects: 1,
|
||||
@@ -4226,17 +4219,13 @@ mod tests {
|
||||
object_heal_prob_div: 1,
|
||||
};
|
||||
|
||||
tokio::time::timeout(Duration::from_secs(2), scanner.scan_folder(CancellationToken::new(), folder, &mut into))
|
||||
.await
|
||||
.expect("scan_folder should not hang after list_path_raw finishes")
|
||||
.expect("scan_folder should finish successfully");
|
||||
|
||||
let root = scanner
|
||||
.new_cache
|
||||
.checked_flatten(bucket)
|
||||
.expect("healed cache must contain canonical child links");
|
||||
assert_eq!(root.objects, 1);
|
||||
assert!(heal_starts.load(Ordering::Relaxed) > 0, "test must execute the heal child-link path");
|
||||
tokio::time::timeout(
|
||||
Duration::from_millis(200),
|
||||
scanner.scan_folder(CancellationToken::new(), folder, &mut into),
|
||||
)
|
||||
.await
|
||||
.expect("scan_folder should not hang after list_path_raw finishes")
|
||||
.expect("scan_folder should finish successfully");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -4785,7 +4774,7 @@ mod tests {
|
||||
.expect("unbounded scan should finish after partial progress");
|
||||
|
||||
let root = result
|
||||
.checked_flatten("bucket")
|
||||
.size_recursive("bucket")
|
||||
.expect("completed cache should retain bucket usage");
|
||||
assert_eq!(root.objects, 5);
|
||||
assert!(result.info.snapshot_complete);
|
||||
@@ -4839,106 +4828,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_partial_entry_does_not_carry_missing_old_child() {
|
||||
let (mut scanner, temp_dir) = build_test_scanner().await;
|
||||
let _guard = TestGuard {
|
||||
temp_dir: Some(temp_dir),
|
||||
};
|
||||
let root_hash = hash_path("bucket");
|
||||
scanner.old_cache.cache.insert(
|
||||
root_hash.key(),
|
||||
DataUsageEntry {
|
||||
children: HashSet::from([hash_path("bucket/missing").key()]),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
|
||||
let mut partial = DataUsageEntry {
|
||||
objects: 2,
|
||||
size: 2,
|
||||
..Default::default()
|
||||
};
|
||||
scanner.carry_forward_old_children(&root_hash, &mut partial);
|
||||
scanner.new_cache.replace_hashed(&root_hash, &None, &partial);
|
||||
|
||||
assert!(partial.children.is_empty());
|
||||
let flattened = scanner
|
||||
.new_cache
|
||||
.checked_flatten("bucket")
|
||||
.expect("a partial cache must not retain dangling child links");
|
||||
assert_eq!(flattened.objects, 2);
|
||||
assert_eq!(flattened.size, 2);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_legacy_windows_cache_rebuilds_and_round_trips_portable_keys() {
|
||||
let (scanner, temp_dir) = build_test_scanner().await;
|
||||
let _guard = TestGuard {
|
||||
temp_dir: Some(temp_dir.clone()),
|
||||
};
|
||||
write_test_object_metadata(&temp_dir, "bucket", "prefix/object").await;
|
||||
|
||||
let source = crate::data_usage_define::DataUsageCacheSource::new(0, 0);
|
||||
let scan_plan_digest = crate::data_usage_define::DataUsageScanPlanDigest([9; 32]);
|
||||
let mut legacy = DataUsageCache {
|
||||
info: crate::data_usage_define::DataUsageCacheInfo {
|
||||
name: "bucket".to_string(),
|
||||
next_cycle: 7,
|
||||
source: Some(source),
|
||||
scan_plan_digest: Some(scan_plan_digest),
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
legacy.cache.insert(
|
||||
"bucket".to_string(),
|
||||
DataUsageEntry {
|
||||
children: HashSet::from(["bucket\\prefix".to_string()]),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
legacy.cache.insert(
|
||||
"bucket\\prefix".to_string(),
|
||||
DataUsageEntry {
|
||||
objects: 1,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
let encoded = legacy.marshal_msg().expect("legacy cache should serialize");
|
||||
let mut migrated = DataUsageCache::unmarshal(&encoded).expect("legacy cache should deserialize");
|
||||
assert_eq!(
|
||||
migrated.prepare_for_scan("bucket", 8, 0, source, scan_plan_digest, true),
|
||||
crate::data_usage_define::DataUsageCachePrepareOutcome::Reset
|
||||
);
|
||||
|
||||
let parent = CancellationToken::new();
|
||||
let budget = ScannerCycleBudget::new(&parent, Default::default());
|
||||
let rebuilt = scan_data_folder(
|
||||
budget.token(),
|
||||
budget,
|
||||
vec![scanner.local_disk.clone()],
|
||||
scanner.local_disk,
|
||||
migrated,
|
||||
None,
|
||||
HealScanMode::Normal,
|
||||
SCANNER_SLEEPER.clone(),
|
||||
)
|
||||
.await
|
||||
.expect("portable cache rebuild should complete");
|
||||
let persisted = rebuilt.marshal_msg().expect("rebuilt cache should serialize");
|
||||
let decoded = DataUsageCache::unmarshal(&persisted).expect("rebuilt cache should deserialize");
|
||||
|
||||
assert_eq!(decoded.info.cache_key_format, crate::data_usage_define::DATA_USAGE_CACHE_KEY_FORMAT);
|
||||
assert!(decoded.cache.keys().all(|key| !key.contains('\\')));
|
||||
let root = decoded
|
||||
.checked_flatten("bucket")
|
||||
.expect("rebuilt persisted cache should have a complete root");
|
||||
assert_eq!(root.objects, 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_scan_data_folder_success_clears_resume_hint() {
|
||||
|
||||
@@ -12,7 +12,6 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::data_usage_define::DATA_USAGE_CACHE_KEY_FORMAT;
|
||||
use crate::scanner_budget::ScannerCycleBudget;
|
||||
use crate::scanner_folder::{ScannerItem, scan_data_folder};
|
||||
use crate::sleeper::SCANNER_SLEEPER;
|
||||
@@ -831,7 +830,7 @@ pub(crate) fn cache_root_entry_info(cache: &DataUsageCache) -> std::result::Resu
|
||||
return Err(ScannerError::Other("scanner cache root name is empty".to_string()));
|
||||
}
|
||||
let entry = cache
|
||||
.checked_flatten_complete_scope(&cache.info.name)
|
||||
.checked_flatten(&cache.info.name)
|
||||
.ok_or_else(|| ScannerError::Other(format!("scanner cache root is missing or corrupt: {}", cache.info.name)))?;
|
||||
|
||||
Ok(DataUsageEntryInfo {
|
||||
@@ -853,7 +852,7 @@ fn should_publish_completed_snapshot(completed_count: usize, total_count: usize,
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
enum NamespaceScannerWorkerMode {
|
||||
Coordinator,
|
||||
RemoteV4(uuid::Uuid),
|
||||
RemoteV3(uuid::Uuid),
|
||||
}
|
||||
|
||||
fn namespace_scanner_workers<T>(
|
||||
@@ -869,7 +868,7 @@ fn namespace_scanner_workers<T>(
|
||||
workers.extend(
|
||||
remote_disks
|
||||
.into_iter()
|
||||
.map(|(disk, server_epoch)| (disk, NamespaceScannerWorkerMode::RemoteV4(server_epoch))),
|
||||
.map(|(disk, server_epoch)| (disk, NamespaceScannerWorkerMode::RemoteV3(server_epoch))),
|
||||
);
|
||||
workers
|
||||
}
|
||||
@@ -959,57 +958,7 @@ where
|
||||
let _ = tokio::time::timeout(SCANNER_CACHE_LOCK_LOSS_SHUTDOWN_TIMEOUT, scan).await;
|
||||
}
|
||||
|
||||
pub(crate) fn current_cache_root_entry(
|
||||
cache: &DataUsageCache,
|
||||
name: &str,
|
||||
source: DataUsageCacheSource,
|
||||
next_cycle: u64,
|
||||
leader_epoch: u64,
|
||||
scan_plan_digest: DataUsageScanPlanDigest,
|
||||
) -> std::result::Result<Option<DataUsageEntryInfo>, ScannerError> {
|
||||
let metadata_is_current = cache.info.name == name
|
||||
&& cache.info.source == Some(source)
|
||||
&& cache.info.snapshot_complete
|
||||
&& cache.info.scan_plan_digest == Some(scan_plan_digest)
|
||||
&& cache.info.last_update.is_some()
|
||||
&& cache.info.next_cycle == next_cycle
|
||||
&& cache.info.leader_epoch == leader_epoch
|
||||
&& cache.info.cache_key_format == DATA_USAGE_CACHE_KEY_FORMAT;
|
||||
if !metadata_is_current {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
cache_root_entry_info(cache).map(Some)
|
||||
}
|
||||
|
||||
pub(crate) enum DataUsageCacheScanState {
|
||||
Current(Box<DataUsageEntryInfo>),
|
||||
Prepared {
|
||||
outcome: DataUsageCachePrepareOutcome,
|
||||
invalid_current: Option<ScannerError>,
|
||||
},
|
||||
}
|
||||
|
||||
pub(crate) fn current_cache_root_or_prepare(
|
||||
cache: &mut DataUsageCache,
|
||||
name: &str,
|
||||
source: DataUsageCacheSource,
|
||||
next_cycle: u64,
|
||||
leader_epoch: u64,
|
||||
scan_plan_digest: DataUsageScanPlanDigest,
|
||||
require_source: bool,
|
||||
) -> DataUsageCacheScanState {
|
||||
match current_cache_root_entry(cache, name, source, next_cycle, leader_epoch, scan_plan_digest) {
|
||||
Ok(Some(root)) => DataUsageCacheScanState::Current(Box::new(root)),
|
||||
current => DataUsageCacheScanState::Prepared {
|
||||
invalid_current: current.err(),
|
||||
outcome: cache.prepare_for_scan(name, next_cycle, leader_epoch, source, scan_plan_digest, require_source),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn cache_snapshot_is_current(
|
||||
pub(crate) fn cache_snapshot_is_current(
|
||||
cache: &DataUsageCache,
|
||||
name: &str,
|
||||
source: DataUsageCacheSource,
|
||||
@@ -1017,10 +966,13 @@ fn cache_snapshot_is_current(
|
||||
leader_epoch: u64,
|
||||
scan_plan_digest: DataUsageScanPlanDigest,
|
||||
) -> bool {
|
||||
matches!(
|
||||
current_cache_root_entry(cache, name, source, next_cycle, leader_epoch, scan_plan_digest),
|
||||
Ok(Some(_))
|
||||
)
|
||||
cache.info.name == name
|
||||
&& cache.info.source == Some(source)
|
||||
&& cache.info.snapshot_complete
|
||||
&& cache.info.scan_plan_digest == Some(scan_plan_digest)
|
||||
&& cache.info.last_update.is_some()
|
||||
&& cache.info.next_cycle == next_cycle
|
||||
&& cache.info.leader_epoch == leader_epoch
|
||||
}
|
||||
|
||||
fn completed_data_usage_info(
|
||||
@@ -1109,7 +1061,6 @@ mod publish_gate_tests {
|
||||
source: Some(source),
|
||||
snapshot_complete: false,
|
||||
scan_plan_digest: Some(TEST_PLAN_DIGEST),
|
||||
cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
@@ -1151,7 +1102,6 @@ mod publish_gate_tests {
|
||||
source: Some(source),
|
||||
snapshot_complete: true,
|
||||
scan_plan_digest: Some(TEST_PLAN_DIGEST),
|
||||
cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
@@ -1473,145 +1423,17 @@ mod publish_gate_tests {
|
||||
assert!(!cache_snapshot_is_current(&cache, DATA_USAGE_ROOT, source, 10, 2, TEST_PLAN_DIGEST));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn current_cache_snapshot_rejects_persisted_windows_key_mismatch() {
|
||||
let source = DataUsageCacheSource::new(1, 2);
|
||||
let mut cache = DataUsageCache {
|
||||
info: DataUsageCacheInfo {
|
||||
name: "bucket".to_string(),
|
||||
next_cycle: 10,
|
||||
last_update: Some(SystemTime::UNIX_EPOCH),
|
||||
source: Some(source),
|
||||
snapshot_complete: true,
|
||||
scan_plan_digest: Some(TEST_PLAN_DIGEST),
|
||||
cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
cache.cache.insert(
|
||||
"bucket".to_string(),
|
||||
DataUsageEntry {
|
||||
children: HashSet::from(["bucket/prefix".to_string()]),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
cache.cache.insert(
|
||||
"bucket\\prefix".to_string(),
|
||||
DataUsageEntry {
|
||||
objects: 3,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
|
||||
assert!(!cache_snapshot_is_current(&cache, "bucket", source, 10, 0, TEST_PLAN_DIGEST));
|
||||
match current_cache_root_or_prepare(&mut cache, "bucket", source, 10, 0, TEST_PLAN_DIGEST, true) {
|
||||
DataUsageCacheScanState::Prepared {
|
||||
outcome: DataUsageCachePrepareOutcome::Reset,
|
||||
invalid_current: Some(_),
|
||||
} => {}
|
||||
_ => panic!("an invalid current cache must enter the rebuild path"),
|
||||
}
|
||||
assert!(cache.cache.is_empty());
|
||||
assert_eq!(cache.info.cache_key_format, DATA_USAGE_CACHE_KEY_FORMAT);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn current_cache_snapshot_rejects_structurally_valid_legacy_key_format() {
|
||||
let source = DataUsageCacheSource::new(1, 2);
|
||||
let mut cache = DataUsageCache {
|
||||
info: DataUsageCacheInfo {
|
||||
name: "bucket".to_string(),
|
||||
next_cycle: 10,
|
||||
last_update: Some(SystemTime::UNIX_EPOCH),
|
||||
source: Some(source),
|
||||
snapshot_complete: true,
|
||||
scan_plan_digest: Some(TEST_PLAN_DIGEST),
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
cache.cache.insert(
|
||||
"bucket".to_string(),
|
||||
DataUsageEntry {
|
||||
children: HashSet::from(["bucket\\prefix".to_string()]),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
cache.cache.insert(
|
||||
"bucket\\prefix".to_string(),
|
||||
DataUsageEntry {
|
||||
objects: 3,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
assert_eq!(cache.checked_flatten("bucket").map(|entry| entry.objects), Some(3));
|
||||
|
||||
match current_cache_root_or_prepare(&mut cache, "bucket", source, 10, 0, TEST_PLAN_DIGEST, true) {
|
||||
DataUsageCacheScanState::Prepared {
|
||||
outcome: DataUsageCachePrepareOutcome::Reset,
|
||||
invalid_current: None,
|
||||
} => {}
|
||||
_ => panic!("a legacy key format must enter the rebuild path"),
|
||||
}
|
||||
assert!(cache.cache.is_empty());
|
||||
assert_eq!(cache.info.cache_key_format, DATA_USAGE_CACHE_KEY_FORMAT);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn current_cache_snapshot_rejects_current_bucket_cache_with_detached_entry() {
|
||||
let source = DataUsageCacheSource::new(1, 2);
|
||||
let mut cache = DataUsageCache {
|
||||
info: DataUsageCacheInfo {
|
||||
name: "bucket".to_string(),
|
||||
next_cycle: 10,
|
||||
last_update: Some(SystemTime::UNIX_EPOCH),
|
||||
source: Some(source),
|
||||
snapshot_complete: true,
|
||||
scan_plan_digest: Some(TEST_PLAN_DIGEST),
|
||||
cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
cache.cache.insert(
|
||||
"bucket".to_string(),
|
||||
DataUsageEntry {
|
||||
objects: 1,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
cache.cache.insert(
|
||||
"bucket/detached".to_string(),
|
||||
DataUsageEntry {
|
||||
objects: 2,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
assert_eq!(cache.checked_flatten("bucket").map(|entry| entry.objects), Some(1));
|
||||
|
||||
match current_cache_root_or_prepare(&mut cache, "bucket", source, 10, 0, TEST_PLAN_DIGEST, true) {
|
||||
DataUsageCacheScanState::Prepared {
|
||||
outcome: DataUsageCachePrepareOutcome::Reset,
|
||||
invalid_current: Some(_),
|
||||
} => {}
|
||||
_ => panic!("a detached complete bucket cache must enter the rebuild path"),
|
||||
}
|
||||
assert!(cache.cache.is_empty());
|
||||
assert_eq!(cache.info.cache_key_format, DATA_USAGE_CACHE_KEY_FORMAT);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn namespace_scanner_worker_selection_keeps_coordinator_fallback_disks() {
|
||||
let server_epoch = uuid::Uuid::new_v4();
|
||||
let workers = namespace_scanner_workers(vec!["local", "legacy-remote"], vec![("v4", server_epoch)]);
|
||||
let workers = namespace_scanner_workers(vec!["local", "legacy-remote"], vec![("v3", server_epoch)]);
|
||||
|
||||
assert_eq!(
|
||||
workers,
|
||||
vec![
|
||||
("local", NamespaceScannerWorkerMode::Coordinator),
|
||||
("legacy-remote", NamespaceScannerWorkerMode::Coordinator),
|
||||
("v4", NamespaceScannerWorkerMode::RemoteV4(server_epoch)),
|
||||
("v3", NamespaceScannerWorkerMode::RemoteV3(server_epoch)),
|
||||
]
|
||||
);
|
||||
assert!(namespace_scanner_workers::<()>(Vec::new(), Vec::new()).is_empty());
|
||||
@@ -1685,7 +1507,6 @@ mod publish_gate_tests {
|
||||
source: Some(source),
|
||||
snapshot_complete: true,
|
||||
scan_plan_digest: Some(first),
|
||||
cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
@@ -1774,15 +1595,6 @@ async fn send_cache_root_entry_info(
|
||||
pending_maintenance_work: &AtomicBool,
|
||||
) -> std::result::Result<(), ScannerError> {
|
||||
let root = cache_root_entry_info(cache)?;
|
||||
send_cache_root_entry(bucket_result_tx, root, cache, pending_maintenance_work).await
|
||||
}
|
||||
|
||||
async fn send_cache_root_entry(
|
||||
bucket_result_tx: &mpsc::Sender<DataUsageEntryInfo>,
|
||||
root: DataUsageEntryInfo,
|
||||
cache: &DataUsageCache,
|
||||
pending_maintenance_work: &AtomicBool,
|
||||
) -> std::result::Result<(), ScannerError> {
|
||||
record_bucket_pending_maintenance_work(cache, pending_maintenance_work);
|
||||
bucket_result_tx
|
||||
.send(root)
|
||||
@@ -1878,16 +1690,13 @@ async fn persist_and_publish_cache_snapshot(
|
||||
);
|
||||
return None;
|
||||
}
|
||||
if matches!(
|
||||
current_cache_root_entry(
|
||||
&persisted,
|
||||
DATA_USAGE_ROOT,
|
||||
source,
|
||||
cache_snapshot.info.next_cycle,
|
||||
cache_snapshot.info.leader_epoch,
|
||||
scan_plan_digest,
|
||||
),
|
||||
Ok(Some(_))
|
||||
if cache_snapshot_is_current(
|
||||
&persisted,
|
||||
DATA_USAGE_ROOT,
|
||||
source,
|
||||
cache_snapshot.info.next_cycle,
|
||||
cache_snapshot.info.leader_epoch,
|
||||
scan_plan_digest,
|
||||
) {
|
||||
cache_snapshot = persisted;
|
||||
} else {
|
||||
@@ -2520,7 +2329,6 @@ impl ScannerIOCache for SetDisks {
|
||||
source: Some(source),
|
||||
snapshot_complete: true,
|
||||
scan_plan_digest: Some(scan_plan_digest),
|
||||
cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
|
||||
..Default::default()
|
||||
},
|
||||
cache: HashMap::new(),
|
||||
@@ -2642,7 +2450,7 @@ impl ScannerIOCache for SetDisks {
|
||||
subsystem = LOG_SUBSYSTEM_IO,
|
||||
pool = self.pool_index,
|
||||
set = self.set_index,
|
||||
v4_disks = remote_disk_count,
|
||||
v3_disks = remote_disk_count,
|
||||
unsupported_remote_disks,
|
||||
state = "unsupported_remote_disks_using_coordinator",
|
||||
"Scanner set assigned remote disks without namespace scanner support to coordinator-driven workers"
|
||||
@@ -2739,7 +2547,6 @@ impl ScannerIOCache for SetDisks {
|
||||
source: Some(source),
|
||||
snapshot_complete: false,
|
||||
scan_plan_digest: Some(scan_plan_digest),
|
||||
cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
|
||||
..Default::default()
|
||||
},
|
||||
cache: HashMap::new(),
|
||||
@@ -2831,7 +2638,7 @@ impl ScannerIOCache for SetDisks {
|
||||
let dirty_usage_buckets_clone = dirty_usage_buckets.clone();
|
||||
let cache_cycle_floor_clone = cache_cycle_floor.clone();
|
||||
let remote_server_epoch = match worker_mode {
|
||||
NamespaceScannerWorkerMode::RemoteV4(server_epoch) => Some(server_epoch),
|
||||
NamespaceScannerWorkerMode::RemoteV3(server_epoch) => Some(server_epoch),
|
||||
NamespaceScannerWorkerMode::Coordinator => None,
|
||||
};
|
||||
futs.push(tokio::spawn(async move {
|
||||
@@ -3141,71 +2948,48 @@ impl ScannerIOCache for SetDisks {
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let scan_state = current_cache_root_or_prepare(
|
||||
&mut cache,
|
||||
&bucket.name,
|
||||
source,
|
||||
want_cycle,
|
||||
leader_epoch,
|
||||
bucket_scan_plan_digest,
|
||||
require_cache_source,
|
||||
);
|
||||
let outcome = match scan_state {
|
||||
DataUsageCacheScanState::Current(root) => {
|
||||
if cache_guard.is_lock_lost() {
|
||||
record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await;
|
||||
error!(
|
||||
target: "rustfs::scanner::io",
|
||||
event = EVENT_SCANNER_CACHE_PERSIST_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_IO,
|
||||
bucket = %bucket.name,
|
||||
cache_name = %cache_name,
|
||||
state = "lock_lost_before_reuse",
|
||||
"Current scanner bucket cache root publish skipped after lock loss"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
if let Err(e) =
|
||||
send_cache_root_entry(&bucket_result_tx_clone, *root, &cache, &pending_maintenance_work_clone)
|
||||
.await
|
||||
{
|
||||
record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await;
|
||||
error!(
|
||||
target: "rustfs::scanner::io",
|
||||
event = EVENT_SCANNER_DATA_USAGE_STREAM,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_IO,
|
||||
bucket = %bucket.name,
|
||||
state = "send_current_root_failed",
|
||||
error = %e,
|
||||
"Current scanner bucket cache root entry publish failed"
|
||||
);
|
||||
}
|
||||
if cache_snapshot_is_current(&cache, &bucket.name, source, want_cycle, leader_epoch, bucket_scan_plan_digest)
|
||||
{
|
||||
if cache_guard.is_lock_lost() {
|
||||
record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await;
|
||||
error!(
|
||||
target: "rustfs::scanner::io",
|
||||
event = EVENT_SCANNER_CACHE_PERSIST_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_IO,
|
||||
bucket = %bucket.name,
|
||||
cache_name = %cache_name,
|
||||
state = "lock_lost_before_reuse",
|
||||
"Current scanner bucket cache root publish skipped after lock loss"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
DataUsageCacheScanState::Prepared {
|
||||
outcome,
|
||||
invalid_current,
|
||||
} => {
|
||||
if let Some(e) = invalid_current {
|
||||
warn!(
|
||||
target: "rustfs::scanner::io",
|
||||
event = EVENT_SCANNER_CACHE_PERSIST_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_IO,
|
||||
bucket = %bucket.name,
|
||||
cache_name = %cache_name,
|
||||
state = "current_cache_invalid",
|
||||
error = %e,
|
||||
"Current scanner bucket cache is invalid; rebuilding"
|
||||
);
|
||||
}
|
||||
outcome
|
||||
if let Err(e) =
|
||||
send_cache_root_entry_info(&bucket_result_tx_clone, &cache, &pending_maintenance_work_clone).await
|
||||
{
|
||||
record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await;
|
||||
error!(
|
||||
target: "rustfs::scanner::io",
|
||||
event = EVENT_SCANNER_DATA_USAGE_STREAM,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_IO,
|
||||
bucket = %bucket.name,
|
||||
state = "send_current_root_failed",
|
||||
error = %e,
|
||||
"Current scanner bucket cache root entry publish failed"
|
||||
);
|
||||
}
|
||||
};
|
||||
continue;
|
||||
}
|
||||
|
||||
match outcome {
|
||||
match cache.prepare_for_scan(
|
||||
&bucket.name,
|
||||
want_cycle,
|
||||
leader_epoch,
|
||||
source,
|
||||
bucket_scan_plan_digest,
|
||||
require_cache_source,
|
||||
) {
|
||||
DataUsageCachePrepareOutcome::RejectedNewerCycle => {
|
||||
cache_cycle_floor_clone.fetch_max(cache.info.next_cycle, Ordering::AcqRel);
|
||||
record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await;
|
||||
@@ -3577,7 +3361,6 @@ impl ScannerIOCache for SetDisks {
|
||||
source: Some(source),
|
||||
snapshot_complete: false,
|
||||
scan_plan_digest: Some(scan_plan_digest),
|
||||
cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
|
||||
..Default::default()
|
||||
},
|
||||
cache: HashMap::new(),
|
||||
@@ -4885,67 +4668,6 @@ mod tests {
|
||||
root.add_child(&crate::hash_path("bucket/missing"));
|
||||
dangling.replace("bucket", DATA_USAGE_ROOT, root);
|
||||
assert!(cache_root_entry_info(&dangling).is_err());
|
||||
|
||||
let mut detached = DataUsageCache {
|
||||
info: DataUsageCacheInfo {
|
||||
name: DATA_USAGE_ROOT.to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
detached.replace("bucket", DATA_USAGE_ROOT, DataUsageEntry::default());
|
||||
detached.replace(
|
||||
"bucket/detached",
|
||||
"",
|
||||
DataUsageEntry {
|
||||
objects: 1,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
assert!(cache_root_entry_info(&detached).is_err());
|
||||
|
||||
let mut detached_bucket = DataUsageCache {
|
||||
info: DataUsageCacheInfo {
|
||||
name: "bucket".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
detached_bucket.replace(
|
||||
"bucket",
|
||||
DATA_USAGE_ROOT,
|
||||
DataUsageEntry {
|
||||
objects: 1,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
detached_bucket.replace(
|
||||
"bucket/detached",
|
||||
"",
|
||||
DataUsageEntry {
|
||||
objects: 1,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
assert!(cache_root_entry_info(&detached_bucket).is_err());
|
||||
|
||||
let mut compacted_with_child = DataUsageCache {
|
||||
info: DataUsageCacheInfo {
|
||||
name: "bucket".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
compacted_with_child.replace(
|
||||
"bucket",
|
||||
DATA_USAGE_ROOT,
|
||||
DataUsageEntry {
|
||||
compacted: true,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
compacted_with_child.replace("bucket/prefix", "bucket", DataUsageEntry::default());
|
||||
assert!(cache_root_entry_info(&compacted_with_child).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -53,7 +53,7 @@ pub struct DeleteBucketOptions {
|
||||
pub no_recreate: bool,
|
||||
/// Force deletion even if bucket is not empty.
|
||||
pub force: bool,
|
||||
/// Delete empty directory remnants after the local peer verifies no object metadata exists.
|
||||
/// Force deletion only after the local peer verifies the bucket is empty.
|
||||
pub force_if_empty: bool,
|
||||
/// Site replication delete operation.
|
||||
pub srdelete_op: SRBucketDeleteOp,
|
||||
|
||||
@@ -40,7 +40,6 @@ pub const SUFFIX_TRANSITIONED_VERSION_ID: &str = "transitioned-versionID";
|
||||
pub const SUFFIX_TRANSITIONED_VERSION_STATE: &str = "transitioned-version-state";
|
||||
pub const SUFFIX_TRANSITION_TIER: &str = "transition-tier";
|
||||
pub const SUFFIX_TRANSITION_TIER_DESTINATION_ID: &str = "transition-tier-destination-id";
|
||||
pub const SUFFIX_TRANSITION_TRANSACTION_ID: &str = "transition-transaction-id";
|
||||
pub const SUFFIX_RESTORE_OPERATION_ID: &str = "restore-operation-id";
|
||||
pub const SUFFIX_FREE_VERSION: &str = "free-version";
|
||||
pub const SUFFIX_PURGESTATUS: &str = "purgestatus";
|
||||
|
||||
+2
-22
@@ -92,6 +92,7 @@ fn resolve_domain(domain: &str) -> std::io::Result<HashSet<IpAddr>> {
|
||||
(domain, 0)
|
||||
.to_socket_addrs()
|
||||
.map(|v| v.map(|v| v.ip()).collect::<HashSet<_>>())
|
||||
.map_err(Error::other)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -283,7 +284,7 @@ pub async fn get_host_ip(host: Host<&str>) -> std::io::Result<HashSet<IpAddr>> {
|
||||
}
|
||||
Err(err) => {
|
||||
error!("Failed to resolve domain {domain} using system resolver, err: {err}");
|
||||
Err(err)
|
||||
Err(Error::other(err))
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -607,27 +608,6 @@ mod test {
|
||||
assert!(get_host_ip(invalid_host).await.is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_get_host_ip_preserves_resolver_error_provenance() {
|
||||
let _resolver_guard = set_mock_dns_resolver(|_| Err(IoError::from_raw_os_error(-3)));
|
||||
|
||||
let err = get_host_ip(Host::Domain("temporarily-unavailable.example"))
|
||||
.await
|
||||
.unwrap_err();
|
||||
|
||||
assert_eq!(err.raw_os_error(), Some(-3));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_resolve_domain_preserves_system_resolver_error_provenance() {
|
||||
let _resolver_lock = DNS_RESOLVER_TEST_LOCK.lock().unwrap();
|
||||
reset_dns_resolver_inner();
|
||||
|
||||
let err = resolve_domain("rustfs-resolver-provenance.invalid").unwrap_err();
|
||||
|
||||
assert_ne!(err.kind(), std::io::ErrorKind::Other, "system resolver error was wrapped: {err}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_dns_cache_entry_expiry() {
|
||||
let ips: HashSet<IpAddr> = [IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1))].into_iter().collect();
|
||||
|
||||
@@ -360,7 +360,7 @@ pub fn find_ellipses_patterns(arg: &str) -> Result<ArgPattern> {
|
||||
Some(caps) => caps,
|
||||
None => {
|
||||
return Err(Error::other(format!(
|
||||
"Invalid ellipsis format. Ellipsis range must be provided in format {{N...M}} where N and M are decimal or hexadecimal positive integers, M must be greater than N, with a maximum expanded range size of {MAX_ELLIPSES_RANGE_SIZE}"
|
||||
"Invalid ellipsis format in ({arg}), Ellipsis range must be provided in format {{N...M}} where N and M are decimal or hexadecimal positive integers, M must be greater than N, with a maximum expanded range size of {MAX_ELLIPSES_RANGE_SIZE}"
|
||||
)));
|
||||
}
|
||||
};
|
||||
@@ -399,7 +399,7 @@ pub fn find_ellipses_patterns(arg: &str) -> Result<ArgPattern> {
|
||||
|| p.suffix.contains(CLOSE_BRACES)
|
||||
{
|
||||
return Err(Error::other(format!(
|
||||
"Invalid ellipsis format. Ellipsis range must be provided in format {{N...M}} where N and M are decimal or hexadecimal positive integers, M must be greater than N, with a maximum expanded range size of {MAX_ELLIPSES_RANGE_SIZE}"
|
||||
"Invalid ellipsis format in ({arg}), Ellipsis range must be provided in format {{N...M}} where N and M are decimal or hexadecimal positive integers, M must be greater than N, with a maximum expanded range size of {MAX_ELLIPSES_RANGE_SIZE}"
|
||||
)));
|
||||
}
|
||||
}
|
||||
@@ -938,15 +938,6 @@ mod tests {
|
||||
assert!(err.to_string().contains("decimal or hexadecimal"), "unexpected error message: {err}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_find_ellipses_patterns_leftover_brace_error_does_not_echo_input() {
|
||||
let err = find_ellipses_patterns("http://:brace-secret@server/{1...2}}").unwrap_err();
|
||||
assert!(
|
||||
!err.to_string().contains("brace-secret"),
|
||||
"ellipsis error leaked endpoint credentials: {err}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_ellipses_range_rejects_oversized_ranges() {
|
||||
let err = parse_ellipses_range("{0...10000}").unwrap_err();
|
||||
|
||||
@@ -12,10 +12,6 @@ for later deletion.
|
||||
|
||||
## Open Items
|
||||
|
||||
- `rustfs-5416` Helm distributed startup wait setting: charts that predate explicit local endpoint identity expose startupWaitTimeoutSeconds for their peer DNS/TCP init gate. The new chart keeps the value accepted but ignores it after moving startup convergence into RustFS. Remove the value and its documentation after the minimum supported direct-upgrade chart includes localEndpointHost.autoInject and no longer renders the peer gate.
|
||||
- `rustfs-5416-wait-mode` startup wait-mode validation: releases before explicit local endpoint identity treat an unknown RUSTFS_STARTUP_TOPOLOGY_WAIT_MODE value as auto. New servers retain that fallback only when no explicit local endpoint host is configured; an anchor requires a recognized mode so a typo cannot bypass DNS locality. Remove the fallback and reject every unknown value after every supported direct-upgrade chart validates this setting before rollout.
|
||||
- `rustfs-5416-kubernetes-alias-dns` Kubernetes endpoint identity fallback: deployments created before explicit local endpoint identity may use resolvable aliases that do not match the Pod hostname. An implicit auto-mode zero match retains legacy DNS locality with a bounded deadline, while ambiguous matches and invalid explicit anchors still fail closed. Remove the fallback after every supported direct-upgrade chart and deployment manifest provides a canonical RUSTFS_LOCAL_ENDPOINT_HOST for domain-based distributed topologies.
|
||||
- `rustfs-5416-zero-retry-delay` startup retry-delay validation: releases before bounded topology convergence accept RUSTFS_STARTUP_TOPOLOGY_RETRY_MAX_DELAY values of 0 or 0ms. New servers replace those values with the safe nonzero default so a direct upgrade neither fails startup nor enters a busy loop. Reject zero after the minimum supported direct-upgrade release validates or rewrites this setting before rollout.
|
||||
- `scanner-usage-v2` persisted scanner usage migration: pre-v2 scanners write `.usage.json`, so upgraded clusters read that primary/backup pair only while `.usage.v2.json` is absent and continue removing deleted buckets from legacy copies that still exist. The additive usage_snapshot_complete field in `.usage.v2.json` must remain optional while mixed-version clusters are supported; a missing field means the snapshot is not authoritative. Remove the legacy object fallback and cleanup only after every supported direct-upgrade source writes `.usage.v2.json`.
|
||||
- `ns-scanner-rpc-v3` namespace scanner capability and activity handshake: old peers and legacy internode transports lack the authenticated startup-epoch handshake. The oldest peers send an empty activity request and receive a field-empty protocol-0 response. Protocol v4 binds the challenge and response topology but cannot authenticate distributed dirty-usage state. Current protocol v5 binds the request version, acknowledgement target and generation, and the response dirty-usage state. Servers retain protocol-0 and protocol-v4 codecs for rolling upgrades, while the distributed scanner publishes usage only after every peer returns authenticated protocol v5 state. Scanner selection treats HTTP 404/405/426 and the legacy MethodNotAllowed default as an explicit lack of remote scanner v3 support and assigns those disks to coordinator-driven workers; transient capability failures remain incomplete and do not activate the fallback. Remove the coordinator fallback after the minimum supported RustFS peer version implements namespace scanner protocol v3, remove protocol-0 activity requests and responses after every supported peer implements authenticated scanner activity protocol v4, and remove the protocol-v4 activity codec after every supported peer implements protocol v5; future protocol revisions must keep the same dual-version server/codec window before changing the advertised version.
|
||||
- `#4648` walk-dir stream completion capability: old clients can append fallback output to an already-used metacache writer after a terminal body error, so servers emit terminal walk errors only to clients that sign the `walk_dir_stream_completion=error-v1` query capability and its request-body digest. Remove the legacy clean-EOF path after the minimum supported RustFS peer version always advertises this capability.
|
||||
|
||||
@@ -98,39 +98,6 @@ rc admin ilm transition run local/mybucket --prefix logs/ --tier cold --max-obje
|
||||
|
||||
Inspect the aggregate counters before widening scope. Full object-key lists are intentionally not returned by the admin response. If `RUSTFS_RPC_SECRET` or other credentials were pasted into an issue, chat, log, or ticket while debugging tiering, rotate them on every node, restart the cluster with the new value, and redact the exposed copy before sharing more diagnostics.
|
||||
|
||||
## Reconcile an unknown transition upload
|
||||
|
||||
Historical transition transactions in `upload_outcome_unknown` state can use an explicit two-stage operator workflow when the tier probe is ambiguous and the provider supports exact version deletion. The endpoint refuses transactions that are still inside their ownership window or are in any other state.
|
||||
|
||||
First inspect the transaction without changing it:
|
||||
|
||||
```text
|
||||
GET /rustfs/admin/v3/ilm/transition/reconcile/<transaction-id>
|
||||
```
|
||||
|
||||
If independent provider evidence identifies the exact remote version to remove, submit that opaque version identifier with explicit confirmation:
|
||||
|
||||
```json
|
||||
POST /rustfs/admin/v3/ilm/transition/reconcile/<transaction-id>
|
||||
{
|
||||
"action": "delete_candidate",
|
||||
"confirm": true,
|
||||
"remote_version_id": "<exact-provider-version>"
|
||||
}
|
||||
```
|
||||
|
||||
This operation performs only an exact version delete. Its response reports whether the transaction journal was still observed after the delete; background recovery may have finalized the same transaction concurrently. If the journal remains, inspect the transaction again and finalize it only after the live provider probe proves that the candidate is missing:
|
||||
|
||||
```json
|
||||
POST /rustfs/admin/v3/ilm/transition/reconcile/<transaction-id>
|
||||
{
|
||||
"action": "finalize_missing",
|
||||
"confirm": true
|
||||
}
|
||||
```
|
||||
|
||||
`finalize_missing` re-runs the provider probe and fails closed for `unversioned_present`, `versioned_present`, `ambiguous`, `unsupported`, or probe errors. It never accepts an operator assertion in place of a live `missing` result. Providers without an authoritative probe or exact version deletion remain pending; this endpoint does not infer provider capabilities, accept external absence assertions, or select a candidate automatically.
|
||||
|
||||
## Historical fixes (for context, already merged)
|
||||
|
||||
- Expire/GET race (`NoSuchVersion` during expiry of a tiered object):
|
||||
|
||||
@@ -59,7 +59,7 @@
|
||||
{
|
||||
default = rustPlatform.buildRustPackage {
|
||||
pname = "rustfs";
|
||||
version = "1.0.0-beta.12";
|
||||
version = "1.0.0-beta.11";
|
||||
|
||||
src = ./.;
|
||||
|
||||
|
||||
+14
-68
@@ -47,60 +47,6 @@ without manual intervention:
|
||||
If you previously set `replicaCount=16` and now want a different topology,
|
||||
set both `replicaCount` and `drivesPerNode` explicitly.
|
||||
|
||||
For distributed deployments that use the chart-generated
|
||||
`RUSTFS_VOLUMES`, `localEndpointHost.autoInject` defaults to automatic
|
||||
selection. Without `secret.existingSecret`, the chart injects a private
|
||||
Downward API variable, `RUSTFS_CHART_POD_NAME`, and uses it to build the pod's
|
||||
fully qualified hostname in
|
||||
`RUSTFS_LOCAL_ENDPOINT_HOST`. RustFS can then identify local drives without
|
||||
waiting for every peer's DNS record or TCP listener. A user-defined `POD_NAME`
|
||||
is preserved and does not interfere with the private chart variable. Setting
|
||||
`RUSTFS_STARTUP_TOPOLOGY_WAIT_MODE` explicitly to `bounded`, `fail-fast`,
|
||||
`failfast`, or `strict` also keeps legacy DNS-based locality discovery. A
|
||||
dynamically sourced wait mode keeps the legacy path because its value cannot be
|
||||
validated while rendering. Otherwise, Kubernetes auto-detection selects
|
||||
orchestrated startup when RustFS consumes the generated anchor.
|
||||
|
||||
An existing Secret is opaque to the chart and may historically contain more
|
||||
than credentials, so the chart does not inject an anchor whenever
|
||||
`secret.existingSecret` is set. In Kubernetes auto/orchestrated mode, RustFS
|
||||
then derives a DNS-free identity from the kernel hostname when exactly one
|
||||
domain endpoint at the server port has the same full hostname or first label.
|
||||
All-IP topologies retain direct IP locality detection. A domain topology with
|
||||
zero matches retains legacy DNS locality discovery; implicit auto mode bounds
|
||||
that compatibility path by `RUSTFS_STARTUP_TOPOLOGY_WAIT_TIMEOUT` (180 seconds
|
||||
by default). Multiple matching candidates remain an error. Set
|
||||
`RUSTFS_LOCAL_ENDPOINT_HOST` explicitly to avoid DNS discovery. For a
|
||||
credentials-only Secret, set
|
||||
`localEndpointHost.autoInject=true` to add the chart anchor without changing
|
||||
the historical ConfigMap-then-Secret `envFrom` precedence. If injection is
|
||||
explicitly enabled with an incompatible hidden `RUSTFS_VOLUMES`,
|
||||
`RUSTFS_ADDRESS`, or `RUSTFS_STARTUP_TOPOLOGY_WAIT_MODE`, RustFS also fails
|
||||
during endpoint construction; it does not silently fall back to a different
|
||||
topology.
|
||||
|
||||
When `config.rustfs.volumes` is set explicitly, the chart does not infer a
|
||||
local endpoint identity. RustFS applies the same kernel-hostname inference to
|
||||
custom domain topologies in Kubernetes auto/orchestrated mode; aliases that do
|
||||
not match the Pod hostname retain legacy DNS locality, with the bounded auto
|
||||
fallback described above. They may provide `RUSTFS_LOCAL_ENDPOINT_HOST`
|
||||
through `extraEnv` for DNS-free startup. An explicit `RUSTFS_VOLUMES`, explicit
|
||||
`RUSTFS_LOCAL_ENDPOINT_HOST`, bounded/dynamic or unrecognized startup mode, or
|
||||
`localEndpointHost.autoInject=false`, also disables chart injection. A
|
||||
`RUSTFS_ADDRESS` override alone does not disable it; the effective address and
|
||||
generated topology must agree on the endpoint port. Custom anchor-based
|
||||
configurations must resolve to `orchestrated` startup mode and must not receive
|
||||
a conflicting mode from an `envFrom` source.
|
||||
`startupWaitTimeoutSeconds` is retained for values-file compatibility but is
|
||||
deprecated and ignored.
|
||||
Historical `RUSTFS_STARTUP_TOPOLOGY_RETRY_MAX_DELAY` values of `0` or `0ms`
|
||||
are replaced with the safe default retry cap instead of causing a busy loop or
|
||||
blocking a direct upgrade.
|
||||
|
||||
Upgrade the chart and RustFS image together. An older image that does not
|
||||
recognize `RUSTFS_LOCAL_ENDPOINT_HOST` retains its previous DNS-based startup
|
||||
behavior.
|
||||
|
||||
---
|
||||
|
||||
# Parameters Overview
|
||||
@@ -111,7 +57,6 @@ behavior.
|
||||
| affinity.podAntiAffinity.enabled | bool | `true` | |
|
||||
| affinity.podAntiAffinity.topologyKey | string | `"kubernetes.io/hostname"` | |
|
||||
| clusterDomain | string | `"cluster.local"` | Kubernetes cluster DNS domain used to build in-cluster FQDNs for `RUSTFS_VOLUMES` (distributed mode) and mTLS server certificate SANs. Override for clusters not using the default `cluster.local`. Provide the DNS root only, without a `svc.` prefix or leading/trailing dots. |
|
||||
| localEndpointHost.autoInject | bool or null | `null` | Automatically inject `RUSTFS_LOCAL_ENDPOINT_HOST` for chart-generated distributed topologies unless `secret.existingSecret` is set. Use `true` for a credentials-only existing Secret or `false` to preserve legacy DNS locality explicitly. |
|
||||
| commonLabels | object | `{}` | Labels to add to all deployed objects. |
|
||||
| config.rustfs.address | string | `":9000"` | |
|
||||
| config.rustfs.console_address | string | `":9001"` | |
|
||||
@@ -121,7 +66,7 @@ behavior.
|
||||
| config.rustfs.obs_environment | string | `"development"` | |
|
||||
| config.rustfs.obs_log_directory | string | `"/logs"` | Log directory inside the RustFS container. Set to `""` to disable log PVCs and mounts. |
|
||||
| config.rustfs.region | string | `"us-east-1"` | |
|
||||
| config.rustfs.volumes | string | `""` | Explicit distributed volume topology. When empty, the chart generates the topology and normally injects `RUSTFS_LOCAL_ENDPOINT_HOST`; custom topologies must configure local endpoint identity explicitly when needed. |
|
||||
| config.rustfs.volumes | string | `""` | |
|
||||
| config.rustfs.log_rotation.size | int | `"100"` | Default log rotation size mb for rustfs. |
|
||||
| config.rustfs.log_rotation.time | string | `"hour"` | Default log rotation time for rustfs. |
|
||||
| config.rustfs.log_rotation.keep_files | int | `"30"` | Default log keep files for rustfs. |
|
||||
@@ -162,7 +107,7 @@ behavior.
|
||||
| config.rustfs.kms.vault.vault_token | string | `""`| The vault token. Rendered into a dedicated Secret (`<fullname>-kms-secret`), never into the ConfigMap. |
|
||||
| config.rustfs.kms.vault.vault_mount_path | string | `"transit"`| The vault mount path, only works if `vault_backend` equals `vault-transit` . |
|
||||
| config.rustfs.kms.vault.default_key | string | `"transit"`| The master key id for RustFS. |
|
||||
| extraEnv | list | `[]` | Extra environment variables for the RustFS container. An explicit `RUSTFS_LOCAL_ENDPOINT_HOST` or `RUSTFS_VOLUMES`, or a bounded, dynamic, or unrecognized startup mode, disables generated anchor injection. `POD_NAME` and `RUSTFS_ADDRESS` remain independent overrides. |
|
||||
| extraEnv | map | `[]` | Extra environment variables for RustFS container. |
|
||||
| extraVolumes | list | `[]` | Extra volumes to add to the pod spec. Supported in both standalone (Deployment) and distributed (StatefulSet) modes. |
|
||||
| extraVolumeMounts | list | `[]` | Extra volume mounts to add to the RustFS container. Supported in both standalone (Deployment) and distributed (StatefulSet) modes. |
|
||||
| containerSecurityContext.capabilities.drop[0] | string | `"ALL"` | |
|
||||
@@ -245,7 +190,7 @@ uer. `ClusterIssuer` or `Issuer`. |
|
||||
| resources.limits.memory | string | `"512Mi"` | |
|
||||
| resources.requests.cpu | string | `"100m"` | |
|
||||
| resources.requests.memory | string | `"128Mi"` | |
|
||||
| secret.existingSecret | string | `""` | Use an existing Secret. Automatic endpoint-anchor injection is disabled because the Secret is opaque; set `localEndpointHost.autoInject=true` only after confirming it contains credentials rather than runtime topology, address, or startup-mode overrides. |
|
||||
| secret.existingSecret | string | `""` | Use existing secret with a credentials. |
|
||||
| secret.rustfs.access_key | string | `"rustfsadmin"` | RustFS Access Key ID |
|
||||
| secret.rustfs.secret_key | string | `"rustfsadmin"` | RustFS Secret Key ID |
|
||||
| service.type | string | `"ClusterIP"` | |
|
||||
@@ -257,7 +202,6 @@ uer. `ClusterIssuer` or `Issuer`. |
|
||||
| serviceAccount.automount | bool | `true` | |
|
||||
| serviceAccount.create | bool | `true` | |
|
||||
| serviceAccount.name | string | `""` | |
|
||||
| startupWaitTimeoutSeconds | int | `300` | Deprecated and ignored; retained for values-file compatibility. |
|
||||
| storageclass.dataStorageSize | string | `"256Mi"` | The storage size for data PVC. |
|
||||
| storageclass.logStorageSize | string | `"256Mi"` | The storage size for logs PVC. |
|
||||
| storageclass.name | string | `"local-path"` | The name for StorageClass. |
|
||||
@@ -347,17 +291,20 @@ Notes:
|
||||
* **Pools are append-only.** The list index determines the StatefulSet name —
|
||||
never remove or reorder entries. Retire a pool with
|
||||
`rc admin decommission` before removing it from the list.
|
||||
* With chart-generated volumes, each pod receives an explicit local endpoint
|
||||
identity. An unavailable peer no longer blocks a pod from reaching RustFS's
|
||||
own startup and quorum checks.
|
||||
* During the expansion rollout, pods restart until every pod of every pool is
|
||||
resolvable — the server refuses to start with unresolvable peers, so expect
|
||||
a few crash/restart cycles before the cluster converges. This is harmless.
|
||||
* After the cluster converges, run `rc admin rebalance start <alias>` to
|
||||
spread existing objects across the new pool.
|
||||
* Pod anti-affinity in pool mode is scoped per pool and **preferred**
|
||||
(soft), not required: two pools can share nodes, and each pool's own pods
|
||||
spread across distinct nodes when capacity allows. Preferred affinity keeps
|
||||
additional pools schedulable when the cluster has fewer nodes than total
|
||||
pods. Single-pool deployments (`pools.enabled=false`) keep the chart's
|
||||
existing required anti-affinity unchanged.
|
||||
spread across distinct nodes when capacity allows. Soft affinity is
|
||||
load-bearing for in-place expansion — with required rules, the
|
||||
not-yet-rolled pods of the existing pool block the new pool's pods from
|
||||
their nodes while the rolled pods crash on the unresolvable (Pending)
|
||||
peers, deadlocking the rollout on any cluster with fewer nodes than the
|
||||
total pod count. Single-pool deployments (`pools.enabled=false`) keep the
|
||||
chart's existing required anti-affinity unchanged.
|
||||
* The PodDisruptionBudget spans all pools: with the default
|
||||
`pdb.maxUnavailable: 1`, at most one pod of the whole cluster may be
|
||||
evicted at a time. This is deliberately conservative — quorum safety
|
||||
@@ -368,8 +315,7 @@ Notes:
|
||||
## Requirement
|
||||
|
||||
* Helm V3
|
||||
* The RustFS image from the same release as the chart. If `image.rustfs.tag`
|
||||
is overridden, that image must support `RUSTFS_LOCAL_ENDPOINT_HOST`.
|
||||
* RustFS >= 1.0.0-alpha.69
|
||||
|
||||
Due to the traefik and ingress has different session sticky/affinity annotations, and rustfs support both those two controller, you should specify parameter `ingress.className` to select the right one which suits for you.
|
||||
|
||||
|
||||
@@ -2,8 +2,8 @@ apiVersion: v2
|
||||
name: rustfs
|
||||
description: RustFS helm chart to deploy RustFS on kubernetes cluster.
|
||||
type: application
|
||||
version: "0.12.0"
|
||||
appVersion: "1.0.0-beta.12"
|
||||
version: "0.11.0"
|
||||
appVersion: "1.0.0-beta.11"
|
||||
home: https://rustfs.com
|
||||
icon: https://media.sys.truenas.net/apps/rustfs/icons/icon.svg
|
||||
maintainers:
|
||||
|
||||
@@ -322,23 +322,21 @@ Render RUSTFS_SERVER_DOMAINS
|
||||
{{- join "," $domains -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/* Render an mTLS probe command
|
||||
{{/* Render probe command for liveness and readiness
|
||||
*/}}
|
||||
|
||||
{{- define "rustfs.probeCommand" -}}
|
||||
{{- $root := .root -}}
|
||||
{{- $endpointPath := .endpointPath -}}
|
||||
{{- $endpoint_port := $root.Values.service.endpoint.port | default 9000 -}}
|
||||
{{- $console_port := $root.Values.service.console.port | default 9001 -}}
|
||||
{{- $endpoint_port := .Values.service.endpoint.port | default 9000 -}}
|
||||
{{- $console_port := .Values.service.console.port | default 9001 -}}
|
||||
{{- $args := "-skf" -}}
|
||||
|
||||
{{- if and $root.Values.mtls.enabled -}}
|
||||
{{- $args = printf "%s --cert %s --key %s" $args $root.Values.mtls.clientCertPath $root.Values.mtls.clientKeyPath -}}
|
||||
{{- if and .Values.mtls.enabled -}}
|
||||
{{- $args = printf "%s --cert %s --key %s" $args .Values.mtls.clientCertPath .Values.mtls.clientKeyPath -}}
|
||||
{{- end -}}
|
||||
- /bin/sh
|
||||
- -c
|
||||
- |
|
||||
curl {{ $args }} https://127.0.0.1:{{ $endpoint_port }}{{ $endpointPath }} && \
|
||||
curl {{ $args }} https://127.0.0.1:{{ $endpoint_port }}/health/ready && \
|
||||
curl {{ $args }} https://127.0.0.1:{{ $console_port }}/rustfs/console/health
|
||||
{{- end -}}
|
||||
|
||||
@@ -352,7 +350,7 @@ livenessProbe:
|
||||
{{- if .Values.mtls.enabled }}
|
||||
exec:
|
||||
command:
|
||||
{{ include "rustfs.probeCommand" (dict "root" . "endpointPath" "/health") | nindent 6 }}
|
||||
{{ include "rustfs.probeCommand" . | nindent 6 }}
|
||||
{{- else }}
|
||||
httpGet:
|
||||
path: /health
|
||||
@@ -371,7 +369,7 @@ readinessProbe:
|
||||
{{- if .Values.mtls.enabled }}
|
||||
exec:
|
||||
command:
|
||||
{{ include "rustfs.probeCommand" (dict "root" . "endpointPath" "/health/ready") | nindent 6 }}
|
||||
{{ include "rustfs.probeCommand" . | nindent 6 }}
|
||||
{{- else }}
|
||||
httpGet:
|
||||
path: /health/ready
|
||||
|
||||
@@ -1,37 +1,12 @@
|
||||
{{- $logDir := .Values.config.rustfs.obs_log_directory }}
|
||||
{{- $logDirEnabled := ne $logDir "" }}
|
||||
{{- $poolsEnabled := and .Values.pools .Values.pools.enabled }}
|
||||
{{- $generatedVolumes := empty .Values.config.rustfs.volumes }}
|
||||
{{- $localEndpointHostAutoInject := empty .Values.secret.existingSecret }}
|
||||
{{- with .Values.localEndpointHost }}
|
||||
{{- if and (hasKey . "autoInject") (ne .autoInject nil) }}
|
||||
{{- if not (kindIs "bool" .autoInject) }}
|
||||
{{- fail "localEndpointHost.autoInject must be a boolean or null" }}
|
||||
{{- end }}
|
||||
{{- $localEndpointHostAutoInject = .autoInject }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- $injectLocalEndpointHost := and $generatedVolumes $localEndpointHostAutoInject }}
|
||||
{{- range $env := .Values.extraEnv -}}
|
||||
{{- $name := default "" $env.name -}}
|
||||
{{- if eq $name "RUSTFS_VOLUMES" -}}
|
||||
{{- $injectLocalEndpointHost = false -}}
|
||||
{{- else if eq $name "RUSTFS_LOCAL_ENDPOINT_HOST" -}}
|
||||
{{- $injectLocalEndpointHost = false -}}
|
||||
{{- else if eq $name "RUSTFS_STARTUP_TOPOLOGY_WAIT_MODE" -}}
|
||||
{{- if hasKey $env "value" -}}
|
||||
{{- $mode := lower (trim (toString $env.value)) -}}
|
||||
{{- if and (ne $mode "") (ne $mode "auto") (ne $mode "orchestrated") -}}
|
||||
{{- $injectLocalEndpointHost = false -}}
|
||||
{{- end -}}
|
||||
{{- else -}}
|
||||
{{- $injectLocalEndpointHost = false -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
{{- if and .Values.mode.distributed.enabled (le (int .Values.replicaCount) 1) -}}
|
||||
{{- fail "Distributed mode requires replicaCount >= 2" -}}
|
||||
{{- end -}}
|
||||
{{- if and .Values.mode.distributed.enabled (le (int .Values.startupWaitTimeoutSeconds) 0) -}}
|
||||
{{- fail "startupWaitTimeoutSeconds must be greater than 0 in distributed mode" -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- if .Values.mode.distributed.enabled }}
|
||||
{{- $pools := include "rustfs.pools" . | fromJsonArray }}
|
||||
@@ -99,10 +74,13 @@ spec:
|
||||
{{- if $.Values.affinity.podAntiAffinity.enabled }}
|
||||
podAntiAffinity:
|
||||
{{- if $poolsEnabled }}
|
||||
{{- /* Pool-scoped and PREFERRED, not required: additional pools
|
||||
must remain schedulable when the cluster has fewer nodes
|
||||
than total pods. The scheduler still spreads them when
|
||||
capacity allows. */}}
|
||||
{{- /* Pool-scoped and PREFERRED, not required: during in-place
|
||||
expansion the not-yet-rolled pods' required rules block new
|
||||
pods from their nodes, while the new pods' DNS must resolve
|
||||
before the roll can proceed - required rules deadlock
|
||||
expansion whenever the cluster has fewer nodes than total
|
||||
pods. Soft anti-affinity converges (the scheduler still
|
||||
spreads when capacity allows). */}}
|
||||
preferredDuringSchedulingIgnoredDuringExecution:
|
||||
- weight: 100
|
||||
podAffinityTerm:
|
||||
@@ -149,6 +127,10 @@ spec:
|
||||
env:
|
||||
- name: DRIVES_PER_NODE
|
||||
value: {{ $drivesPerNode | quote }}
|
||||
- name: ENDPOINT_PORT
|
||||
value: {{ $.Values.service.endpoint.port | quote }}
|
||||
- name: STARTUP_WAIT_TIMEOUT_SECONDS
|
||||
value: {{ $.Values.startupWaitTimeoutSeconds | quote }}
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
@@ -164,6 +146,44 @@ spec:
|
||||
mkdir -p /mnt/rustfs/logs
|
||||
chmod 755 /mnt/rustfs/logs
|
||||
{{- end }}
|
||||
deadline=$(($(date +%s) + STARTUP_WAIT_TIMEOUT_SECONDS))
|
||||
wait_for_peer() {
|
||||
description=$1
|
||||
shift
|
||||
until "$@" >/dev/null 2>&1; do
|
||||
if [ "$(date +%s)" -ge "$deadline" ]; then
|
||||
echo "Timed out after ${STARTUP_WAIT_TIMEOUT_SECONDS}s waiting for ${description}" >&2
|
||||
exit 1
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
}
|
||||
|
||||
echo "Waiting for distributed peer DNS records"
|
||||
{{- range $peerPool := $pools }}
|
||||
{{- range $i := until (int $peerPool.replicaCount) }}
|
||||
wait_for_peer "DNS for {{ $peerPool.fullname }}-{{ $i }}" \
|
||||
nslookup "{{ $peerPool.fullname }}-{{ $i }}.{{ include "rustfs.fullname" $ }}-headless.{{ $.Release.Namespace }}.svc.{{ include "rustfs.clusterDomain" $ }}"
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
ordinal=${HOSTNAME##*-}
|
||||
echo "Waiting for earlier distributed peers to listen on port ${ENDPOINT_PORT}"
|
||||
{{- range $peerPool := $pools }}
|
||||
{{- if lt (int $peerPool.index) (int $pool.index) }}
|
||||
{{- range $i := until (int $peerPool.replicaCount) }}
|
||||
wait_for_peer "{{ $peerPool.fullname }}-{{ $i }}:${ENDPOINT_PORT}" \
|
||||
nc -z -w 2 "{{ $peerPool.fullname }}-{{ $i }}.{{ include "rustfs.fullname" $ }}-headless.{{ $.Release.Namespace }}.svc.{{ include "rustfs.clusterDomain" $ }}" "$ENDPOINT_PORT"
|
||||
{{- end }}
|
||||
{{- else if eq (int $peerPool.index) (int $pool.index) }}
|
||||
i=0
|
||||
while [ "$i" -lt "$ordinal" ]; do
|
||||
wait_for_peer "{{ $peerPool.fullname }}-${i}:${ENDPOINT_PORT}" \
|
||||
nc -z -w 2 "{{ $peerPool.fullname }}-${i}.{{ include "rustfs.fullname" $ }}-headless.{{ $.Release.Namespace }}.svc.{{ include "rustfs.clusterDomain" $ }}" "$ENDPOINT_PORT"
|
||||
i=$((i + 1))
|
||||
done
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
volumeMounts:
|
||||
{{- if gt $drivesPerNode 1 }}
|
||||
{{- range $i := until $drivesPerNode }}
|
||||
@@ -190,19 +210,9 @@ spec:
|
||||
containerPort: {{ $.Values.service.endpoint.port }}
|
||||
- name: console
|
||||
containerPort: {{ $.Values.service.console.port }}
|
||||
{{- if or $injectLocalEndpointHost (not (empty $.Values.extraEnv)) }}
|
||||
{{- with $.Values.extraEnv }}
|
||||
env:
|
||||
{{- if $injectLocalEndpointHost }}
|
||||
- name: RUSTFS_CHART_POD_NAME
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: metadata.name
|
||||
- name: RUSTFS_LOCAL_ENDPOINT_HOST
|
||||
value: {{ printf "$(RUSTFS_CHART_POD_NAME).%s-headless.%s.svc.%s" (include "rustfs.fullname" $) $.Release.Namespace (include "rustfs.clusterDomain" $) | quote }}
|
||||
{{- end }}
|
||||
{{- with $.Values.extraEnv }}
|
||||
{{- toYaml . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
envFrom:
|
||||
- configMapRef:
|
||||
|
||||
+4
-26
@@ -30,22 +30,11 @@ drivesPerNode: null
|
||||
# Provide the DNS root only, without a `svc.` prefix or leading/trailing dots.
|
||||
clusterDomain: cluster.local
|
||||
|
||||
# Deprecated and ignored. Retained so existing values files continue to
|
||||
# render after peer startup coordination moved into RustFS.
|
||||
# RUSTFS_COMPAT_TODO(rustfs-5416): Keep the removed init-gate setting visible to existing values files. Remove after the minimum supported direct-upgrade chart includes localEndpointHost.autoInject and no longer renders peer gates.
|
||||
# Maximum time an init container waits for distributed peers to publish DNS
|
||||
# and for earlier peers to start listening. The wait is shared across all
|
||||
# peers, not applied once per peer.
|
||||
startupWaitTimeoutSeconds: 300
|
||||
|
||||
# Inject a pod-specific endpoint identity for chart-generated distributed
|
||||
# topologies. The default enables injection unless secret.existingSecret is
|
||||
# set, because the chart cannot inspect runtime overrides hidden in that
|
||||
# Secret. Without injection, the server infers a unique matching domain
|
||||
# endpoint from its kernel hostname in Kubernetes auto/orchestrated mode; custom
|
||||
# aliases retain bounded legacy DNS locality when no hostname matches. Configure
|
||||
# RUSTFS_LOCAL_ENDPOINT_HOST explicitly for DNS-free startup. Set true for a
|
||||
# credentials-only existing Secret or false to suppress chart injection.
|
||||
localEndpointHost:
|
||||
autoInject: null
|
||||
|
||||
# This sets the container image more information can be found here: https://kubernetes.io/docs/concepts/containers/images/
|
||||
image:
|
||||
rustfs: # This sets the rustfs image repository and tag.
|
||||
@@ -234,18 +223,7 @@ config:
|
||||
default_key: ""
|
||||
|
||||
|
||||
# Extra environment variables for the RustFS container.
|
||||
# In distributed mode with chart-generated volumes (config.rustfs.volumes is
|
||||
# empty), the chart injects a private Downward API pod name and
|
||||
# RUSTFS_LOCAL_ENDPOINT_HOST to identify the pod's local endpoints. An explicit
|
||||
# RUSTFS_LOCAL_ENDPOINT_HOST or RUSTFS_VOLUMES entry disables automatic anchor
|
||||
# injection. An explicit bounded or fail-fast
|
||||
# RUSTFS_STARTUP_TOPOLOGY_WAIT_MODE, or an unrecognized static value, also keeps
|
||||
# legacy DNS-based locality discovery. Otherwise Kubernetes auto-detection
|
||||
# selects orchestrated startup when RustFS consumes the generated anchor.
|
||||
# With custom volumes, the chart does not inject them and extraEnv may define
|
||||
# the local endpoint identity explicitly.
|
||||
extraEnv: []
|
||||
extraEnv: [] # This is for setting extra environment variables in the rustfs container. It should be a list of key value pairs. For example:
|
||||
# extraEnv:
|
||||
# - name: RUSTFS_ERASURE_SET_DRIVE_COUNT
|
||||
# value: "16"
|
||||
|
||||
+2
-5
@@ -1,9 +1,9 @@
|
||||
%global _enable_debug_packages 0
|
||||
%global _empty_manifest_terminate_build 0
|
||||
%global prerelease beta.12
|
||||
%global prerelease beta.11
|
||||
Name: rustfs
|
||||
Version: 1.0.0
|
||||
Release: beta.12
|
||||
Release: beta.11
|
||||
Summary: High-performance distributed object storage for MinIO alternative
|
||||
|
||||
License: Apache-2.0
|
||||
@@ -58,9 +58,6 @@ install %_builddir/%{name}-%{version}-%{prerelease}/target/%_arch/%_arch-unknown
|
||||
%_bindir/rustfs
|
||||
|
||||
%changelog
|
||||
* Thu Jul 30 2026 overtrue <anzhengchao@gmail.com>
|
||||
- Update RPM package to RustFS 1.0.0-beta.12
|
||||
|
||||
* Thu Jul 23 2026 overtrue <anzhengchao@gmail.com>
|
||||
- Update RPM package to RustFS 1.0.0-beta.11
|
||||
|
||||
|
||||
@@ -20,10 +20,8 @@ use crate::admin::storage_api::error::StorageError;
|
||||
use crate::admin::storage_api::lifecycle::{
|
||||
ManualTransitionCancelCheck, ManualTransitionJobRecord, ManualTransitionJobState, ManualTransitionProgressSink,
|
||||
ManualTransitionQueueSnapshot, ManualTransitionRunOptions, ManualTransitionRunReport, ManualTransitionScopeAdmission,
|
||||
ManualTransitionScopeAdmissionClaim, TransitionOperatorDeleteResult, TransitionOperatorError,
|
||||
claim_manual_transition_scope_admission, delete_manual_transition_scope_admission_if_current,
|
||||
delete_transition_candidate_for_operator, enqueue_transition_for_existing_objects_scoped,
|
||||
finalize_missing_transition_transaction_for_operator, inspect_transition_transaction_for_operator,
|
||||
ManualTransitionScopeAdmissionClaim, claim_manual_transition_scope_admission,
|
||||
delete_manual_transition_scope_admission_if_current, enqueue_transition_for_existing_objects_scoped,
|
||||
load_manual_transition_job_record, load_manual_transition_job_record_with_etag, load_manual_transition_scope_admission,
|
||||
manual_transition_job_lease_expired, manual_transition_queue_snapshot, manual_transition_scope_admission_lease_expired,
|
||||
persist_manual_transition_job_progress, renew_manual_transition_job_lease, request_manual_transition_job_cancel,
|
||||
@@ -35,7 +33,6 @@ use crate::server::{ADMIN_PREFIX, RemoteAddr};
|
||||
use http::{HeaderMap, HeaderValue};
|
||||
use hyper::{Method, StatusCode};
|
||||
use matchit::Params;
|
||||
use rustfs_config::MAX_ADMIN_REQUEST_BODY_SIZE;
|
||||
use rustfs_policy::policy::action::{Action, AdminAction};
|
||||
use rustfs_utils::{
|
||||
MaskedAccessKey,
|
||||
@@ -59,7 +56,6 @@ const MAX_MANUAL_TRANSITION_DURATION_SECONDS: u64 = 3600;
|
||||
const LOG_COMPONENT_ADMIN: &str = "admin";
|
||||
const LOG_SUBSYSTEM_ILM_TRANSITION: &str = "ilm_transition";
|
||||
const EVENT_ADMIN_ILM_TRANSITION_STATE: &str = "admin_ilm_transition_state";
|
||||
const EVENT_ADMIN_ILM_TRANSITION_RECONCILE: &str = "admin_ilm_transition_reconcile";
|
||||
|
||||
static ACTIVE_MANUAL_TRANSITION_SCOPES: OnceLock<Mutex<Vec<ManualTransitionRunScope>>> = OnceLock::new();
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
@@ -222,16 +218,6 @@ pub fn register_ilm_transition_route(r: &mut S3Router<AdminOperation>) -> std::i
|
||||
format!("{ADMIN_PREFIX}/v3/ilm/transition/jobs/{{job_id}}").as_str(),
|
||||
AdminOperation(&ManualTransitionJobCancelHandler {}),
|
||||
)?;
|
||||
r.insert(
|
||||
Method::GET,
|
||||
format!("{ADMIN_PREFIX}/v3/ilm/transition/reconcile/{{transaction_id}}").as_str(),
|
||||
AdminOperation(&TransitionReconcileInspectHandler {}),
|
||||
)?;
|
||||
r.insert(
|
||||
Method::POST,
|
||||
format!("{ADMIN_PREFIX}/v3/ilm/transition/reconcile/{{transaction_id}}").as_str(),
|
||||
AdminOperation(&TransitionReconcileApplyHandler {}),
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -403,10 +389,6 @@ fn log_manual_transition_completed(
|
||||
}
|
||||
|
||||
async fn authorize_manual_transition_request(req: &S3Request<Body>) -> S3Result<String> {
|
||||
authorize_transition_admin_request(req, AdminAction::SetTierAction).await
|
||||
}
|
||||
|
||||
async fn authorize_transition_admin_request(req: &S3Request<Body>, action: AdminAction) -> S3Result<String> {
|
||||
let Some(input_cred) = req.credentials.as_ref() else {
|
||||
return Err(s3_error!(InvalidRequest, "authentication required"));
|
||||
};
|
||||
@@ -419,122 +401,19 @@ async fn authorize_transition_admin_request(req: &S3Request<Body>, action: Admin
|
||||
.get::<Option<RemoteAddr>>()
|
||||
.and_then(|opt| opt.map(|addr| addr.0));
|
||||
|
||||
validate_admin_request(&req.headers, &cred, owner, false, vec![Action::AdminAction(action)], remote_addr).await?;
|
||||
validate_admin_request(
|
||||
&req.headers,
|
||||
&cred,
|
||||
owner,
|
||||
false,
|
||||
vec![Action::AdminAction(AdminAction::SetTierAction)],
|
||||
remote_addr,
|
||||
)
|
||||
.await?;
|
||||
|
||||
Ok(actor)
|
||||
}
|
||||
|
||||
fn transition_transaction_id_from_params(params: &Params<'_, '_>) -> S3Result<Uuid> {
|
||||
Uuid::parse_str(params.get("transaction_id").unwrap_or(""))
|
||||
.map_err(|_| s3_error!(InvalidArgument, "invalid transition transaction id"))
|
||||
}
|
||||
|
||||
fn map_transition_operator_error(err: TransitionOperatorError) -> S3Error {
|
||||
match err {
|
||||
TransitionOperatorError::NotFound => s3_error!(NoSuchKey, "transition transaction not found"),
|
||||
TransitionOperatorError::NotExpired => {
|
||||
s3_error!(OperationAborted, "transition transaction is still inside its active ownership window")
|
||||
}
|
||||
TransitionOperatorError::InvalidState(_) => {
|
||||
s3_error!(OperationAborted, "transition transaction is not eligible for operator reconciliation")
|
||||
}
|
||||
TransitionOperatorError::RemoteVersionRequired => {
|
||||
s3_error!(InvalidArgument, "an exact non-empty remote version is required")
|
||||
}
|
||||
TransitionOperatorError::CandidateNotMissing(_) => {
|
||||
s3_error!(OperationAborted, "remote candidate is not proven missing")
|
||||
}
|
||||
TransitionOperatorError::CandidateVersionMismatch { .. } => {
|
||||
s3_error!(OperationAborted, "remote candidate version does not match requested exact version")
|
||||
}
|
||||
TransitionOperatorError::Store(_) | TransitionOperatorError::Remote(_) => {
|
||||
s3_error!(InternalError, "transition reconciliation failed")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
enum TransitionReconcileAction {
|
||||
DeleteCandidate,
|
||||
FinalizeMissing,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct TransitionReconcileRequest {
|
||||
action: TransitionReconcileAction,
|
||||
confirm: bool,
|
||||
#[serde(default)]
|
||||
remote_version_id: Option<String>,
|
||||
}
|
||||
|
||||
enum ValidatedTransitionReconcileAction<'a> {
|
||||
DeleteCandidate(&'a str),
|
||||
FinalizeMissing,
|
||||
}
|
||||
|
||||
fn validate_transition_reconcile_request(
|
||||
request: &TransitionReconcileRequest,
|
||||
) -> S3Result<ValidatedTransitionReconcileAction<'_>> {
|
||||
if !request.confirm {
|
||||
return Err(s3_error!(
|
||||
InvalidRequest,
|
||||
"transition reconciliation requires confirm=true; use GET to inspect without changes"
|
||||
));
|
||||
}
|
||||
match request.action {
|
||||
TransitionReconcileAction::DeleteCandidate => request
|
||||
.remote_version_id
|
||||
.as_deref()
|
||||
.filter(|version_id| !version_id.is_empty())
|
||||
.map(ValidatedTransitionReconcileAction::DeleteCandidate)
|
||||
.ok_or_else(|| s3_error!(InvalidArgument, "delete_candidate requires remote_version_id")),
|
||||
TransitionReconcileAction::FinalizeMissing if request.remote_version_id.is_none() => {
|
||||
Ok(ValidatedTransitionReconcileAction::FinalizeMissing)
|
||||
}
|
||||
TransitionReconcileAction::FinalizeMissing => {
|
||||
Err(s3_error!(InvalidArgument, "finalize_missing must not include remote_version_id"))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct TransitionCandidateDeleteResponse {
|
||||
outcome: &'static str,
|
||||
result: TransitionOperatorDeleteResult,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct TransitionFinalizeMissingResponse {
|
||||
outcome: &'static str,
|
||||
journal_retained: bool,
|
||||
transaction_id: Uuid,
|
||||
}
|
||||
|
||||
fn log_transition_reconcile_applied(
|
||||
transaction_id: Uuid,
|
||||
action: &str,
|
||||
outcome: &str,
|
||||
request_id: &str,
|
||||
actor: &str,
|
||||
remote_addr: &str,
|
||||
) {
|
||||
info!(
|
||||
event = EVENT_ADMIN_ILM_TRANSITION_RECONCILE,
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_ILM_TRANSITION,
|
||||
operation = "transition_operator_reconcile",
|
||||
transaction_id = %transaction_id,
|
||||
action,
|
||||
outcome,
|
||||
request_id = %request_id,
|
||||
actor = %actor,
|
||||
remote_addr = %remote_addr,
|
||||
"admin transition reconciliation applied"
|
||||
);
|
||||
}
|
||||
|
||||
fn response_state(report: &ManualTransitionRunReport) -> &'static str {
|
||||
if report.was_truncated() || report.has_partial_enqueue() || report.tier_failure > 0 || report.transition_failed > 0 {
|
||||
"partial"
|
||||
@@ -1023,81 +902,6 @@ impl Operation for ManualTransitionJobCancelHandler {
|
||||
}
|
||||
}
|
||||
|
||||
pub struct TransitionReconcileInspectHandler {}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl Operation for TransitionReconcileInspectHandler {
|
||||
async fn call(&self, req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
|
||||
authorize_transition_admin_request(&req, AdminAction::ListTierAction).await?;
|
||||
let transaction_id = transition_transaction_id_from_params(¶ms)?;
|
||||
let Some(store) = object_store_from_extensions(&req.extensions) else {
|
||||
return Err(s3_error!(InternalError, "object store is not initialized"));
|
||||
};
|
||||
let status = inspect_transition_transaction_for_operator(store, transaction_id)
|
||||
.await
|
||||
.map_err(map_transition_operator_error)?;
|
||||
json_response(&status, StatusCode::OK)
|
||||
}
|
||||
}
|
||||
|
||||
pub struct TransitionReconcileApplyHandler {}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl Operation for TransitionReconcileApplyHandler {
|
||||
async fn call(&self, req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
|
||||
let request_id = admin_request_id(&req.headers).unwrap_or_default().to_string();
|
||||
let remote_addr = admin_remote_addr(&req).unwrap_or_default();
|
||||
let actor = authorize_transition_admin_request(&req, AdminAction::SetTierAction).await?;
|
||||
let transaction_id = transition_transaction_id_from_params(¶ms)?;
|
||||
let Some(store) = object_store_from_extensions(&req.extensions) else {
|
||||
return Err(s3_error!(InternalError, "object store is not initialized"));
|
||||
};
|
||||
let mut input = req.input;
|
||||
let body = input
|
||||
.store_all_limited(MAX_ADMIN_REQUEST_BODY_SIZE)
|
||||
.await
|
||||
.map_err(|_| s3_error!(InvalidRequest, "transition reconciliation body is too large or unreadable"))?;
|
||||
let request: TransitionReconcileRequest = serde_json::from_slice(&body)
|
||||
.map_err(|_| s3_error!(InvalidRequest, "transition reconciliation request must be valid JSON"))?;
|
||||
|
||||
match validate_transition_reconcile_request(&request)? {
|
||||
ValidatedTransitionReconcileAction::DeleteCandidate(remote_version_id) => {
|
||||
let result = delete_transition_candidate_for_operator(store, transaction_id, remote_version_id)
|
||||
.await
|
||||
.map_err(map_transition_operator_error)?;
|
||||
let outcome = if result.journal_observed_after_delete {
|
||||
"exact_delete_completed_journal_observed"
|
||||
} else {
|
||||
"exact_delete_completed_journal_already_finalized"
|
||||
};
|
||||
log_transition_reconcile_applied(transaction_id, "delete_candidate", outcome, &request_id, &actor, &remote_addr);
|
||||
json_response(&TransitionCandidateDeleteResponse { outcome, result }, StatusCode::OK)
|
||||
}
|
||||
ValidatedTransitionReconcileAction::FinalizeMissing => {
|
||||
finalize_missing_transition_transaction_for_operator(store, transaction_id)
|
||||
.await
|
||||
.map_err(map_transition_operator_error)?;
|
||||
log_transition_reconcile_applied(
|
||||
transaction_id,
|
||||
"finalize_missing",
|
||||
"journal_deleted_after_missing_probe",
|
||||
&request_id,
|
||||
&actor,
|
||||
&remote_addr,
|
||||
);
|
||||
json_response(
|
||||
&TransitionFinalizeMissingResponse {
|
||||
outcome: "journal_finalized",
|
||||
journal_retained: false,
|
||||
transaction_id,
|
||||
},
|
||||
StatusCode::OK,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -1127,65 +931,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transition_reconcile_request_is_explicit_and_fail_closed() {
|
||||
let unconfirmed: TransitionReconcileRequest =
|
||||
serde_json::from_slice(br#"{"action":"delete_candidate","confirm":false,"remote_version_id":"v1"}"#)
|
||||
.expect("request should decode");
|
||||
assert!(validate_transition_reconcile_request(&unconfirmed).is_err());
|
||||
|
||||
let missing_version: TransitionReconcileRequest =
|
||||
serde_json::from_slice(br#"{"action":"delete_candidate","confirm":true}"#).expect("request should decode");
|
||||
assert!(validate_transition_reconcile_request(&missing_version).is_err());
|
||||
|
||||
let unsafe_finalize: TransitionReconcileRequest =
|
||||
serde_json::from_slice(br#"{"action":"finalize_missing","confirm":true,"remote_version_id":"v1"}"#)
|
||||
.expect("request should decode");
|
||||
assert!(validate_transition_reconcile_request(&unsafe_finalize).is_err());
|
||||
|
||||
let delete: TransitionReconcileRequest =
|
||||
serde_json::from_slice(br#"{"action":"delete_candidate","confirm":true,"remote_version_id":"opaque-v1"}"#)
|
||||
.expect("request should decode");
|
||||
assert!(matches!(
|
||||
validate_transition_reconcile_request(&delete),
|
||||
Ok(ValidatedTransitionReconcileAction::DeleteCandidate("opaque-v1"))
|
||||
));
|
||||
|
||||
let finalize: TransitionReconcileRequest =
|
||||
serde_json::from_slice(br#"{"action":"finalize_missing","confirm":true}"#).expect("request should decode");
|
||||
assert!(matches!(
|
||||
validate_transition_reconcile_request(&finalize),
|
||||
Ok(ValidatedTransitionReconcileAction::FinalizeMissing)
|
||||
));
|
||||
|
||||
assert!(
|
||||
serde_json::from_slice::<TransitionReconcileRequest>(
|
||||
br#"{"action":"finalize_missing","confirm":true,"unexpected":true}"#
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transition_reconcile_routes_use_read_and_write_tier_actions() {
|
||||
let src = include_str!("ilm_transition.rs");
|
||||
let inspect = src
|
||||
.split("impl Operation for TransitionReconcileInspectHandler")
|
||||
.nth(1)
|
||||
.and_then(|block| block.split("impl Operation for TransitionReconcileApplyHandler").next())
|
||||
.expect("inspect handler block");
|
||||
assert!(inspect.contains("AdminAction::ListTierAction"));
|
||||
assert!(!inspect.contains("AdminAction::SetTierAction"));
|
||||
|
||||
let apply = src
|
||||
.split("impl Operation for TransitionReconcileApplyHandler")
|
||||
.nth(1)
|
||||
.and_then(|block| block.split("#[cfg(test)]").next())
|
||||
.expect("apply handler block");
|
||||
assert!(apply.contains("AdminAction::SetTierAction"));
|
||||
assert!(!apply.contains("AdminAction::ListTierAction"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn manual_transition_query_defaults_to_bounded_run() {
|
||||
let (bucket, options, run_mode) =
|
||||
|
||||
@@ -148,7 +148,7 @@ impl Operation for ListCannedPolicies {
|
||||
return Err(s3_error!(InternalError, "iam is not initialized"));
|
||||
};
|
||||
|
||||
let policies = iam_store.list_policies(&query.bucket).await.map_err(|e| {
|
||||
let policies = iam_store.list_polices(&query.bucket).await.map_err(|e| {
|
||||
warn!(
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_POLICY,
|
||||
|
||||
@@ -1044,7 +1044,6 @@ mod tests {
|
||||
op: MrfOpKind::Object,
|
||||
delete_marker_version_id: None,
|
||||
delete_marker: false,
|
||||
replica: false,
|
||||
delete_marker_mtime: None,
|
||||
},
|
||||
MrfReplicateEntry {
|
||||
@@ -1056,7 +1055,6 @@ mod tests {
|
||||
op: MrfOpKind::Object,
|
||||
delete_marker_version_id: None,
|
||||
delete_marker: false,
|
||||
replica: false,
|
||||
delete_marker_mtime: None,
|
||||
},
|
||||
],
|
||||
|
||||
@@ -7205,7 +7205,7 @@ async fn start_site_bucket_resync(bucket: &str, target_arn: &str, resync_id: &st
|
||||
return bucket_status;
|
||||
};
|
||||
let _targets_guard = lock_bucket_targets_metadata(bucket).await;
|
||||
let _transaction_guard = match metadata_sys::acquire_bucket_metadata_transaction_lock(bucket).await {
|
||||
let _transaction_guard = match metadata_sys::acquire_bucket_targets_transaction_lock(bucket).await {
|
||||
Ok(guard) => guard,
|
||||
Err(_) => {
|
||||
bucket_status.status = "failed".to_string();
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user