mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-15 09:33:13 +00:00
Compare commits
20 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| ab387b849f | |||
| 6106cd3772 | |||
| 3b9c67e79b | |||
| d36166ffb5 | |||
| 963a107b33 | |||
| 02b4e082e8 | |||
| b4133d69e6 | |||
| 134081b27b | |||
| 7217cccc91 | |||
| dafc922e72 | |||
| 6fcf0d250e | |||
| 65e55c0f8e | |||
| 8c75a3834a | |||
| f96346124e | |||
| a7de957eb8 | |||
| c2e23411e8 | |||
| 4a234c0fe3 | |||
| 1b1b217826 | |||
| 7e8b500420 | |||
| e342457830 |
Generated
+117
-118
@@ -284,7 +284,7 @@ dependencies = [
|
||||
"serde_json",
|
||||
"strum 0.27.2",
|
||||
"strum_macros 0.27.2",
|
||||
"thiserror 2.0.19",
|
||||
"thiserror 2.0.20",
|
||||
"uuid",
|
||||
]
|
||||
|
||||
@@ -572,7 +572,7 @@ dependencies = [
|
||||
"nom 7.1.3",
|
||||
"num-traits",
|
||||
"rusticata-macros",
|
||||
"thiserror 2.0.19",
|
||||
"thiserror 2.0.20",
|
||||
"time",
|
||||
]
|
||||
|
||||
@@ -726,7 +726,7 @@ dependencies = [
|
||||
"serde_json",
|
||||
"serde_nanos",
|
||||
"serde_repr",
|
||||
"thiserror 2.0.19",
|
||||
"thiserror 2.0.20",
|
||||
"time",
|
||||
"tokio",
|
||||
"tokio-rustls",
|
||||
@@ -773,9 +773,9 @@ checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de"
|
||||
|
||||
[[package]]
|
||||
name = "async-trait"
|
||||
version = "0.1.91"
|
||||
version = "0.1.92"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ae36dc4177970ef04fde5178d3e2429882def40e57a451f919c098f72baa6cec"
|
||||
checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -812,7 +812,7 @@ dependencies = [
|
||||
"crc32fast",
|
||||
"futures-lite",
|
||||
"pin-project",
|
||||
"thiserror 2.0.19",
|
||||
"thiserror 2.0.20",
|
||||
"tokio",
|
||||
"tokio-util",
|
||||
]
|
||||
@@ -903,9 +903,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-lc-rs"
|
||||
version = "1.17.3"
|
||||
version = "1.18.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "00bdb5da18dac48ca2cc7cd4a98e533e8635a58e2361d13a1a4ee3888e0d72f1"
|
||||
checksum = "ce2b2dcc879c3bae0d371e77c99f2238400ef24ec001394befa67b6e543add9e"
|
||||
dependencies = [
|
||||
"aws-lc-sys",
|
||||
"untrusted 0.7.1",
|
||||
@@ -914,9 +914,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-lc-sys"
|
||||
version = "0.43.0"
|
||||
version = "0.44.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "43103168cc76fe62678a375e722fc9cb3a0146159ac5828bc4f0dfd755c2224c"
|
||||
checksum = "f09fae7be8bb3174e05c6afdb34199e6dc0c7c04ba9fa237b1967adfbde27483"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"cmake",
|
||||
@@ -1253,7 +1253,7 @@ dependencies = [
|
||||
"regex-lite",
|
||||
"roxmltree",
|
||||
"serde_json",
|
||||
"thiserror 2.0.19",
|
||||
"thiserror 2.0.20",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1792,7 +1792,7 @@ dependencies = [
|
||||
"semver",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"thiserror 2.0.19",
|
||||
"thiserror 2.0.20",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1840,9 +1840,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "cc"
|
||||
version = "1.4.1"
|
||||
version = "1.4.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9066c49992464636f92905fa096ec58baaa4d57ec19a5c096c68d3e25ef3d136"
|
||||
checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e"
|
||||
dependencies = [
|
||||
"find-msvc-tools",
|
||||
"jobserver",
|
||||
@@ -3415,7 +3415,7 @@ version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e"
|
||||
dependencies = [
|
||||
"thiserror 2.0.19",
|
||||
"thiserror 2.0.20",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -4033,7 +4033,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -4484,7 +4484,7 @@ dependencies = [
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sha2 0.11.0",
|
||||
"thiserror 2.0.19",
|
||||
"thiserror 2.0.20",
|
||||
"time",
|
||||
"tokio",
|
||||
"url",
|
||||
@@ -4505,7 +4505,7 @@ dependencies = [
|
||||
"rand 0.10.2",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"thiserror 2.0.19",
|
||||
"thiserror 2.0.20",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
@@ -4538,7 +4538,7 @@ dependencies = [
|
||||
"rustc_version",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"thiserror 2.0.19",
|
||||
"thiserror 2.0.20",
|
||||
"tokio",
|
||||
"tokio-stream",
|
||||
"tonic",
|
||||
@@ -4645,7 +4645,7 @@ dependencies = [
|
||||
"serde_json",
|
||||
"serde_with",
|
||||
"sha2 0.11.0",
|
||||
"thiserror 2.0.19",
|
||||
"thiserror 2.0.20",
|
||||
"tokio",
|
||||
"tokio-stream",
|
||||
"tracing",
|
||||
@@ -4677,7 +4677,7 @@ dependencies = [
|
||||
"serde",
|
||||
"serde_json",
|
||||
"serde_with",
|
||||
"thiserror 2.0.19",
|
||||
"thiserror 2.0.20",
|
||||
"time",
|
||||
"url",
|
||||
]
|
||||
@@ -4894,7 +4894,7 @@ dependencies = [
|
||||
"ipnet",
|
||||
"jni",
|
||||
"rand 0.10.2",
|
||||
"thiserror 2.0.19",
|
||||
"thiserror 2.0.20",
|
||||
"tinyvec",
|
||||
"tokio",
|
||||
"tracing",
|
||||
@@ -4915,7 +4915,7 @@ dependencies = [
|
||||
"prefix-trie",
|
||||
"rand 0.10.2",
|
||||
"ring",
|
||||
"thiserror 2.0.19",
|
||||
"thiserror 2.0.20",
|
||||
"tinyvec",
|
||||
"tracing",
|
||||
"url",
|
||||
@@ -4942,7 +4942,7 @@ dependencies = [
|
||||
"resolv-conf",
|
||||
"smallvec",
|
||||
"system-configuration",
|
||||
"thiserror 2.0.19",
|
||||
"thiserror 2.0.20",
|
||||
"tokio",
|
||||
"tracing",
|
||||
]
|
||||
@@ -5479,7 +5479,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46"
|
||||
dependencies = [
|
||||
"hermit-abi",
|
||||
"libc",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -5601,7 +5601,7 @@ dependencies = [
|
||||
"jni-sys",
|
||||
"log",
|
||||
"simd_cesu8",
|
||||
"thiserror 2.0.19",
|
||||
"thiserror 2.0.20",
|
||||
"walkdir",
|
||||
"windows-link",
|
||||
]
|
||||
@@ -5650,9 +5650,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "js-sys"
|
||||
version = "0.3.103"
|
||||
version = "0.3.104"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102"
|
||||
checksum = "0e0c1080212aad755ea003d18543e8768dd432c48819efd73a7bf1e39b7a5a3a"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"futures-util",
|
||||
@@ -5963,7 +5963,7 @@ dependencies = [
|
||||
"once_cell",
|
||||
"serde",
|
||||
"sha2 0.10.9",
|
||||
"thiserror 2.0.19",
|
||||
"thiserror 2.0.20",
|
||||
"uuid",
|
||||
]
|
||||
|
||||
@@ -5989,7 +5989,7 @@ dependencies = [
|
||||
"rustls",
|
||||
"slog",
|
||||
"slog-stdlog",
|
||||
"thiserror 2.0.19",
|
||||
"thiserror 2.0.20",
|
||||
"tokio",
|
||||
"tokio-rustls",
|
||||
"tokio-util",
|
||||
@@ -6104,7 +6104,7 @@ version = "0.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bb4bdc8b0ce69932332cf76d24af69c3a155242af95c226b2ab6c2e371ed1149"
|
||||
dependencies = [
|
||||
"thiserror 2.0.19",
|
||||
"thiserror 2.0.20",
|
||||
"zerocopy",
|
||||
"zerocopy-derive",
|
||||
]
|
||||
@@ -6452,7 +6452,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3ff7ae19c74aba9e0ed6e4071cd52aa364e020076fa3cc6ef17e43662f756f3c"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"thiserror 2.0.19",
|
||||
"thiserror 2.0.20",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -6482,7 +6482,7 @@ dependencies = [
|
||||
"quote",
|
||||
"syn 2.0.119",
|
||||
"termcolor",
|
||||
"thiserror 2.0.19",
|
||||
"thiserror 2.0.20",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -6506,7 +6506,7 @@ dependencies = [
|
||||
"rustls",
|
||||
"serde",
|
||||
"socket2",
|
||||
"thiserror 2.0.19",
|
||||
"thiserror 2.0.20",
|
||||
"tokio",
|
||||
"tokio-rustls",
|
||||
"tokio-util",
|
||||
@@ -6539,7 +6539,7 @@ dependencies = [
|
||||
"serde_json",
|
||||
"sha1 0.10.7",
|
||||
"sha2 0.10.9",
|
||||
"thiserror 2.0.19",
|
||||
"thiserror 2.0.20",
|
||||
"uuid",
|
||||
]
|
||||
|
||||
@@ -6975,7 +6975,7 @@ dependencies = [
|
||||
"itertools 0.14.0",
|
||||
"parking_lot",
|
||||
"percent-encoding",
|
||||
"thiserror 2.0.19",
|
||||
"thiserror 2.0.20",
|
||||
"tokio",
|
||||
"tracing",
|
||||
"url",
|
||||
@@ -7062,7 +7062,7 @@ dependencies = [
|
||||
"futures-sink",
|
||||
"js-sys",
|
||||
"pin-project-lite",
|
||||
"thiserror 2.0.19",
|
||||
"thiserror 2.0.20",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
@@ -7106,7 +7106,7 @@ dependencies = [
|
||||
"opentelemetry_sdk",
|
||||
"prost 0.14.4",
|
||||
"reqwest",
|
||||
"thiserror 2.0.19",
|
||||
"thiserror 2.0.20",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -7150,7 +7150,7 @@ dependencies = [
|
||||
"percent-encoding",
|
||||
"portable-atomic",
|
||||
"rand 0.9.5",
|
||||
"thiserror 2.0.19",
|
||||
"thiserror 2.0.20",
|
||||
"tokio",
|
||||
"tokio-stream",
|
||||
]
|
||||
@@ -7204,7 +7204,7 @@ dependencies = [
|
||||
"rc2",
|
||||
"sha1 0.10.7",
|
||||
"sha2 0.10.9",
|
||||
"thiserror 2.0.19",
|
||||
"thiserror 2.0.20",
|
||||
"x509-parser",
|
||||
]
|
||||
|
||||
@@ -7297,7 +7297,7 @@ dependencies = [
|
||||
"log",
|
||||
"rand 0.10.2",
|
||||
"sha2 0.11.0",
|
||||
"thiserror 2.0.19",
|
||||
"thiserror 2.0.20",
|
||||
"tokio",
|
||||
"windows",
|
||||
"windows-strings",
|
||||
@@ -7425,7 +7425,7 @@ checksum = "97f6fccfd2d9d2df765ca23ff85fe5cc437fb0e6d3e164e4d3cbe09d14780c93"
|
||||
dependencies = [
|
||||
"arrayvec",
|
||||
"bitflags 2.13.1",
|
||||
"thiserror 2.0.19",
|
||||
"thiserror 2.0.20",
|
||||
"zerocopy",
|
||||
"zerocopy-derive",
|
||||
]
|
||||
@@ -7925,7 +7925,7 @@ dependencies = [
|
||||
"lazy_static",
|
||||
"memchr",
|
||||
"parking_lot",
|
||||
"thiserror 2.0.19",
|
||||
"thiserror 2.0.20",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -7974,7 +7974,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf"
|
||||
dependencies = [
|
||||
"heck",
|
||||
"itertools 0.14.0",
|
||||
"itertools 0.10.5",
|
||||
"log",
|
||||
"multimap",
|
||||
"once_cell",
|
||||
@@ -7994,7 +7994,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042"
|
||||
dependencies = [
|
||||
"heck",
|
||||
"itertools 0.14.0",
|
||||
"itertools 0.10.5",
|
||||
"log",
|
||||
"multimap",
|
||||
"petgraph 0.8.3",
|
||||
@@ -8015,7 +8015,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"itertools 0.14.0",
|
||||
"itertools 0.10.5",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.119",
|
||||
@@ -8028,7 +8028,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"itertools 0.14.0",
|
||||
"itertools 0.10.5",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.119",
|
||||
@@ -8147,7 +8147,7 @@ dependencies = [
|
||||
"spin 0.12.2",
|
||||
"symbolic-demangle",
|
||||
"tempfile",
|
||||
"thiserror 2.0.19",
|
||||
"thiserror 2.0.20",
|
||||
"url",
|
||||
"uuid",
|
||||
]
|
||||
@@ -8205,7 +8205,7 @@ dependencies = [
|
||||
"rustc-hash",
|
||||
"rustls",
|
||||
"socket2",
|
||||
"thiserror 2.0.19",
|
||||
"thiserror 2.0.20",
|
||||
"tokio",
|
||||
"tracing",
|
||||
"web-time",
|
||||
@@ -8228,7 +8228,7 @@ dependencies = [
|
||||
"rustls",
|
||||
"rustls-pki-types",
|
||||
"slab",
|
||||
"thiserror 2.0.19",
|
||||
"thiserror 2.0.20",
|
||||
"tinyvec",
|
||||
"tracing",
|
||||
"web-time",
|
||||
@@ -8245,7 +8245,7 @@ dependencies = [
|
||||
"once_cell",
|
||||
"socket2",
|
||||
"tracing",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -8400,7 +8400,7 @@ checksum = "5dc94ed8e3de45f6d8d052869d48c0dbeebcaa7a6c345ec7f0f917e10347428e"
|
||||
dependencies = [
|
||||
"clocksource",
|
||||
"parking_lot",
|
||||
"thiserror 2.0.19",
|
||||
"thiserror 2.0.20",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -8540,7 +8540,7 @@ checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac"
|
||||
dependencies = [
|
||||
"getrandom 0.2.17",
|
||||
"libredox",
|
||||
"thiserror 2.0.19",
|
||||
"thiserror 2.0.20",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -8679,7 +8679,7 @@ dependencies = [
|
||||
"http 1.5.0",
|
||||
"reqwest",
|
||||
"serde",
|
||||
"thiserror 2.0.19",
|
||||
"thiserror 2.0.20",
|
||||
"tower-service",
|
||||
]
|
||||
|
||||
@@ -8809,7 +8809,7 @@ dependencies = [
|
||||
"rustls-native-certs",
|
||||
"rustls-pki-types",
|
||||
"rustls-webpki",
|
||||
"thiserror 2.0.19",
|
||||
"thiserror 2.0.20",
|
||||
"tokio",
|
||||
"tokio-rustls",
|
||||
]
|
||||
@@ -8842,7 +8842,7 @@ dependencies = [
|
||||
"rustls-native-certs",
|
||||
"rustls-pki-types",
|
||||
"rustls-webpki",
|
||||
"thiserror 2.0.19",
|
||||
"thiserror 2.0.20",
|
||||
"tokio",
|
||||
"tokio-rustls",
|
||||
"tokio-util",
|
||||
@@ -8913,7 +8913,7 @@ dependencies = [
|
||||
"ssh-encoding",
|
||||
"ssh-key",
|
||||
"subtle",
|
||||
"thiserror 2.0.19",
|
||||
"thiserror 2.0.20",
|
||||
"tokio",
|
||||
"typenum",
|
||||
"universal-hash",
|
||||
@@ -8946,7 +8946,7 @@ dependencies = [
|
||||
"log",
|
||||
"serde",
|
||||
"serde_bytes",
|
||||
"thiserror 2.0.19",
|
||||
"thiserror 2.0.20",
|
||||
"tokio",
|
||||
"tokio-util",
|
||||
"wasm-bindgen-futures",
|
||||
@@ -9139,7 +9139,7 @@ dependencies = [
|
||||
"sysinfo",
|
||||
"temp-env",
|
||||
"tempfile",
|
||||
"thiserror 2.0.19",
|
||||
"thiserror 2.0.20",
|
||||
"time",
|
||||
"tokio",
|
||||
"tokio-rustls",
|
||||
@@ -9176,7 +9176,7 @@ dependencies = [
|
||||
"serde",
|
||||
"serde_json",
|
||||
"temp-env",
|
||||
"thiserror 2.0.19",
|
||||
"thiserror 2.0.20",
|
||||
"tokio",
|
||||
"tracing",
|
||||
"url",
|
||||
@@ -9270,7 +9270,7 @@ dependencies = [
|
||||
"serde_json",
|
||||
"sha2 0.11.0",
|
||||
"test-case",
|
||||
"thiserror 2.0.19",
|
||||
"thiserror 2.0.20",
|
||||
"time",
|
||||
]
|
||||
|
||||
@@ -9388,7 +9388,7 @@ dependencies = [
|
||||
"smallvec",
|
||||
"temp-env",
|
||||
"tempfile",
|
||||
"thiserror 2.0.19",
|
||||
"thiserror 2.0.20",
|
||||
"time",
|
||||
"tokio",
|
||||
"tokio-util",
|
||||
@@ -9430,7 +9430,7 @@ dependencies = [
|
||||
"hotpath",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"thiserror 2.0.19",
|
||||
"thiserror 2.0.20",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -9452,7 +9452,7 @@ dependencies = [
|
||||
"serde",
|
||||
"serde_json",
|
||||
"tempfile",
|
||||
"thiserror 2.0.19",
|
||||
"thiserror 2.0.20",
|
||||
"time",
|
||||
"tokio",
|
||||
"tracing",
|
||||
@@ -9483,7 +9483,7 @@ dependencies = [
|
||||
"serial_test",
|
||||
"temp-env",
|
||||
"tempfile",
|
||||
"thiserror 2.0.19",
|
||||
"thiserror 2.0.20",
|
||||
"tokio",
|
||||
"tokio-util",
|
||||
"tracing",
|
||||
@@ -9521,7 +9521,7 @@ dependencies = [
|
||||
"serial_test",
|
||||
"temp-env",
|
||||
"tempfile",
|
||||
"thiserror 2.0.19",
|
||||
"thiserror 2.0.20",
|
||||
"time",
|
||||
"tokio",
|
||||
"tokio-util",
|
||||
@@ -9537,7 +9537,7 @@ dependencies = [
|
||||
"hotpath",
|
||||
"memmap2",
|
||||
"rustfs-io-metrics",
|
||||
"thiserror 2.0.19",
|
||||
"thiserror 2.0.20",
|
||||
"tokio",
|
||||
"tracing",
|
||||
]
|
||||
@@ -9555,7 +9555,7 @@ dependencies = [
|
||||
"rustfs-s3-ops",
|
||||
"rustfs-utils",
|
||||
"sysinfo",
|
||||
"thiserror 2.0.19",
|
||||
"thiserror 2.0.20",
|
||||
"tokio",
|
||||
"tracing",
|
||||
]
|
||||
@@ -9579,7 +9579,7 @@ dependencies = [
|
||||
"rustls-native-certs",
|
||||
"sha2 0.11.0",
|
||||
"socket2",
|
||||
"thiserror 2.0.19",
|
||||
"thiserror 2.0.20",
|
||||
"tracing",
|
||||
"twox-hash",
|
||||
"webpki-roots 1.0.9",
|
||||
@@ -9627,7 +9627,7 @@ dependencies = [
|
||||
"serde",
|
||||
"serde_json",
|
||||
"temp-env",
|
||||
"thiserror 2.0.19",
|
||||
"thiserror 2.0.20",
|
||||
"time",
|
||||
"tokio",
|
||||
"tower",
|
||||
@@ -9672,7 +9672,7 @@ dependencies = [
|
||||
"subtle",
|
||||
"temp-env",
|
||||
"tempfile",
|
||||
"thiserror 2.0.19",
|
||||
"thiserror 2.0.20",
|
||||
"tokio",
|
||||
"tokio-util",
|
||||
"tracing",
|
||||
@@ -9722,7 +9722,7 @@ dependencies = [
|
||||
"serde_json",
|
||||
"smallvec",
|
||||
"smartstring",
|
||||
"thiserror 2.0.19",
|
||||
"thiserror 2.0.20",
|
||||
"tokio",
|
||||
"tonic",
|
||||
"tracing",
|
||||
@@ -9742,7 +9742,7 @@ dependencies = [
|
||||
"sha2 0.11.0",
|
||||
"tar",
|
||||
"tempfile",
|
||||
"thiserror 2.0.19",
|
||||
"thiserror 2.0.20",
|
||||
"walkdir",
|
||||
"zip",
|
||||
"zstd",
|
||||
@@ -9790,7 +9790,7 @@ dependencies = [
|
||||
"serde_json",
|
||||
"starshard",
|
||||
"tempfile",
|
||||
"thiserror 2.0.19",
|
||||
"thiserror 2.0.20",
|
||||
"tokio",
|
||||
"tokio-util",
|
||||
"tracing",
|
||||
@@ -9830,7 +9830,7 @@ dependencies = [
|
||||
"moka",
|
||||
"starshard",
|
||||
"sysinfo",
|
||||
"thiserror 2.0.19",
|
||||
"thiserror 2.0.20",
|
||||
"tokio",
|
||||
"tracing",
|
||||
]
|
||||
@@ -9878,7 +9878,7 @@ dependencies = [
|
||||
"sysinfo",
|
||||
"temp-env",
|
||||
"tempfile",
|
||||
"thiserror 2.0.19",
|
||||
"thiserror 2.0.20",
|
||||
"tokio",
|
||||
"tokio-util",
|
||||
"tracing",
|
||||
@@ -9914,7 +9914,7 @@ dependencies = [
|
||||
"strum 0.28.0",
|
||||
"temp-env",
|
||||
"test-case",
|
||||
"thiserror 2.0.19",
|
||||
"thiserror 2.0.20",
|
||||
"time",
|
||||
"tokio",
|
||||
"tracing",
|
||||
@@ -9946,7 +9946,6 @@ dependencies = [
|
||||
"md-5 0.11.0",
|
||||
"percent-encoding",
|
||||
"proptest",
|
||||
"quick-xml",
|
||||
"regex",
|
||||
"russh",
|
||||
"russh-sftp",
|
||||
@@ -9971,7 +9970,7 @@ dependencies = [
|
||||
"socket2",
|
||||
"subtle",
|
||||
"tempfile",
|
||||
"thiserror 2.0.19",
|
||||
"thiserror 2.0.20",
|
||||
"time",
|
||||
"tokio",
|
||||
"tokio-rustls",
|
||||
@@ -10056,7 +10055,7 @@ dependencies = [
|
||||
"serde_json",
|
||||
"sha1 0.11.0",
|
||||
"sha2 0.11.0",
|
||||
"thiserror 2.0.19",
|
||||
"thiserror 2.0.20",
|
||||
"tokio",
|
||||
"tokio-test",
|
||||
"tokio-util",
|
||||
@@ -10125,7 +10124,7 @@ dependencies = [
|
||||
"s3s",
|
||||
"serde_json",
|
||||
"serial_test",
|
||||
"thiserror 2.0.19",
|
||||
"thiserror 2.0.20",
|
||||
"tokio",
|
||||
"tokio-util",
|
||||
"tracing",
|
||||
@@ -10182,7 +10181,7 @@ dependencies = [
|
||||
"sha2 0.11.0",
|
||||
"temp-env",
|
||||
"tempfile",
|
||||
"thiserror 2.0.19",
|
||||
"thiserror 2.0.20",
|
||||
"time",
|
||||
"tokio",
|
||||
"tokio-util",
|
||||
@@ -10196,7 +10195,7 @@ name = "rustfs-security-governance"
|
||||
version = "1.0.0-rc.1"
|
||||
dependencies = [
|
||||
"hotpath",
|
||||
"thiserror 2.0.19",
|
||||
"thiserror 2.0.20",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -10211,7 +10210,7 @@ dependencies = [
|
||||
"rustfs-utils",
|
||||
"s3s",
|
||||
"serde_urlencoded",
|
||||
"thiserror 2.0.19",
|
||||
"thiserror 2.0.20",
|
||||
"time",
|
||||
"tracing",
|
||||
"tracing-subscriber",
|
||||
@@ -10274,7 +10273,7 @@ dependencies = [
|
||||
"snap",
|
||||
"sysinfo",
|
||||
"tempfile",
|
||||
"thiserror 2.0.19",
|
||||
"thiserror 2.0.20",
|
||||
"tokio",
|
||||
"tokio-postgres",
|
||||
"tokio-postgres-rustls",
|
||||
@@ -10318,7 +10317,7 @@ dependencies = [
|
||||
"serde_json",
|
||||
"sha2 0.11.0",
|
||||
"tempfile",
|
||||
"thiserror 2.0.19",
|
||||
"thiserror 2.0.20",
|
||||
"tokio",
|
||||
"tracing",
|
||||
]
|
||||
@@ -10342,7 +10341,7 @@ dependencies = [
|
||||
"serde_json",
|
||||
"serial_test",
|
||||
"temp-env",
|
||||
"thiserror 2.0.19",
|
||||
"thiserror 2.0.20",
|
||||
"tokio",
|
||||
"tower",
|
||||
"tracing",
|
||||
@@ -10411,7 +10410,7 @@ dependencies = [
|
||||
"criterion",
|
||||
"hotpath",
|
||||
"tempfile",
|
||||
"thiserror 2.0.19",
|
||||
"thiserror 2.0.20",
|
||||
"tokio",
|
||||
"tokio-stream",
|
||||
"zip",
|
||||
@@ -10470,7 +10469,7 @@ dependencies = [
|
||||
"errno",
|
||||
"libc",
|
||||
"linux-raw-sys",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -10543,7 +10542,7 @@ dependencies = [
|
||||
"security-framework",
|
||||
"security-framework-sys",
|
||||
"webpki-root-certs",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -10637,7 +10636,7 @@ dependencies = [
|
||||
"std-next",
|
||||
"subtle",
|
||||
"sync_wrapper",
|
||||
"thiserror 2.0.19",
|
||||
"thiserror 2.0.20",
|
||||
"time",
|
||||
"tokio",
|
||||
"tower",
|
||||
@@ -11187,7 +11186,7 @@ checksum = "0d585997b0ac10be3c5ee635f1bab02d512760d14b7c468801ac8a01d9ae5f1d"
|
||||
dependencies = [
|
||||
"num-bigint 0.4.8",
|
||||
"num-traits",
|
||||
"thiserror 2.0.19",
|
||||
"thiserror 2.0.20",
|
||||
"time",
|
||||
]
|
||||
|
||||
@@ -11479,7 +11478,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "04082e93ed1a06debd9148c928234b46d2cf260bc65f44e1d1d3fa594c5beebc"
|
||||
dependencies = [
|
||||
"simdutf8",
|
||||
"thiserror 2.0.19",
|
||||
"thiserror 2.0.20",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -11567,7 +11566,7 @@ dependencies = [
|
||||
"pin-project",
|
||||
"rustls",
|
||||
"rustls-pki-types",
|
||||
"thiserror 2.0.19",
|
||||
"thiserror 2.0.20",
|
||||
"tokio",
|
||||
"tokio-rustls",
|
||||
]
|
||||
@@ -11752,7 +11751,7 @@ dependencies = [
|
||||
"getrandom 0.4.3",
|
||||
"once_cell",
|
||||
"rustix",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -11819,11 +11818,11 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "thiserror"
|
||||
version = "2.0.19"
|
||||
version = "2.0.20"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "09a43598840e33d5b0331f38c5e30d13bb11c11210a4b58f0d9b18a5a5eefcd9"
|
||||
checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f"
|
||||
dependencies = [
|
||||
"thiserror-impl 2.0.19",
|
||||
"thiserror-impl 2.0.20",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -11839,9 +11838,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "thiserror-impl"
|
||||
version = "2.0.19"
|
||||
version = "2.0.20"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "43cbfe0cf76104d42a574802844187e84a305e531ed54455f11fbde0f10541cd"
|
||||
checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -12299,7 +12298,7 @@ checksum = "050686193eb999b4bb3bc2acfa891a13da00f79734704c4b8b4ef1a10b368a3c"
|
||||
dependencies = [
|
||||
"crossbeam-channel",
|
||||
"symlink",
|
||||
"thiserror 2.0.19",
|
||||
"thiserror 2.0.20",
|
||||
"time",
|
||||
"tracing-subscriber",
|
||||
]
|
||||
@@ -12434,7 +12433,7 @@ dependencies = [
|
||||
"rustls",
|
||||
"rustls-pki-types",
|
||||
"sha1 0.10.7",
|
||||
"thiserror 2.0.19",
|
||||
"thiserror 2.0.20",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -12472,7 +12471,7 @@ dependencies = [
|
||||
"derive_more",
|
||||
"libc",
|
||||
"md-5 0.10.6",
|
||||
"thiserror 2.0.19",
|
||||
"thiserror 2.0.20",
|
||||
"tokio",
|
||||
"x509-parser",
|
||||
]
|
||||
@@ -12620,7 +12619,7 @@ dependencies = [
|
||||
"rustify_derive",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"thiserror 2.0.19",
|
||||
"thiserror 2.0.20",
|
||||
"tracing",
|
||||
"url",
|
||||
]
|
||||
@@ -12700,9 +12699,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen"
|
||||
version = "0.2.126"
|
||||
version = "0.2.127"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4"
|
||||
checksum = "1b70935747edd64d89de3efa29d73789b806c15798f8e7dca4d8ac356b50ce70"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"once_cell",
|
||||
@@ -12713,9 +12712,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-futures"
|
||||
version = "0.4.76"
|
||||
version = "0.4.77"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c62df1340f32221cb9c54d6a27b030e3dba64361d4a95bed55f9aacb44da291d"
|
||||
checksum = "6b7777d5cc23d0e91404e53ce2d5e8ec7acae3026b16233dba62cd3246457950"
|
||||
dependencies = [
|
||||
"js-sys",
|
||||
"wasm-bindgen",
|
||||
@@ -12723,9 +12722,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-macro"
|
||||
version = "0.2.126"
|
||||
version = "0.2.127"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1"
|
||||
checksum = "77775f8f3f7217702089053b94958f8f54061a3f663417df76e19cbdcca29bc1"
|
||||
dependencies = [
|
||||
"quote",
|
||||
"wasm-bindgen-macro-support",
|
||||
@@ -12733,9 +12732,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-macro-support"
|
||||
version = "0.2.126"
|
||||
version = "0.2.127"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e"
|
||||
checksum = "e11d33f857dc2fb11b8bc75aee111aa9cbeb12cd9f25efd3d4c2a3dd4e235284"
|
||||
dependencies = [
|
||||
"bumpalo",
|
||||
"proc-macro2",
|
||||
@@ -12746,9 +12745,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-shared"
|
||||
version = "0.2.126"
|
||||
version = "0.2.127"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24"
|
||||
checksum = "7ef64dbcc55df09c7e5a46182d181c2cfa3e925f3da937ea764728b4bbb9dcbf"
|
||||
dependencies = [
|
||||
"unicode-ident",
|
||||
]
|
||||
@@ -12768,9 +12767,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "web-sys"
|
||||
version = "0.3.103"
|
||||
version = "0.3.104"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8622dcb61c0bcc9fffa6938bed81210af2da9a7e4a1a834b2e37a59b6dfb6141"
|
||||
checksum = "c435338968042f4f59a557f690a253676d47ce13ceb55d70100e7facf6620a30"
|
||||
dependencies = [
|
||||
"js-sys",
|
||||
"wasm-bindgen",
|
||||
@@ -12854,7 +12853,7 @@ version = "0.1.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
|
||||
dependencies = [
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -13136,7 +13135,7 @@ dependencies = [
|
||||
"nom 7.1.3",
|
||||
"oid-registry",
|
||||
"rusticata-macros",
|
||||
"thiserror 2.0.19",
|
||||
"thiserror 2.0.20",
|
||||
"time",
|
||||
]
|
||||
|
||||
|
||||
+3
-3
@@ -139,7 +139,7 @@ async_zip = { default-features = false, version = "0.0.18" }
|
||||
mysql_async = { default-features = false, version = "0.37" }
|
||||
async-compression = { version = "0.4.43" }
|
||||
async-recursion = "1.1.1"
|
||||
async-trait = "0.1.91"
|
||||
async-trait = "0.1.92"
|
||||
async-nats = { version = "0.50.0", default-features = false }
|
||||
axum = "0.8.9"
|
||||
futures = "0.3.33"
|
||||
@@ -302,7 +302,7 @@ sysinfo = "0.39.6"
|
||||
temp-env = "0.3.6"
|
||||
tempfile = "3.27.0"
|
||||
test-case = "3.3.1"
|
||||
thiserror = "2.0.19"
|
||||
thiserror = "2.0.20"
|
||||
tracing = { version = "0.1.44" }
|
||||
tracing-appender = "0.2.5"
|
||||
tracing-core = "0.1.36"
|
||||
@@ -354,7 +354,7 @@ hotpath = { version = "0.23.1", default-features = false }
|
||||
insta = { version = "1.48" }
|
||||
|
||||
[workspace.metadata.cargo-shear]
|
||||
ignored = ["rustfs"]
|
||||
ignored = ["hotpath", "rustfs"]
|
||||
|
||||
[profile.dev]
|
||||
# Full debuginfo roughly doubles compile+link time and produces multi-GB
|
||||
|
||||
@@ -436,13 +436,14 @@ pub mod rpc {
|
||||
pub use crate::cluster::rpc::{
|
||||
AuthenticatedChannel, KMS_SIGNAL_SUBSYSTEM, LocalPeerS3Client, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS,
|
||||
PeerRestClient, PeerS3Client, S3PeerSys, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC,
|
||||
ScannerBucketListing, ScannerPeerActivity, TONIC_RPC_PREFIX, TonicInterceptor, gen_signature_headers,
|
||||
gen_tonic_replay_scope_headers, gen_tonic_signature_headers, gen_tonic_signature_interceptor,
|
||||
node_service_time_out_client, node_service_time_out_client_no_auth, normalize_tonic_rpc_audience,
|
||||
set_tonic_canonical_body_digest, sign_ns_scanner_capability, sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge,
|
||||
tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason, verify_rpc_signature, verify_tonic_boot_epoch_response,
|
||||
verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof,
|
||||
verify_tonic_rpc_signature, verify_tonic_rpc_signature_with_bootstrap,
|
||||
ScannerBucketListing, ScannerPeerActivity, TONIC_RPC_PREFIX, TonicInterceptor, build_put_file_auth_trailer,
|
||||
check_and_record_signed_rpc_nonce, gen_signature_headers, gen_tonic_replay_scope_headers, gen_tonic_signature_headers,
|
||||
gen_tonic_signature_interceptor, node_service_time_out_client, node_service_time_out_client_no_auth,
|
||||
normalize_tonic_rpc_audience, set_tonic_canonical_body_digest, sign_ns_scanner_capability, sign_tonic_rpc_response_proof,
|
||||
tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason,
|
||||
verify_put_file_auth_trailer, verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest,
|
||||
verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof, verify_tonic_rpc_signature,
|
||||
verify_tonic_rpc_signature_with_bootstrap,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -27,7 +27,10 @@
|
||||
//! Advisory: <https://github.com/rustfs/rustfs/security/advisories/GHSA-r5qv-rc46-hv8q>
|
||||
|
||||
use crate::cluster::rpc::context_propagation::{inject_request_id_into_http_headers, inject_trace_context_into_http_headers};
|
||||
use crate::storage_api_contracts::internode::NS_SCANNER_PROTOCOL_VERSION;
|
||||
use crate::storage_api_contracts::internode::{
|
||||
NS_SCANNER_PROTOCOL_VERSION, PUT_FILE_AUTH_TRAILER_DIGEST_LEN, PUT_FILE_AUTH_TRAILER_LEN, PUT_FILE_AUTH_TRAILER_MAC_LEN,
|
||||
PUT_FILE_AUTH_TRAILER_MAGIC,
|
||||
};
|
||||
use base64::Engine as _;
|
||||
use base64::engine::general_purpose;
|
||||
use hmac::{Hmac, KeyInit, Mac};
|
||||
@@ -71,6 +74,7 @@ const RPC_REPLAY_SCOPE_VERSION_V3: &str = "3";
|
||||
const RPC_RESPONSE_PROOF_DOMAIN: &[u8] = b"rustfs-rpc-response-proof-v1\0";
|
||||
const RPC_REPLAY_SCOPE_DOMAIN: &[u8] = b"rustfs-rpc-replay-scope-v3\0";
|
||||
const RPC_BOOT_EPOCH_PROOF_DOMAIN: &[u8] = b"rustfs-rpc-boot-epoch-proof-v1\0";
|
||||
const HTTP_PUT_FILE_AUTH_DOMAIN: &[u8] = b"rustfs-http-put-file-auth-v1\0";
|
||||
const UNSIGNED_PAYLOAD: &str = "UNSIGNED-PAYLOAD";
|
||||
const UNSIGNED_PAYLOAD_NONCE: &str = "unsigned";
|
||||
const SIGNATURE_VALID_DURATION: i64 = 300; // 5 minutes
|
||||
@@ -475,6 +479,13 @@ fn signature_payload(url: &str, method: &Method, timestamp: i64) -> String {
|
||||
format!("{url}|{method}|{timestamp}")
|
||||
}
|
||||
|
||||
fn canonical_path_and_query(url: &str) -> std::io::Result<String> {
|
||||
let uri: Uri = url.parse().map_err(|_| std::io::Error::other("Invalid RPC URL"))?;
|
||||
uri.path_and_query()
|
||||
.map(ToString::to_string)
|
||||
.ok_or_else(|| std::io::Error::other("Invalid RPC URL"))
|
||||
}
|
||||
|
||||
fn redacted_rpc_path(url: &str) -> String {
|
||||
url.parse::<Uri>()
|
||||
.ok()
|
||||
@@ -502,6 +513,76 @@ fn verify_signature(secret: &str, url: &str, method: &Method, timestamp: i64, si
|
||||
mac.verify_slice(&signature).is_ok()
|
||||
}
|
||||
|
||||
fn update_put_file_auth_mac(
|
||||
mac: &mut HmacSha256,
|
||||
url: &str,
|
||||
method: &Method,
|
||||
nonce: Uuid,
|
||||
body_sha256: &str,
|
||||
) -> std::io::Result<()> {
|
||||
if !valid_content_sha256(body_sha256) || body_sha256 == UNSIGNED_PAYLOAD {
|
||||
return Err(std::io::Error::other("Invalid RPC content SHA-256"));
|
||||
}
|
||||
let path_and_query = canonical_path_and_query(url)?;
|
||||
mac.update(HTTP_PUT_FILE_AUTH_DOMAIN);
|
||||
for part in [
|
||||
path_and_query.as_bytes(),
|
||||
b"|",
|
||||
method.as_str().as_bytes(),
|
||||
b"|",
|
||||
nonce.as_bytes(),
|
||||
b"|",
|
||||
body_sha256.as_bytes(),
|
||||
] {
|
||||
mac.update(part);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn put_file_auth_mac(url: &str, method: &Method, nonce: Uuid, body_sha256: &str) -> std::io::Result<[u8; 32]> {
|
||||
let mut mac = <HmacSha256 as KeyInit>::new_from_slice(get_shared_secret()?.as_bytes())
|
||||
.map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
|
||||
update_put_file_auth_mac(&mut mac, url, method, nonce, body_sha256)?;
|
||||
Ok(mac.finalize().into_bytes().into())
|
||||
}
|
||||
|
||||
fn verify_put_file_auth_mac(url: &str, method: &Method, nonce: Uuid, body_sha256: &str, signature: &[u8]) -> std::io::Result<()> {
|
||||
let mut mac = <HmacSha256 as KeyInit>::new_from_slice(get_shared_secret()?.as_bytes())
|
||||
.map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
|
||||
update_put_file_auth_mac(&mut mac, url, method, nonce, body_sha256)?;
|
||||
mac.verify_slice(signature)
|
||||
.map_err(|_| std::io::Error::other("Invalid put_file auth trailer"))
|
||||
}
|
||||
|
||||
pub fn build_put_file_auth_trailer(url: &str, method: &Method, nonce: Uuid, body_sha256: &str) -> std::io::Result<Vec<u8>> {
|
||||
let mac = put_file_auth_mac(url, method, nonce, body_sha256)?;
|
||||
let mut trailer = Vec::with_capacity(PUT_FILE_AUTH_TRAILER_LEN);
|
||||
trailer.extend_from_slice(PUT_FILE_AUTH_TRAILER_MAGIC);
|
||||
trailer.extend_from_slice(body_sha256.as_bytes());
|
||||
trailer.extend_from_slice(&mac);
|
||||
Ok(trailer)
|
||||
}
|
||||
|
||||
pub fn verify_put_file_auth_trailer(url: &str, method: &Method, nonce: Uuid, trailer: &[u8]) -> std::io::Result<String> {
|
||||
if trailer.len() != PUT_FILE_AUTH_TRAILER_LEN {
|
||||
return Err(std::io::Error::other("Invalid put_file auth trailer length"));
|
||||
}
|
||||
if &trailer[..PUT_FILE_AUTH_TRAILER_MAGIC.len()] != PUT_FILE_AUTH_TRAILER_MAGIC {
|
||||
return Err(std::io::Error::other("Invalid put_file auth trailer"));
|
||||
}
|
||||
let digest_start = PUT_FILE_AUTH_TRAILER_MAGIC.len();
|
||||
let digest_end = digest_start + PUT_FILE_AUTH_TRAILER_DIGEST_LEN;
|
||||
let body_sha256 = std::str::from_utf8(&trailer[digest_start..digest_end])
|
||||
.map_err(|_| std::io::Error::other("Invalid RPC content SHA-256"))?;
|
||||
if !valid_content_sha256(body_sha256) || body_sha256 == UNSIGNED_PAYLOAD {
|
||||
return Err(std::io::Error::other("Invalid RPC content SHA-256"));
|
||||
}
|
||||
let mac_start = digest_end;
|
||||
let mac_end = mac_start + PUT_FILE_AUTH_TRAILER_MAC_LEN;
|
||||
verify_put_file_auth_mac(url, method, nonce, body_sha256, &trailer[mac_start..mac_end])?;
|
||||
Ok(body_sha256.to_string())
|
||||
}
|
||||
|
||||
fn update_ns_scanner_capability_mac(mac: &mut HmacSha256, challenge: Uuid, server_epoch: Uuid) {
|
||||
mac.update(NS_SCANNER_CAPABILITY_AUTH_DOMAIN);
|
||||
mac.update(&NS_SCANNER_PROTOCOL_VERSION.to_be_bytes());
|
||||
@@ -806,7 +887,13 @@ fn tonic_rpc_metric_operation(path: &str) -> &'static str {
|
||||
}
|
||||
}
|
||||
|
||||
fn check_and_record_nonce(nonce: Uuid, signed_at: i64, rpc_path: &str) -> std::io::Result<()> {
|
||||
fn check_and_record_nonce_with_scope(
|
||||
nonce: Uuid,
|
||||
signed_at: i64,
|
||||
rpc_path: &str,
|
||||
operation: &'static str,
|
||||
backend: &'static str,
|
||||
) -> std::io::Result<()> {
|
||||
let wall_time = OffsetDateTime::now_utc().unix_timestamp();
|
||||
let (result, metrics) = {
|
||||
let mut cache = LOCAL_RPC_NONCE_CACHE
|
||||
@@ -826,8 +913,8 @@ fn check_and_record_nonce(nonce: Uuid, signed_at: i64, rpc_path: &str) -> std::i
|
||||
expires_at,
|
||||
capacity: *REPLAY_CACHE_CAPACITY,
|
||||
metric_scope: RpcReplayCacheMetricScope {
|
||||
operation: tonic_rpc_metric_operation(rpc_path),
|
||||
backend: INTERNODE_TRANSPORT_BACKEND_GRPC,
|
||||
operation,
|
||||
backend,
|
||||
rpc_path,
|
||||
},
|
||||
})
|
||||
@@ -836,6 +923,37 @@ fn check_and_record_nonce(nonce: Uuid, signed_at: i64, rpc_path: &str) -> std::i
|
||||
result
|
||||
}
|
||||
|
||||
fn check_and_record_tonic_nonce(nonce: Uuid, signed_at: i64, rpc_path: &str) -> std::io::Result<()> {
|
||||
check_and_record_nonce_with_scope(
|
||||
nonce,
|
||||
signed_at,
|
||||
rpc_path,
|
||||
tonic_rpc_metric_operation(rpc_path),
|
||||
INTERNODE_TRANSPORT_BACKEND_GRPC,
|
||||
)
|
||||
}
|
||||
|
||||
pub fn check_and_record_signed_rpc_nonce(
|
||||
headers: &HeaderMap,
|
||||
nonce: Uuid,
|
||||
rpc_path: &str,
|
||||
operation: &'static str,
|
||||
backend: &'static str,
|
||||
) -> std::io::Result<()> {
|
||||
if nonce.is_nil() {
|
||||
return Err(std::io::Error::other("Invalid RPC nonce"));
|
||||
}
|
||||
let timestamp_header = headers
|
||||
.get(TIMESTAMP_HEADER)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.ok_or_else(|| std::io::Error::other("Missing timestamp header"))?;
|
||||
let timestamp = timestamp_header
|
||||
.parse::<i64>()
|
||||
.map_err(|_| std::io::Error::other("Invalid timestamp format"))?;
|
||||
check_timestamp(timestamp)?;
|
||||
check_and_record_nonce_with_scope(nonce, timestamp, rpc_path, operation, backend)
|
||||
}
|
||||
|
||||
/// Build headers with authentication signature
|
||||
pub fn build_auth_headers(url: &str, method: &Method, headers: &mut HeaderMap) -> std::io::Result<()> {
|
||||
let auth_headers = gen_signature_headers(url, method)?;
|
||||
@@ -1095,7 +1213,7 @@ fn verify_tonic_replay_scope_signature(audience: &str, path: &str, headers: &Hea
|
||||
if boot_epoch != tonic_rpc_boot_epoch() {
|
||||
return Err(std::io::Error::other("RPC boot epoch is stale"));
|
||||
}
|
||||
check_and_record_nonce(nonce, signed_at, path)
|
||||
check_and_record_tonic_nonce(nonce, signed_at, path)
|
||||
}
|
||||
|
||||
/// Verify gRPC authentication, preferring v2 without downgrade on malformed v2 metadata.
|
||||
@@ -1160,6 +1278,8 @@ pub fn tonic_rpc_auth_failure_reason(error: &std::io::Error) -> &'static str {
|
||||
"Invalid unsigned RPC nonce" => "invalid_unsigned_v2_nonce",
|
||||
"Missing RPC content SHA-256" => "missing_content_sha256",
|
||||
"Invalid RPC content SHA-256" => "invalid_content_sha256",
|
||||
"Invalid put_file auth trailer length" => "invalid_put_file_auth_trailer_length",
|
||||
"Invalid put_file auth trailer" => "invalid_put_file_auth_trailer",
|
||||
"Missing signature header" => "missing_v1_signature",
|
||||
"Invalid signature" => "invalid_v1_signature",
|
||||
"Invalid RPC HMAC key" => "invalid_hmac_key",
|
||||
@@ -1286,7 +1406,7 @@ fn verify_tonic_rpc_signature_with_strictness(
|
||||
return Err(std::io::Error::other("Invalid RPC v2 signature"));
|
||||
}
|
||||
if let Some(nonce) = parsed_nonce {
|
||||
check_and_record_nonce(nonce, timestamp, path)?;
|
||||
check_and_record_tonic_nonce(nonce, timestamp, path)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -2031,6 +2151,8 @@ mod tests {
|
||||
("Request timestamp expired", "timestamp_expired"),
|
||||
("Missing RPC content SHA-256", "missing_content_sha256"),
|
||||
("Invalid RPC content SHA-256", "invalid_content_sha256"),
|
||||
("Invalid put_file auth trailer length", "invalid_put_file_auth_trailer_length"),
|
||||
("Invalid put_file auth trailer", "invalid_put_file_auth_trailer"),
|
||||
] {
|
||||
assert_eq!(
|
||||
tonic_rpc_auth_failure_reason(&std::io::Error::other(message)),
|
||||
@@ -2178,6 +2300,37 @@ mod tests {
|
||||
assert_eq!(tampered.to_string(), "RPC content SHA-256 mismatch");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn put_file_auth_trailer_binds_url_nonce_and_body_digest() {
|
||||
ensure_test_rpc_secret();
|
||||
let url = concat!(
|
||||
"/rustfs/rpc/put_file_stream?disk=disk-a&volume=bucket&path=object%2Fpart.1",
|
||||
"&append=false&size=11&put_file_auth=digest-trailer-v1&put_file_nonce=11111111-2222-4333-8444-555555555555"
|
||||
);
|
||||
let nonce = Uuid::parse_str("11111111-2222-4333-8444-555555555555").expect("nonce");
|
||||
let body_sha256 = hex_simd::encode_to_string(Sha256::digest(b"hello world"), hex_simd::AsciiCase::Lower);
|
||||
let trailer = build_put_file_auth_trailer(url, &Method::PUT, nonce, &body_sha256).expect("trailer should build");
|
||||
|
||||
assert_eq!(trailer.len(), PUT_FILE_AUTH_TRAILER_LEN);
|
||||
let verified = verify_put_file_auth_trailer(url, &Method::PUT, nonce, &trailer).expect("trailer should verify");
|
||||
assert_eq!(verified, body_sha256);
|
||||
|
||||
let different_url = url.replace("size=11", "size=12");
|
||||
let err = verify_put_file_auth_trailer(&different_url, &Method::PUT, nonce, &trailer)
|
||||
.expect_err("trailer must bind the signed URL");
|
||||
assert_eq!(err.to_string(), "Invalid put_file auth trailer");
|
||||
|
||||
let err =
|
||||
verify_put_file_auth_trailer(url, &Method::PUT, Uuid::new_v4(), &trailer).expect_err("trailer must bind the nonce");
|
||||
assert_eq!(err.to_string(), "Invalid put_file auth trailer");
|
||||
|
||||
let mut tampered = trailer;
|
||||
tampered[PUT_FILE_AUTH_TRAILER_MAGIC.len()] = b'0';
|
||||
let err =
|
||||
verify_put_file_auth_trailer(url, &Method::PUT, nonce, &tampered).expect_err("trailer must bind the digest bytes");
|
||||
assert_eq!(err.to_string(), "Invalid put_file auth trailer");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier_mutation_rpc_contract_requires_method_bound_v2_body_digest() {
|
||||
ensure_test_rpc_secret();
|
||||
|
||||
@@ -12,14 +12,15 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::cluster::rpc::{build_auth_headers, verify_ns_scanner_capability};
|
||||
use crate::cluster::rpc::{build_auth_headers, build_put_file_auth_trailer, verify_ns_scanner_capability};
|
||||
use crate::disk::error::{Error, Result};
|
||||
use crate::disk::{FileReader, FileWriter};
|
||||
use crate::storage_api_contracts::internode::{
|
||||
NS_SCANNER_BODY_SHA256_QUERY, NS_SCANNER_CAPABILITY_CHALLENGE_QUERY, NS_SCANNER_CYCLE_QUERY, NS_SCANNER_LEADER_EPOCH_QUERY,
|
||||
NS_SCANNER_PROTOCOL_VERSION, NS_SCANNER_PROTOCOL_VERSION_QUERY, NS_SCANNER_REQUEST_ID_QUERY, NS_SCANNER_SERVER_EPOCH_QUERY,
|
||||
NS_SCANNER_SESSION_ID_QUERY, NS_SCANNER_SESSION_SEQUENCE_QUERY, NsScannerCapabilityResponse, WALK_DIR_BODY_SHA256_QUERY,
|
||||
WALK_DIR_STREAM_COMPLETION_QUERY, WALK_DIR_STREAM_COMPLETION_V1,
|
||||
NS_SCANNER_SESSION_ID_QUERY, NS_SCANNER_SESSION_SEQUENCE_QUERY, NsScannerCapabilityResponse, PUT_FILE_AUTH_QUERY,
|
||||
PUT_FILE_AUTH_V1, PUT_FILE_NONCE_QUERY, WALK_DIR_BODY_SHA256_QUERY, WALK_DIR_STREAM_COMPLETION_QUERY,
|
||||
WALK_DIR_STREAM_COMPLETION_V1,
|
||||
};
|
||||
use async_trait::async_trait;
|
||||
use http::{HeaderMap, HeaderValue, Method, header::CONTENT_TYPE};
|
||||
@@ -29,9 +30,11 @@ use rustfs_config::{
|
||||
};
|
||||
use rustfs_rio::{HttpReader, HttpWriter};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::pin::Pin;
|
||||
use std::sync::{Arc, OnceLock};
|
||||
use std::task::{Context, Poll};
|
||||
use std::time::Duration;
|
||||
use tokio::io::AsyncReadExt;
|
||||
use tokio::io::{AsyncReadExt, AsyncWrite};
|
||||
use uuid::Uuid;
|
||||
|
||||
static INTERNODE_DATA_TRANSPORT: OnceLock<std::result::Result<Arc<dyn InternodeDataTransport>, String>> = OnceLock::new();
|
||||
@@ -166,10 +169,12 @@ impl InternodeDataTransport for TcpHttpInternodeDataTransport {
|
||||
}
|
||||
|
||||
async fn open_write(&self, request: WriteStreamRequest) -> Result<FileWriter> {
|
||||
let url = build_put_file_stream_url(&request);
|
||||
let nonce = Uuid::new_v4();
|
||||
let url = build_put_file_stream_url(&request, Some(nonce));
|
||||
let mut headers = json_headers();
|
||||
build_auth_headers(&url, &Method::PUT, &mut headers)?;
|
||||
Ok(Box::new(HttpWriter::new(url, Method::PUT, headers).await?))
|
||||
let writer = HttpWriter::new(url.clone(), Method::PUT, headers).await?;
|
||||
Ok(Box::new(PutFileAuthWriter::new(writer, url, nonce)))
|
||||
}
|
||||
|
||||
async fn open_walk_dir(&self, request: WalkDirStreamRequest) -> Result<FileReader> {
|
||||
@@ -236,8 +241,8 @@ fn build_read_file_stream_url(request: &ReadStreamRequest) -> String {
|
||||
)
|
||||
}
|
||||
|
||||
fn build_put_file_stream_url(request: &WriteStreamRequest) -> String {
|
||||
format!(
|
||||
fn build_put_file_stream_url(request: &WriteStreamRequest, auth_nonce: Option<Uuid>) -> String {
|
||||
let mut url = format!(
|
||||
"{}{}?disk={}&volume={}&path={}&append={}&size={}",
|
||||
request.endpoint,
|
||||
PUT_FILE_STREAM_PATH,
|
||||
@@ -246,7 +251,104 @@ fn build_put_file_stream_url(request: &WriteStreamRequest) -> String {
|
||||
urlencoding::encode(&request.path),
|
||||
request.append,
|
||||
request.size
|
||||
)
|
||||
);
|
||||
if let Some(nonce) = auth_nonce {
|
||||
url.push_str(&format!(
|
||||
"&{}={}&{}={}",
|
||||
PUT_FILE_AUTH_QUERY, PUT_FILE_AUTH_V1, PUT_FILE_NONCE_QUERY, nonce
|
||||
));
|
||||
}
|
||||
url
|
||||
}
|
||||
|
||||
struct PutFileAuthWriter<W> {
|
||||
inner: W,
|
||||
url: String,
|
||||
nonce: Uuid,
|
||||
hasher: Sha256,
|
||||
trailer: Option<Vec<u8>>,
|
||||
trailer_offset: usize,
|
||||
}
|
||||
|
||||
impl<W> PutFileAuthWriter<W> {
|
||||
fn new(inner: W, url: String, nonce: Uuid) -> Self {
|
||||
Self {
|
||||
inner,
|
||||
url,
|
||||
nonce,
|
||||
hasher: Sha256::new(),
|
||||
trailer: None,
|
||||
trailer_offset: 0,
|
||||
}
|
||||
}
|
||||
|
||||
fn ensure_trailer(&mut self) -> std::io::Result<()> {
|
||||
if self.trailer.is_some() {
|
||||
return Ok(());
|
||||
}
|
||||
let digest = hex_simd::encode_to_string(self.hasher.clone().finalize(), hex_simd::AsciiCase::Lower);
|
||||
self.trailer = Some(build_put_file_auth_trailer(&self.url, &Method::PUT, self.nonce, &digest)?);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn poll_write_trailer(&mut self, cx: &mut Context<'_>) -> Poll<std::io::Result<()>>
|
||||
where
|
||||
W: AsyncWrite + Unpin,
|
||||
{
|
||||
self.ensure_trailer()?;
|
||||
let Some(trailer) = self.trailer.as_ref() else {
|
||||
return Poll::Ready(Err(std::io::Error::other("put_file auth trailer missing")));
|
||||
};
|
||||
while self.trailer_offset < trailer.len() {
|
||||
let written = match Pin::new(&mut self.inner).poll_write(cx, &trailer[self.trailer_offset..]) {
|
||||
Poll::Ready(Ok(0)) => {
|
||||
return Poll::Ready(Err(std::io::Error::new(
|
||||
std::io::ErrorKind::WriteZero,
|
||||
"failed to write put_file auth trailer",
|
||||
)));
|
||||
}
|
||||
Poll::Ready(Ok(written)) => written,
|
||||
Poll::Ready(Err(err)) => return Poll::Ready(Err(err)),
|
||||
Poll::Pending => return Poll::Pending,
|
||||
};
|
||||
self.trailer_offset += written;
|
||||
}
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
}
|
||||
|
||||
impl<W> AsyncWrite for PutFileAuthWriter<W>
|
||||
where
|
||||
W: AsyncWrite + Unpin,
|
||||
{
|
||||
fn poll_write(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &[u8]) -> Poll<std::io::Result<usize>> {
|
||||
if self.trailer.is_some() {
|
||||
return Poll::Ready(Err(std::io::Error::new(
|
||||
std::io::ErrorKind::BrokenPipe,
|
||||
"cannot write after put_file auth trailer",
|
||||
)));
|
||||
}
|
||||
match Pin::new(&mut self.inner).poll_write(cx, buf) {
|
||||
Poll::Ready(Ok(written)) => {
|
||||
self.hasher.update(&buf[..written]);
|
||||
Poll::Ready(Ok(written))
|
||||
}
|
||||
other => other,
|
||||
}
|
||||
}
|
||||
|
||||
fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
|
||||
Pin::new(&mut self.inner).poll_flush(cx)
|
||||
}
|
||||
|
||||
fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
|
||||
match self.poll_write_trailer(cx) {
|
||||
Poll::Ready(Ok(())) => {}
|
||||
Poll::Ready(Err(err)) => return Poll::Ready(Err(err)),
|
||||
Poll::Pending => return Poll::Pending,
|
||||
}
|
||||
Pin::new(&mut self.inner).poll_shutdown(cx)
|
||||
}
|
||||
}
|
||||
|
||||
fn build_walk_dir_url(request: &WalkDirStreamRequest) -> String {
|
||||
@@ -455,14 +557,17 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn put_file_stream_url_encodes_query_values() {
|
||||
let url = build_put_file_stream_url(&WriteStreamRequest {
|
||||
endpoint: "http://node1:9000".to_string(),
|
||||
disk: "http://node1:9000/data/rustfs0".to_string(),
|
||||
volume: "bucket".to_string(),
|
||||
path: "object/part.1".to_string(),
|
||||
append: false,
|
||||
size: 4096,
|
||||
});
|
||||
let url = build_put_file_stream_url(
|
||||
&WriteStreamRequest {
|
||||
endpoint: "http://node1:9000".to_string(),
|
||||
disk: "http://node1:9000/data/rustfs0".to_string(),
|
||||
volume: "bucket".to_string(),
|
||||
path: "object/part.1".to_string(),
|
||||
append: false,
|
||||
size: 4096,
|
||||
},
|
||||
None,
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
url,
|
||||
@@ -470,6 +575,63 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn put_file_stream_url_advertises_auth_nonce_when_enabled() {
|
||||
let nonce = Uuid::parse_str("11111111-2222-4333-8444-555555555555").expect("nonce");
|
||||
let url = build_put_file_stream_url(
|
||||
&WriteStreamRequest {
|
||||
endpoint: "http://node1:9000".to_string(),
|
||||
disk: "http://node1:9000/data/rustfs0".to_string(),
|
||||
volume: "bucket".to_string(),
|
||||
path: "object/part.1".to_string(),
|
||||
append: false,
|
||||
size: 4096,
|
||||
},
|
||||
Some(nonce),
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
url,
|
||||
concat!(
|
||||
"http://node1:9000/rustfs/rpc/put_file_stream?disk=http%3A%2F%2Fnode1%3A9000%2Fdata%2Frustfs0",
|
||||
"&volume=bucket&path=object%2Fpart.1&append=false&size=4096",
|
||||
"&put_file_auth=digest-trailer-v1&put_file_nonce=11111111-2222-4333-8444-555555555555"
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn put_file_auth_writer_appends_trailer_on_shutdown() {
|
||||
use tokio::io::AsyncWriteExt;
|
||||
|
||||
let _ = rustfs_credentials::set_global_rpc_secret("put-file-auth-writer-test-secret".to_string());
|
||||
let nonce = Uuid::parse_str("11111111-2222-4333-8444-555555555555").expect("nonce");
|
||||
let url = concat!(
|
||||
"http://node1:9000/rustfs/rpc/put_file_stream?disk=disk-a&volume=bucket&path=object%2Fpart.1",
|
||||
"&append=false&size=11&put_file_auth=digest-trailer-v1&put_file_nonce=11111111-2222-4333-8444-555555555555"
|
||||
)
|
||||
.to_string();
|
||||
let mut sink = Vec::new();
|
||||
|
||||
{
|
||||
let mut writer = PutFileAuthWriter::new(&mut sink, url.clone(), nonce);
|
||||
writer.write_all(b"hello world").await.expect("body write should succeed");
|
||||
writer.shutdown().await.expect("shutdown should append auth trailer");
|
||||
let err = writer
|
||||
.write_all(b"!")
|
||||
.await
|
||||
.expect_err("post-trailer writes must be rejected");
|
||||
assert_eq!(err.kind(), std::io::ErrorKind::BrokenPipe);
|
||||
}
|
||||
|
||||
assert_eq!(&sink[..11], b"hello world");
|
||||
let trailer = &sink[11..];
|
||||
let expected_digest = hex_simd::encode_to_string(Sha256::digest(b"hello world"), hex_simd::AsciiCase::Lower);
|
||||
let verified = crate::cluster::rpc::verify_put_file_auth_trailer(&url, &Method::PUT, nonce, trailer)
|
||||
.expect("emitted trailer should verify");
|
||||
assert_eq!(verified, expected_digest);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn walk_dir_url_encodes_disk_ref() {
|
||||
let url = build_walk_dir_url(&WalkDirStreamRequest {
|
||||
|
||||
@@ -32,10 +32,11 @@ pub use client::{
|
||||
// Re-exported through `api::rpc`; not every item is consumed inside this crate.
|
||||
#[allow(unused_imports)]
|
||||
pub use http_auth::{
|
||||
TONIC_RPC_PREFIX, build_auth_headers, gen_signature_headers, gen_tonic_replay_scope_headers, gen_tonic_signature_headers,
|
||||
normalize_tonic_rpc_audience, set_tonic_canonical_body_digest, set_tonic_mutation_body_digest, sign_ns_scanner_capability,
|
||||
sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason,
|
||||
verify_ns_scanner_capability, verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest,
|
||||
TONIC_RPC_PREFIX, build_auth_headers, build_put_file_auth_trailer, check_and_record_signed_rpc_nonce, gen_signature_headers,
|
||||
gen_tonic_replay_scope_headers, gen_tonic_signature_headers, normalize_tonic_rpc_audience, set_tonic_canonical_body_digest,
|
||||
set_tonic_mutation_body_digest, sign_ns_scanner_capability, sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge,
|
||||
tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason, verify_ns_scanner_capability, verify_put_file_auth_trailer,
|
||||
verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest,
|
||||
verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof, verify_tonic_rpc_signature,
|
||||
verify_tonic_rpc_signature_with_bootstrap,
|
||||
};
|
||||
|
||||
@@ -91,6 +91,8 @@ const DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD: usize = 1000;
|
||||
const DECOMMISSION_BUCKET_CONCURRENCY_ENV: &str = "RUSTFS_DECOMMISSION_BUCKET_CONCURRENCY";
|
||||
const DECOMMISSION_BUCKET_CONCURRENCY_DEFAULT_CAP: usize = 4;
|
||||
const DECOMMISSION_TARGET_CAPACITY_OVERHEAD_PERCENT: usize = 30;
|
||||
const DECOMMISSION_LISTING_MAX_ATTEMPTS: usize = 3;
|
||||
const DECOMMISSION_LISTING_RETRY_DELAY: std::time::Duration = std::time::Duration::from_secs(5);
|
||||
|
||||
pub const POOL_META_NAME: &str = "pool.bin";
|
||||
pub const POOL_META_FORMAT: u16 = 1;
|
||||
@@ -885,9 +887,149 @@ fn ensure_pool_not_left_in_cmdline_after_decommission(position: usize, cmd_line:
|
||||
|
||||
fn resolve_decommission_listing_worker_result(
|
||||
set_idx: usize,
|
||||
worker_result: std::result::Result<(), tokio::task::JoinError>,
|
||||
worker_result: std::result::Result<Result<()>, tokio::task::JoinError>,
|
||||
) -> Result<()> {
|
||||
worker_result.map_err(|err| Error::other(format!("decommission listing worker {set_idx} task join error: {err}")))
|
||||
worker_result.map_err(|err| Error::other(format!("decommission listing worker {set_idx} task join error: {err}")))?
|
||||
}
|
||||
|
||||
fn should_retry_decommission_listing(err: &Error, attempt: usize, max_attempts: usize) -> bool {
|
||||
!is_err_bucket_not_found(err) && attempt + 1 < max_attempts
|
||||
}
|
||||
|
||||
async fn wait_decommission_listing_retry(rx: &CancellationToken, delay: std::time::Duration) -> bool {
|
||||
tokio::select! {
|
||||
_ = rx.cancelled() => true,
|
||||
_ = tokio::time::sleep(delay) => false,
|
||||
}
|
||||
}
|
||||
|
||||
async fn run_decommission_listing_with_retry<List, ListFuture>(
|
||||
rx: CancellationToken,
|
||||
bucket: String,
|
||||
cb: ListCallback,
|
||||
pool_idx: usize,
|
||||
set_idx: usize,
|
||||
max_attempts: usize,
|
||||
mut list: List,
|
||||
) -> Result<()>
|
||||
where
|
||||
List: FnMut(ListCallback) -> ListFuture,
|
||||
ListFuture: std::future::Future<Output = Result<()>>,
|
||||
{
|
||||
let max_attempts = max_attempts.max(1);
|
||||
|
||||
for attempt in 0..max_attempts {
|
||||
if rx.is_cancelled() {
|
||||
debug!(
|
||||
event = EVENT_DECOMMISSION_BUCKET,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_POOLS,
|
||||
pool_index = pool_idx,
|
||||
set_index = set_idx,
|
||||
bucket = %bucket,
|
||||
state = "listing_worker_cancelled",
|
||||
"Decommission listing worker cancelled"
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
debug!(
|
||||
event = EVENT_DECOMMISSION_BUCKET,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_POOLS,
|
||||
pool_index = pool_idx,
|
||||
set_index = set_idx,
|
||||
bucket = %bucket,
|
||||
attempt = attempt + 1,
|
||||
max_attempts,
|
||||
state = "listing_started",
|
||||
"Decommission listing started"
|
||||
);
|
||||
|
||||
match list(cb.clone()).await {
|
||||
Ok(()) => {
|
||||
debug!(
|
||||
event = EVENT_DECOMMISSION_BUCKET,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_POOLS,
|
||||
pool_index = pool_idx,
|
||||
set_index = set_idx,
|
||||
bucket = %bucket,
|
||||
attempt = attempt + 1,
|
||||
max_attempts,
|
||||
state = "listing_completed",
|
||||
"Decommission listing completed"
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
Err(err) if is_err_bucket_not_found(&err) => {
|
||||
warn!(
|
||||
event = EVENT_DECOMMISSION_BUCKET,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_POOLS,
|
||||
pool_index = pool_idx,
|
||||
set_index = set_idx,
|
||||
bucket = %bucket,
|
||||
attempt = attempt + 1,
|
||||
max_attempts,
|
||||
state = "listing_bucket_missing",
|
||||
"Decommission listing bucket missing"
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
Err(err) if should_retry_decommission_listing(&err, attempt, max_attempts) => {
|
||||
error!(
|
||||
event = EVENT_DECOMMISSION_BUCKET,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_POOLS,
|
||||
pool_index = pool_idx,
|
||||
set_index = set_idx,
|
||||
bucket = %bucket,
|
||||
attempt = attempt + 1,
|
||||
max_attempts,
|
||||
retry_delay_ms = DECOMMISSION_LISTING_RETRY_DELAY.as_millis(),
|
||||
state = "listing_failed_retrying",
|
||||
error = ?err,
|
||||
"Decommission listing failed; retrying"
|
||||
);
|
||||
if wait_decommission_listing_retry(&rx, DECOMMISSION_LISTING_RETRY_DELAY).await {
|
||||
debug!(
|
||||
event = EVENT_DECOMMISSION_BUCKET,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_POOLS,
|
||||
pool_index = pool_idx,
|
||||
set_index = set_idx,
|
||||
bucket = %bucket,
|
||||
state = "listing_worker_cancelled",
|
||||
"Decommission listing worker cancelled during retry wait"
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
Err(err) => {
|
||||
error!(
|
||||
event = EVENT_DECOMMISSION_BUCKET,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_POOLS,
|
||||
pool_index = pool_idx,
|
||||
set_index = set_idx,
|
||||
bucket = %bucket,
|
||||
attempt = attempt + 1,
|
||||
max_attempts,
|
||||
state = "listing_failed",
|
||||
error = ?err,
|
||||
"Decommission listing failed"
|
||||
);
|
||||
return Err(Error::other(format!(
|
||||
"decommission listing failed for bucket {bucket} pool {pool_idx} set {set_idx} attempt {}/{}: {err}",
|
||||
attempt + 1,
|
||||
max_attempts
|
||||
)));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn should_count_decommission_version_complete(ignore: bool, cleanup_ignored: bool, failure: bool) -> bool {
|
||||
@@ -3261,78 +3403,21 @@ impl ECStore {
|
||||
let set_id = set_idx;
|
||||
let worker = tokio::spawn(async move {
|
||||
let _listing_permit = listing_permit;
|
||||
loop {
|
||||
if rx_clone.is_cancelled() {
|
||||
debug!(
|
||||
event = EVENT_DECOMMISSION_BUCKET,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_POOLS,
|
||||
pool_index = idx,
|
||||
set_index = set_id,
|
||||
bucket = %bi.name,
|
||||
state = "listing_worker_cancelled",
|
||||
"Decommission listing worker cancelled"
|
||||
);
|
||||
break;
|
||||
}
|
||||
debug!(
|
||||
event = EVENT_DECOMMISSION_BUCKET,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_POOLS,
|
||||
pool_index = idx,
|
||||
set_index = set_id,
|
||||
bucket = %bi.name,
|
||||
state = "listing_started",
|
||||
"Decommission listing started"
|
||||
);
|
||||
|
||||
match set
|
||||
.list_objects_to_decommission(rx_clone.clone(), bi.clone(), decommission_entry.clone())
|
||||
.await
|
||||
{
|
||||
Ok(_) => {
|
||||
debug!(
|
||||
event = EVENT_DECOMMISSION_BUCKET,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_POOLS,
|
||||
pool_index = idx,
|
||||
set_index = set_id,
|
||||
bucket = %bi.name,
|
||||
state = "listing_completed",
|
||||
"Decommission listing completed"
|
||||
);
|
||||
break;
|
||||
}
|
||||
Err(err) => {
|
||||
error!(
|
||||
event = EVENT_DECOMMISSION_BUCKET,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_POOLS,
|
||||
pool_index = idx,
|
||||
set_index = set_id,
|
||||
bucket = %bi.name,
|
||||
state = "listing_failed",
|
||||
error = ?err,
|
||||
"Decommission listing failed"
|
||||
);
|
||||
if is_err_bucket_not_found(&err) {
|
||||
warn!(
|
||||
event = EVENT_DECOMMISSION_BUCKET,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_POOLS,
|
||||
pool_index = idx,
|
||||
set_index = set_id,
|
||||
bucket = %bi.name,
|
||||
state = "listing_bucket_missing",
|
||||
"Decommission listing bucket missing"
|
||||
);
|
||||
break;
|
||||
}
|
||||
|
||||
tokio::time::sleep(tokio::time::Duration::from_secs(5)).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
run_decommission_listing_with_retry(
|
||||
rx_clone.clone(),
|
||||
bi.name.clone(),
|
||||
decommission_entry.clone(),
|
||||
idx,
|
||||
set_id,
|
||||
DECOMMISSION_LISTING_MAX_ATTEMPTS,
|
||||
|callback| {
|
||||
let set = set.clone();
|
||||
let rx = rx_clone.clone();
|
||||
let bucket = bi.clone();
|
||||
async move { set.list_objects_to_decommission(rx, bucket, callback).await }
|
||||
},
|
||||
)
|
||||
.await
|
||||
});
|
||||
listing_workers.push((set_id, worker));
|
||||
}
|
||||
@@ -4959,8 +5044,8 @@ pub(crate) fn fallback_free_capacity_dedup(disks: &[rustfs_madmin::Disk]) -> usi
|
||||
mod pools_tests {
|
||||
use super::{
|
||||
DECOMMISSION_PROGRESS_SAVE_INTERVAL, DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD, DecomBucketInfo,
|
||||
DecommissionStartPoolState, DecommissionTerminalState, PoolDecommissionInfo, PoolMeta, PoolSpaceInfo, PoolStatus,
|
||||
apply_decommission_status_space_info, bind_decommission_cancelers, bind_missing_decommission_cancelers,
|
||||
DecommissionStartPoolState, DecommissionTerminalState, ListCallback, PoolDecommissionInfo, PoolMeta, PoolSpaceInfo,
|
||||
PoolStatus, apply_decommission_status_space_info, bind_decommission_cancelers, bind_missing_decommission_cancelers,
|
||||
cancel_decommission_canceler, classify_decommission_terminal_state, count_decommission_item,
|
||||
decommission_cancel_signal_result, decommission_item_size, decommission_meta_bucket_options,
|
||||
decommission_start_pool_state, dedup_indices, default_decommission_bucket_concurrency,
|
||||
@@ -4982,17 +5067,18 @@ mod pools_tests {
|
||||
resolve_decommission_spawn_failure_result, resolve_decommission_terminal_mark_after_error_result,
|
||||
resolve_decommission_terminal_mark_result, resolve_decommission_update_after_result,
|
||||
resolve_start_decommission_pool_meta_reload_result, rollback_start_decommission_pool_meta,
|
||||
run_decommission_buckets_bounded, should_cleanup_decommission_source_entry, should_continue_decommission_queue,
|
||||
should_count_decommission_version_complete, should_preserve_decommission_canceled_state,
|
||||
should_reject_decommission_cancel_as_terminal, should_retry_decommission_cancel_reload,
|
||||
should_skip_canceled_decommission_routine, split_decommission_buckets, take_and_cancel_decommission_canceler,
|
||||
take_decommission_canceler, touch_decommission_progress, track_decommission_current_object,
|
||||
track_decommission_current_object_stage, validate_start_decommission_request, wait_decommission_worker_drain,
|
||||
run_decommission_buckets_bounded, run_decommission_listing_with_retry, should_cleanup_decommission_source_entry,
|
||||
should_continue_decommission_queue, should_count_decommission_version_complete,
|
||||
should_preserve_decommission_canceled_state, should_reject_decommission_cancel_as_terminal,
|
||||
should_retry_decommission_cancel_reload, should_retry_decommission_listing, should_skip_canceled_decommission_routine,
|
||||
split_decommission_buckets, take_and_cancel_decommission_canceler, take_decommission_canceler,
|
||||
touch_decommission_progress, track_decommission_current_object, track_decommission_current_object_stage,
|
||||
validate_start_decommission_request, wait_decommission_listing_retry, wait_decommission_worker_drain,
|
||||
with_decommission_entry_context,
|
||||
};
|
||||
use crate::data_movement;
|
||||
use crate::disk::endpoint::Endpoint;
|
||||
use crate::error::Error;
|
||||
use crate::error::{Error, StorageError};
|
||||
use crate::layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
|
||||
use crate::services::rebalance::{RebalStatus, RebalanceInfo, RebalanceMeta, RebalanceStats};
|
||||
use rustfs_filemeta::{FileInfo, FileInfoVersions, MetaCacheEntry, ObjectPartInfo};
|
||||
@@ -5006,6 +5092,10 @@ mod pools_tests {
|
||||
use tokio::sync::Semaphore;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
fn noop_decommission_list_callback() -> ListCallback {
|
||||
Arc::new(|_| Box::pin(async {}))
|
||||
}
|
||||
|
||||
fn decommission_test_pool_endpoint(idx: usize, is_local: bool) -> PoolEndpoints {
|
||||
let port = 9000usize + idx;
|
||||
let mut endpoint =
|
||||
@@ -6046,7 +6136,15 @@ mod pools_tests {
|
||||
|
||||
#[test]
|
||||
fn test_resolve_decommission_listing_worker_result_passthrough_ok() {
|
||||
assert!(resolve_decommission_listing_worker_result(2, Ok(())).is_ok());
|
||||
assert!(resolve_decommission_listing_worker_result(2, Ok(Ok(()))).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_resolve_decommission_listing_worker_result_passthrough_worker_error() {
|
||||
let err = resolve_decommission_listing_worker_result(2, Ok(Err(Error::SlowDown)))
|
||||
.expect_err("listing worker error should be returned");
|
||||
|
||||
assert!(matches!(err, Error::SlowDown));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -6064,6 +6162,80 @@ mod pools_tests {
|
||||
assert!(message.contains("panic"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_should_retry_decommission_listing_respects_attempt_limit_and_bucket_missing() {
|
||||
assert!(should_retry_decommission_listing(&Error::SlowDown, 0, 2));
|
||||
assert!(!should_retry_decommission_listing(&Error::SlowDown, 1, 2));
|
||||
assert!(!should_retry_decommission_listing(
|
||||
&StorageError::BucketNotFound("bucket".to_string()),
|
||||
0,
|
||||
2
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_wait_decommission_listing_retry_reports_canceled_without_sleeping() {
|
||||
let token = CancellationToken::new();
|
||||
token.cancel();
|
||||
|
||||
assert!(wait_decommission_listing_retry(&token, StdDuration::from_secs(30)).await);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn test_run_decommission_listing_with_retry_stops_after_attempt_limit() {
|
||||
let attempts = Arc::new(AtomicUsize::new(0));
|
||||
let err = run_decommission_listing_with_retry(
|
||||
CancellationToken::new(),
|
||||
"bucket-a".to_string(),
|
||||
noop_decommission_list_callback(),
|
||||
1,
|
||||
2,
|
||||
3,
|
||||
{
|
||||
let attempts = attempts.clone();
|
||||
move |_| {
|
||||
let attempts = attempts.clone();
|
||||
async move {
|
||||
attempts.fetch_add(1, Ordering::SeqCst);
|
||||
Err(Error::SlowDown)
|
||||
}
|
||||
}
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect_err("permanent listing failure must not retry forever");
|
||||
|
||||
assert_eq!(attempts.load(Ordering::SeqCst), 3);
|
||||
assert!(err.to_string().contains("attempt 3/3"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_run_decommission_listing_with_retry_treats_bucket_missing_as_complete() {
|
||||
let attempts = Arc::new(AtomicUsize::new(0));
|
||||
run_decommission_listing_with_retry(
|
||||
CancellationToken::new(),
|
||||
"bucket-a".to_string(),
|
||||
noop_decommission_list_callback(),
|
||||
1,
|
||||
2,
|
||||
3,
|
||||
{
|
||||
let attempts = attempts.clone();
|
||||
move |_| {
|
||||
let attempts = attempts.clone();
|
||||
async move {
|
||||
attempts.fetch_add(1, Ordering::SeqCst);
|
||||
Err(StorageError::BucketNotFound("bucket-a".to_string()))
|
||||
}
|
||||
}
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("missing bucket should keep previous decommission listing behavior");
|
||||
|
||||
assert_eq!(attempts.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_should_count_decommission_version_complete_for_cleanup_safe_ignored_result() {
|
||||
assert!(should_count_decommission_version_complete(true, true, false));
|
||||
|
||||
@@ -286,6 +286,23 @@ impl Sets {
|
||||
self.get_disks(self.get_hashed_set_index(key))
|
||||
}
|
||||
|
||||
fn get_disks_for_heal_object(&self, key: &str, opts: &HealOpts) -> Result<Arc<SetDisks>> {
|
||||
match opts.set {
|
||||
Some(set_idx) => self.disk_set.get(set_idx).cloned().ok_or_else(|| {
|
||||
StorageError::InvalidArgument(
|
||||
"heal".to_string(),
|
||||
"set".to_string(),
|
||||
format!(
|
||||
"invalid heal set index {set_idx} for pool {} with {} sets",
|
||||
self.pool_idx,
|
||||
self.disk_set.len()
|
||||
),
|
||||
)
|
||||
}),
|
||||
None => Ok(self.get_disks_by_key(key)),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn storage_info_snapshot(&self) -> rustfs_madmin::StorageInfo {
|
||||
let mut futures = Vec::with_capacity(self.disk_set.len());
|
||||
|
||||
@@ -1101,7 +1118,7 @@ impl crate::storage_api_contracts::heal::HealOperations for Sets {
|
||||
version_id: &str,
|
||||
opts: &HealOpts,
|
||||
) -> Result<(HealResultItem, Option<Error>)> {
|
||||
self.get_disks_by_key(object)
|
||||
self.get_disks_for_heal_object(object, opts)?
|
||||
.heal_object(bucket, object, version_id, opts)
|
||||
.await
|
||||
}
|
||||
@@ -1431,6 +1448,53 @@ mod tests {
|
||||
(temp_dirs, sets)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn heal_object_uses_explicit_set_scope() {
|
||||
let (_temp_dirs, sets) = two_set_test_sets().await;
|
||||
let selected = sets
|
||||
.get_disks_for_heal_object(
|
||||
"object",
|
||||
&HealOpts {
|
||||
set: Some(1),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.expect("requested set should be selected");
|
||||
|
||||
assert!(Arc::ptr_eq(&selected, &sets.disk_set[1]));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn heal_object_without_set_scope_keeps_hash_routing() {
|
||||
let (_temp_dirs, sets) = two_set_test_sets().await;
|
||||
let object = "object";
|
||||
let selected = sets
|
||||
.get_disks_for_heal_object(object, &HealOpts::default())
|
||||
.expect("hash-routed set should be selected");
|
||||
|
||||
assert!(Arc::ptr_eq(&selected, &sets.get_disks_by_key(object)));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn heal_object_rejects_invalid_set_scope() {
|
||||
let (_temp_dirs, sets) = two_set_test_sets().await;
|
||||
let err = sets
|
||||
.get_disks_for_heal_object(
|
||||
"object",
|
||||
&HealOpts {
|
||||
set: Some(2),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.expect_err("out-of-range set scope must fail closed");
|
||||
|
||||
assert!(
|
||||
matches!(err, StorageError::InvalidArgument(_, ref field, ref reason)
|
||||
if field == "set" && reason.contains("invalid heal set index 2 for pool 0 with 2 sets")),
|
||||
"unexpected invalid set error: {err:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_prefix_surfaces_a_hard_error_from_any_set() {
|
||||
let (_temp_dirs, sets) = two_set_test_sets().await;
|
||||
|
||||
@@ -1085,7 +1085,6 @@ where
|
||||
let data_shards = self.data_shards;
|
||||
let read_timeout = self.read_timeout;
|
||||
let metrics_path = self.metrics_path;
|
||||
let read_costs = self.read_costs.clone();
|
||||
let locality_preference_enabled = self.locality_preference_enabled;
|
||||
let stripe_read_start = metrics_path.map(|_| Instant::now());
|
||||
|
||||
@@ -1100,12 +1099,12 @@ where
|
||||
// before the retirement pass mutates `self.readers` below.
|
||||
{
|
||||
let mut sets = FuturesUnordered::new();
|
||||
let reader_iter = ReaderLaunchIter::new(&mut self.readers, &read_costs, locality_preference_enabled);
|
||||
let reader_iter = ReaderLaunchIter::new(&mut self.readers, self.read_costs.as_slice(), locality_preference_enabled);
|
||||
for (i, reader) in reader_iter {
|
||||
if reader.is_none() || !participating[i] {
|
||||
continue;
|
||||
}
|
||||
let read_cost = read_costs.get(i).copied().unwrap_or(ShardReadCost::Unknown);
|
||||
let read_cost = self.read_costs.get(i).copied().unwrap_or(ShardReadCost::Unknown);
|
||||
let recycled_buf = bufs[i].take();
|
||||
scheduled += 1;
|
||||
sets.push(read_shard(
|
||||
@@ -1237,7 +1236,7 @@ where
|
||||
if !self.try_engage_parity(idx, stripe_index) {
|
||||
continue;
|
||||
}
|
||||
let read_cost = read_costs.get(idx).copied().unwrap_or(ShardReadCost::Unknown);
|
||||
let read_cost = self.read_costs.get(idx).copied().unwrap_or(ShardReadCost::Unknown);
|
||||
let recycled_buf = Some(self.buffers.take(idx, shard_size));
|
||||
scheduled += 1;
|
||||
let (i, _read_cost, result, _should_retire) = read_shard(
|
||||
|
||||
@@ -327,16 +327,8 @@ impl ECStore {
|
||||
|
||||
#[tracing::instrument(skip(self, bucktes))]
|
||||
pub async fn init_and_start_rebalance(self: &Arc<Self>, bucktes: Vec<String>) -> Result<String> {
|
||||
let _start_guard = self.start_gate.lock().await;
|
||||
|
||||
let decommission_running = self.is_decommission_running().await;
|
||||
{
|
||||
let rebalance_meta = self.rebalance_meta.read().await;
|
||||
validate_init_rebalance_state(decommission_running, rebalance_meta.as_ref())?;
|
||||
}
|
||||
|
||||
let id = self.init_rebalance_meta(bucktes).await?;
|
||||
if let Err(start_err) = self.start_rebalance().await {
|
||||
let id = self.init_rebalance_start(bucktes).await?;
|
||||
if let Err(start_err) = self.start_rebalance_for_id(&id).await {
|
||||
if let Err(rollback_err) = self
|
||||
.rollback_rebalance_start_without_worker_for_id(Some(&id), start_err.to_string())
|
||||
.await
|
||||
@@ -354,6 +346,47 @@ impl ECStore {
|
||||
Ok(id)
|
||||
}
|
||||
|
||||
#[tracing::instrument(skip(self, bucktes))]
|
||||
pub async fn init_rebalance_start(self: &Arc<Self>, bucktes: Vec<String>) -> Result<String> {
|
||||
let _start_guard = self.start_gate.lock().await;
|
||||
|
||||
let decommission_running = self.is_decommission_running().await;
|
||||
{
|
||||
let rebalance_meta = self.rebalance_meta.read().await;
|
||||
validate_init_rebalance_state(decommission_running, rebalance_meta.as_ref())?;
|
||||
}
|
||||
|
||||
self.init_rebalance_meta(bucktes).await
|
||||
}
|
||||
|
||||
#[tracing::instrument(skip(self))]
|
||||
pub async fn start_rebalance_for_id(self: &Arc<Self>, expected_id: &str) -> Result<()> {
|
||||
let _start_guard = self.start_gate.lock().await;
|
||||
|
||||
{
|
||||
let rebalance_meta = self.rebalance_meta.read().await;
|
||||
let Some(meta) = rebalance_meta.as_ref() else {
|
||||
return Err(Error::ConfigNotFound);
|
||||
};
|
||||
if meta.id != expected_id {
|
||||
return Err(Error::other(format!(
|
||||
"rebalance metadata changed before start: expected {expected_id}, found {}",
|
||||
meta.id
|
||||
)));
|
||||
}
|
||||
if meta.stopped_at.is_some() {
|
||||
return Err(Error::other(format!("rebalance {expected_id} was stopped before start")));
|
||||
}
|
||||
}
|
||||
|
||||
self.start_rebalance().await
|
||||
}
|
||||
|
||||
pub async fn rollback_rebalance_start_for_id(self: &Arc<Self>, expected_id: Option<&str>, start_error: String) -> Result<()> {
|
||||
self.rollback_rebalance_start_without_worker_for_id(expected_id, start_error)
|
||||
.await
|
||||
}
|
||||
|
||||
#[tracing::instrument(skip(self, fi))]
|
||||
pub async fn update_pool_stats(&self, pool_index: usize, bucket: String, fi: &FileInfo) -> Result<()> {
|
||||
self.update_pool_stats_batch(pool_index, bucket, &[fi]).await
|
||||
|
||||
@@ -2584,21 +2584,7 @@ async fn test_init_and_start_rebalance_rejects_second_start_after_gate() {
|
||||
}],
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let endpoint_pools: crate::layout::endpoints::EndpointServerPools = Vec::new().into();
|
||||
let store = Arc::new(crate::store::ECStore {
|
||||
id: uuid::Uuid::new_v4(),
|
||||
disk_map: std::collections::HashMap::new(),
|
||||
pools: Vec::new(),
|
||||
peer_sys: crate::cluster::rpc::S3PeerSys::new(&endpoint_pools),
|
||||
pool_meta: tokio::sync::RwLock::new(crate::core::pools::PoolMeta::default()),
|
||||
rebalance_meta: tokio::sync::RwLock::new(Some(active_meta)),
|
||||
decommission_cancelers: tokio::sync::RwLock::new(Vec::new()),
|
||||
start_gate: tokio::sync::Mutex::new(()),
|
||||
pool_meta_save_gate: tokio::sync::Mutex::new(()),
|
||||
ctx: crate::runtime::instance::bootstrap_ctx(),
|
||||
bucket_fence_registry: std::sync::Arc::default(),
|
||||
});
|
||||
let store = test_store_with_rebalance_meta(active_meta);
|
||||
|
||||
let err = store
|
||||
.init_and_start_rebalance(vec!["bucket".to_string()])
|
||||
@@ -2608,6 +2594,72 @@ async fn test_init_and_start_rebalance_rejects_second_start_after_gate() {
|
||||
assert!(matches!(err, Error::RebalanceAlreadyRunning));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_start_rebalance_for_id_rejects_changed_metadata() {
|
||||
let meta = RebalanceMeta {
|
||||
id: "rebalance-a".to_string(),
|
||||
pool_stats: vec![RebalanceStats {
|
||||
participating: true,
|
||||
info: RebalanceInfo {
|
||||
status: RebalStatus::Started,
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
}],
|
||||
..Default::default()
|
||||
};
|
||||
let store = test_store_with_rebalance_meta(meta);
|
||||
|
||||
let err = store
|
||||
.start_rebalance_for_id("rebalance-b")
|
||||
.await
|
||||
.expect_err("staged start must not start changed metadata");
|
||||
|
||||
assert!(err.to_string().contains("rebalance metadata changed before start"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_start_rebalance_for_id_rejects_stopped_metadata() {
|
||||
let meta = RebalanceMeta {
|
||||
id: "rebalance-a".to_string(),
|
||||
stopped_at: Some(OffsetDateTime::now_utc()),
|
||||
pool_stats: vec![RebalanceStats {
|
||||
participating: true,
|
||||
info: RebalanceInfo {
|
||||
status: RebalStatus::Started,
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
}],
|
||||
..Default::default()
|
||||
};
|
||||
let store = test_store_with_rebalance_meta(meta);
|
||||
|
||||
let err = store
|
||||
.start_rebalance_for_id("rebalance-a")
|
||||
.await
|
||||
.expect_err("staged start must not restart stopped metadata");
|
||||
|
||||
assert!(err.to_string().contains("was stopped before start"));
|
||||
}
|
||||
|
||||
fn test_store_with_rebalance_meta(meta: RebalanceMeta) -> Arc<crate::store::ECStore> {
|
||||
let endpoint_pools: crate::layout::endpoints::EndpointServerPools = Vec::new().into();
|
||||
Arc::new(crate::store::ECStore {
|
||||
id: uuid::Uuid::new_v4(),
|
||||
disk_map: std::collections::HashMap::new(),
|
||||
pools: Vec::new(),
|
||||
peer_sys: crate::cluster::rpc::S3PeerSys::new(&endpoint_pools),
|
||||
pool_meta: tokio::sync::RwLock::new(crate::core::pools::PoolMeta::default()),
|
||||
rebalance_meta: tokio::sync::RwLock::new(Some(meta)),
|
||||
decommission_cancelers: tokio::sync::RwLock::new(Vec::new()),
|
||||
start_gate: tokio::sync::Mutex::new(()),
|
||||
pool_meta_save_gate: tokio::sync::Mutex::new(()),
|
||||
ctx: crate::runtime::instance::bootstrap_ctx(),
|
||||
bucket_fence_registry: std::sync::Arc::default(),
|
||||
})
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_percent_free_ratio_zero_capacity_is_zero() {
|
||||
assert_eq!(percent_free_ratio(100, 0), 0.0);
|
||||
|
||||
@@ -3412,6 +3412,18 @@ impl SetDisks {
|
||||
}
|
||||
|
||||
async fn recover_part_transaction(&self, dst_object: &str, write_quorum: usize) -> disk::error::Result<bool> {
|
||||
struct PartTransactionObservation {
|
||||
transaction_meta: Option<Bytes>,
|
||||
current_meta: Option<Bytes>,
|
||||
rollback: bool,
|
||||
err: Option<DiskError>,
|
||||
}
|
||||
|
||||
enum PartTransactionOutcome {
|
||||
Commit,
|
||||
Rollback,
|
||||
}
|
||||
|
||||
let disks = self.get_disks_internal().await;
|
||||
let transaction_path = part_transaction_path(dst_object);
|
||||
let transaction_meta_path = format!("{transaction_path}/{PART_TRANSACTION_NEW_META}");
|
||||
@@ -3425,36 +3437,76 @@ impl SetDisks {
|
||||
let current_meta_path = current_meta_path.clone();
|
||||
async move {
|
||||
let Some(disk) = disk else {
|
||||
return Ok((None, None, false));
|
||||
return PartTransactionObservation {
|
||||
transaction_meta: None,
|
||||
current_meta: None,
|
||||
rollback: false,
|
||||
err: Some(DiskError::DiskNotFound),
|
||||
};
|
||||
};
|
||||
let transaction_meta = match disk.read_all(RUSTFS_META_MULTIPART_BUCKET, &transaction_meta_path).await {
|
||||
Ok(meta) => Some(meta),
|
||||
Err(DiskError::FileNotFound) => None,
|
||||
Err(err) => return Err(err),
|
||||
Err(err) => {
|
||||
return PartTransactionObservation {
|
||||
transaction_meta: None,
|
||||
current_meta: None,
|
||||
rollback: false,
|
||||
err: Some(err),
|
||||
};
|
||||
}
|
||||
};
|
||||
let rollback = match disk.read_all(RUSTFS_META_MULTIPART_BUCKET, &rollback_path).await {
|
||||
Ok(_) => true,
|
||||
Err(DiskError::FileNotFound) => false,
|
||||
Err(err) => return Err(err),
|
||||
Err(err) => {
|
||||
return PartTransactionObservation {
|
||||
transaction_meta,
|
||||
current_meta: None,
|
||||
rollback: false,
|
||||
err: Some(err),
|
||||
};
|
||||
}
|
||||
};
|
||||
let current_meta = match disk.read_all(RUSTFS_META_MULTIPART_BUCKET, ¤t_meta_path).await {
|
||||
Ok(meta) => Some(meta),
|
||||
Err(DiskError::FileNotFound | DiskError::DiskNotFound) => None,
|
||||
Err(_) => None,
|
||||
};
|
||||
Ok((transaction_meta, current_meta, rollback))
|
||||
PartTransactionObservation {
|
||||
transaction_meta,
|
||||
current_meta,
|
||||
rollback,
|
||||
err: None,
|
||||
}
|
||||
}
|
||||
});
|
||||
let observations = join_all(reads).await.into_iter().collect::<disk::error::Result<Vec<_>>>()?;
|
||||
if observations.iter().all(|(transaction, _, _)| transaction.is_none()) {
|
||||
let observations = join_all(reads).await;
|
||||
let read_errs = observations
|
||||
.iter()
|
||||
.map(|observation| observation.err.clone())
|
||||
.collect::<Vec<_>>();
|
||||
if let Some(err) = reduce_write_quorum_errs(&read_errs, OBJECT_OP_IGNORED_ERRS, write_quorum) {
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
if observations
|
||||
.iter()
|
||||
.filter(|observation| observation.err.is_none())
|
||||
.all(|observation| observation.transaction_meta.is_none())
|
||||
{
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
let mut current_counts: HashMap<Bytes, usize> = HashMap::new();
|
||||
for (_, current, _) in &observations {
|
||||
if let Some(current) = current {
|
||||
let mut transaction_meta_values = HashSet::new();
|
||||
for observation in observations.iter().filter(|observation| observation.err.is_none()) {
|
||||
if let Some(current) = &observation.current_meta {
|
||||
*current_counts.entry(current.clone()).or_default() += 1;
|
||||
}
|
||||
if let Some(transaction_meta) = &observation.transaction_meta {
|
||||
transaction_meta_values.insert(transaction_meta.clone());
|
||||
}
|
||||
}
|
||||
let current_quorum = current_counts
|
||||
.into_iter()
|
||||
@@ -3462,11 +3514,29 @@ impl SetDisks {
|
||||
|
||||
let old_meta_path = format!("{transaction_path}/{PART_TRANSACTION_OLD_META}");
|
||||
let old_meta_absent_path = format!("{transaction_path}/old.meta.absent");
|
||||
let mut outcomes = Vec::with_capacity(observations.len());
|
||||
for observation in &observations {
|
||||
let outcome = if observation.err.is_none() && observation.transaction_meta.is_none() {
|
||||
match &observation.current_meta {
|
||||
Some(current_meta) if transaction_meta_values.contains(current_meta) => Some(PartTransactionOutcome::Commit),
|
||||
_ => Some(PartTransactionOutcome::Rollback),
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
outcomes.push(outcome);
|
||||
}
|
||||
let decisions = observations
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter_map(|(index, (transaction_meta, _, rollback))| {
|
||||
transaction_meta.as_ref().map(|meta| (index, meta.clone(), *rollback))
|
||||
.filter_map(|(index, observation)| {
|
||||
if observation.err.is_some() {
|
||||
return None;
|
||||
}
|
||||
observation
|
||||
.transaction_meta
|
||||
.as_ref()
|
||||
.map(|meta| (index, meta.clone(), observation.rollback))
|
||||
})
|
||||
.map(|(index, transaction_meta, rollback)| {
|
||||
let disk = disks[index].clone();
|
||||
@@ -3475,38 +3545,68 @@ impl SetDisks {
|
||||
let current_quorum = current_quorum.clone();
|
||||
async move {
|
||||
let Some(disk) = disk else {
|
||||
return Err(DiskError::DiskNotFound);
|
||||
return (index, Err(DiskError::DiskNotFound));
|
||||
};
|
||||
let action = if rollback {
|
||||
PartTransactionAction::Rollback
|
||||
} else if current_quorum.as_ref() == Some(&transaction_meta) {
|
||||
PartTransactionAction::Commit
|
||||
} else if let Some(current_quorum) = current_quorum {
|
||||
match disk.read_all(RUSTFS_META_MULTIPART_BUCKET, &old_meta_path).await {
|
||||
Ok(old_meta) if old_meta == current_quorum => PartTransactionAction::Rollback,
|
||||
Ok(_) => PartTransactionAction::Commit,
|
||||
Err(DiskError::FileNotFound) => {
|
||||
match disk.read_all(RUSTFS_META_MULTIPART_BUCKET, &old_meta_absent_path).await {
|
||||
Ok(_) => PartTransactionAction::Commit,
|
||||
Err(_) => return Err(DiskError::FileCorrupt),
|
||||
let result = async {
|
||||
let action = if rollback {
|
||||
PartTransactionAction::Rollback
|
||||
} else if current_quorum.as_ref() == Some(&transaction_meta) {
|
||||
PartTransactionAction::Commit
|
||||
} else if let Some(current_quorum) = current_quorum {
|
||||
match disk.read_all(RUSTFS_META_MULTIPART_BUCKET, &old_meta_path).await {
|
||||
Ok(old_meta) if old_meta == current_quorum => PartTransactionAction::Rollback,
|
||||
Ok(_) => PartTransactionAction::Commit,
|
||||
Err(DiskError::FileNotFound) => {
|
||||
match disk.read_all(RUSTFS_META_MULTIPART_BUCKET, &old_meta_absent_path).await {
|
||||
Ok(_) => PartTransactionAction::Commit,
|
||||
Err(_) => return Err(DiskError::FileCorrupt),
|
||||
}
|
||||
}
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
} else {
|
||||
PartTransactionAction::Rollback
|
||||
} else {
|
||||
PartTransactionAction::Rollback
|
||||
};
|
||||
disk.settle_part_transaction(RUSTFS_META_MULTIPART_BUCKET, dst_object, action)
|
||||
.await?;
|
||||
let outcome = match action {
|
||||
PartTransactionAction::Commit => PartTransactionOutcome::Commit,
|
||||
PartTransactionAction::Rollback => PartTransactionOutcome::Rollback,
|
||||
};
|
||||
Ok(outcome)
|
||||
};
|
||||
disk.settle_part_transaction(RUSTFS_META_MULTIPART_BUCKET, dst_object, action)
|
||||
.await?;
|
||||
Ok(action == PartTransactionAction::Commit)
|
||||
(index, result.await)
|
||||
}
|
||||
});
|
||||
|
||||
let results = join_all(decisions).await;
|
||||
if let Some(err) = results.iter().find_map(|result| result.as_ref().err()) {
|
||||
return Err(err.clone());
|
||||
let mut settle_errs = read_errs;
|
||||
for result in results {
|
||||
match result {
|
||||
(index, Ok(outcome)) => outcomes[index] = Some(outcome),
|
||||
(index, Err(err)) => settle_errs[index] = Some(err),
|
||||
}
|
||||
}
|
||||
Ok(results.iter().any(|result| matches!(result, Ok(true))))
|
||||
if let Some(err) = reduce_write_quorum_errs(&settle_errs, OBJECT_OP_IGNORED_ERRS, write_quorum) {
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
let commit_count = outcomes
|
||||
.iter()
|
||||
.filter(|outcome| matches!(outcome, Some(PartTransactionOutcome::Commit)))
|
||||
.count();
|
||||
if commit_count >= write_quorum {
|
||||
return Ok(true);
|
||||
}
|
||||
let rollback_count = outcomes
|
||||
.iter()
|
||||
.filter(|outcome| matches!(outcome, Some(PartTransactionOutcome::Rollback)))
|
||||
.count();
|
||||
if rollback_count >= write_quorum {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
Err(DiskError::ErasureWriteQuorum)
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) async fn recover_part_transactions(
|
||||
|
||||
@@ -7391,6 +7391,12 @@ mod tests {
|
||||
let (should_heal, _, _) = should_heal_object_on_disk(&err, &[], &meta, &latest_meta);
|
||||
assert!(should_heal);
|
||||
|
||||
let err = Some(DiskError::FileCorrupt);
|
||||
let (should_heal, is_meta, reason) = should_heal_object_on_disk(&err, &[], &meta, &latest_meta);
|
||||
assert!(should_heal);
|
||||
assert!(is_meta);
|
||||
assert_eq!(reason, Some(DiskError::FileCorrupt));
|
||||
|
||||
// Test with no error and no part errors
|
||||
let (should_heal, _, _) = should_heal_object_on_disk(&None, &[CHECK_PART_SUCCESS], &meta, &latest_meta);
|
||||
assert!(!should_heal);
|
||||
|
||||
@@ -3177,6 +3177,63 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn put_object_part_recovers_transaction_with_one_faulty_disk_at_write_quorum() {
|
||||
use tokio::io::AsyncReadExt as _;
|
||||
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks_for_pool_with_default_parity(4, 0, 2).await;
|
||||
assert_eq!(set_disks.default_read_quorum(), 2);
|
||||
assert_eq!(set_disks.default_write_quorum(), 3);
|
||||
|
||||
let bucket = "multipart-degraded-upload-part-bucket";
|
||||
let object = "object";
|
||||
make_bucket_on_all(&disk_stores, bucket).await;
|
||||
|
||||
let upload = set_disks
|
||||
.new_multipart_upload(bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("multipart upload should be created before the disk fault");
|
||||
disk_stores[0]
|
||||
.set_disk_id_state(Some(Uuid::new_v4()))
|
||||
.await
|
||||
.expect("test should mark one disk stale");
|
||||
|
||||
let payload = vec![0x5b; 4096];
|
||||
let mut reader = PutObjReader::from_vec(payload.clone());
|
||||
let part = set_disks
|
||||
.put_object_part(bucket, object, &upload.upload_id, 1, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("upload part should commit with exactly write quorum healthy disks");
|
||||
|
||||
set_disks
|
||||
.clone()
|
||||
.complete_multipart_upload(
|
||||
bucket,
|
||||
object,
|
||||
&upload.upload_id,
|
||||
vec![CompletePart {
|
||||
part_num: part.part_num,
|
||||
etag: part.etag,
|
||||
..Default::default()
|
||||
}],
|
||||
&ObjectOptions::default(),
|
||||
)
|
||||
.await
|
||||
.expect("completion should settle the write-quorum part");
|
||||
|
||||
let mut object_reader = set_disks
|
||||
.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("completed object should be readable through read quorum");
|
||||
let mut restored = Vec::new();
|
||||
object_reader
|
||||
.stream
|
||||
.read_to_end(&mut restored)
|
||||
.await
|
||||
.expect("completed object should stream fully");
|
||||
assert_eq!(restored, payload);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn put_object_part_rechecks_upload_after_commit_lock() {
|
||||
|
||||
@@ -27,9 +27,11 @@ pub(crate) mod internode {
|
||||
NS_SCANNER_BODY_SHA256_QUERY, NS_SCANNER_CAPABILITY_CHALLENGE_QUERY, NS_SCANNER_CYCLE_QUERY,
|
||||
NS_SCANNER_LEADER_EPOCH_QUERY, NS_SCANNER_PROTOCOL_VERSION, NS_SCANNER_PROTOCOL_VERSION_QUERY,
|
||||
NS_SCANNER_REQUEST_ID_QUERY, NS_SCANNER_SERVER_EPOCH_QUERY, NS_SCANNER_SESSION_ID_QUERY,
|
||||
NS_SCANNER_SESSION_SEQUENCE_QUERY, NsScannerCapabilityResponse, SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION,
|
||||
SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, SCANNER_ACTIVITY_PROTOCOL_VERSION, WALK_DIR_BODY_SHA256_QUERY,
|
||||
WALK_DIR_STREAM_COMPLETION_QUERY, WALK_DIR_STREAM_COMPLETION_V1,
|
||||
NS_SCANNER_SESSION_SEQUENCE_QUERY, NsScannerCapabilityResponse, PUT_FILE_AUTH_QUERY, PUT_FILE_AUTH_TRAILER_DIGEST_LEN,
|
||||
PUT_FILE_AUTH_TRAILER_LEN, PUT_FILE_AUTH_TRAILER_MAC_LEN, PUT_FILE_AUTH_TRAILER_MAGIC, PUT_FILE_AUTH_V1,
|
||||
PUT_FILE_NONCE_QUERY, SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION, SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION,
|
||||
SCANNER_ACTIVITY_PROTOCOL_VERSION, WALK_DIR_BODY_SHA256_QUERY, WALK_DIR_STREAM_COMPLETION_QUERY,
|
||||
WALK_DIR_STREAM_COMPLETION_V1,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -197,8 +197,8 @@ impl ECStore {
|
||||
registry: self.bucket_fence_registry.clone(),
|
||||
inner,
|
||||
};
|
||||
let memoized = pieces.enter(bucket);
|
||||
let current = match memoized {
|
||||
let registration = pieces.enter(bucket);
|
||||
let current = match registration.memoized {
|
||||
Some(current) => current,
|
||||
None => match metadata_sys::get_bucket_incarnation_id_in(&self.ctx, bucket).await {
|
||||
Ok(current) => {
|
||||
@@ -210,16 +210,16 @@ impl ECStore {
|
||||
current
|
||||
}
|
||||
Err(err) => {
|
||||
pieces.abandon(bucket);
|
||||
pieces.abandon(bucket, registration.token);
|
||||
return Err(err);
|
||||
}
|
||||
},
|
||||
};
|
||||
if current != expected {
|
||||
pieces.abandon(bucket);
|
||||
pieces.abandon(bucket, registration.token);
|
||||
return Err(StorageError::BucketNotFound(bucket.to_string()));
|
||||
}
|
||||
Ok(pieces.into_guard(bucket))
|
||||
Ok(pieces.into_guard(bucket, registration.token))
|
||||
}
|
||||
|
||||
pub(crate) async fn acquire_bucket_lifecycle_write_lock(&self, bucket: &str) -> Result<rustfs_lock::NamespaceLockGuard> {
|
||||
|
||||
@@ -44,14 +44,51 @@ use std::collections::HashMap;
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use rustfs_lock::NamespaceLockGuard;
|
||||
use rustfs_lock::distributed_lock::LockLostSignal;
|
||||
use uuid::Uuid;
|
||||
|
||||
#[derive(Default)]
|
||||
struct FenceEntry {
|
||||
guards: usize,
|
||||
next_token: u64,
|
||||
guards: Vec<RegisteredGuard>,
|
||||
validated: Option<Uuid>,
|
||||
}
|
||||
|
||||
struct RegisteredGuard {
|
||||
token: u64,
|
||||
loss_probe: LockLossProbe,
|
||||
}
|
||||
|
||||
enum LockLossProbe {
|
||||
Distributed(Arc<LockLostSignal>),
|
||||
Local,
|
||||
#[cfg(test)]
|
||||
Test(Arc<std::sync::atomic::AtomicBool>),
|
||||
}
|
||||
|
||||
impl LockLossProbe {
|
||||
fn from_guard(guard: &NamespaceLockGuard) -> Self {
|
||||
match guard.lock_lost_signal() {
|
||||
Some(signal) => Self::Distributed(signal),
|
||||
None => Self::Local,
|
||||
}
|
||||
}
|
||||
|
||||
fn is_lost(&self) -> bool {
|
||||
match self {
|
||||
Self::Distributed(signal) => signal.is_lost(),
|
||||
Self::Local => false,
|
||||
#[cfg(test)]
|
||||
Self::Test(lost) => lost.load(std::sync::atomic::Ordering::SeqCst),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) struct FenceRegistration {
|
||||
pub(super) token: u64,
|
||||
pub(super) memoized: Option<Uuid>,
|
||||
}
|
||||
|
||||
/// Per-store registry tracking, per bucket, how many lifecycle read guards are
|
||||
/// live on this node and the incarnation id validated under that coverage.
|
||||
#[derive(Default)]
|
||||
@@ -62,11 +99,20 @@ pub(crate) struct BucketFenceRegistry {
|
||||
impl BucketFenceRegistry {
|
||||
/// Register a new live guard for `bucket` and return the memoized
|
||||
/// incarnation id if one is valid for the current coverage window.
|
||||
fn enter(&self, bucket: &str) -> Option<Uuid> {
|
||||
fn enter(&self, bucket: &str, loss_probe: LockLossProbe) -> FenceRegistration {
|
||||
let mut entries = self.entries.lock().expect("bucket fence registry poisoned");
|
||||
let entry = entries.entry(bucket.to_string()).or_default();
|
||||
entry.guards += 1;
|
||||
entry.validated
|
||||
let token = entry.next_token;
|
||||
entry.next_token = entry.next_token.wrapping_add(1);
|
||||
entry.guards.push(RegisteredGuard { token, loss_probe });
|
||||
let has_lost_guard = entry.guards.iter().any(|guard| guard.loss_probe.is_lost());
|
||||
if has_lost_guard {
|
||||
entry.validated = None;
|
||||
}
|
||||
FenceRegistration {
|
||||
token,
|
||||
memoized: if has_lost_guard { None } else { entry.validated },
|
||||
}
|
||||
}
|
||||
|
||||
/// Memoize `incarnation` for `bucket`. Only meaningful while the caller
|
||||
@@ -74,23 +120,27 @@ impl BucketFenceRegistry {
|
||||
fn memoize(&self, bucket: &str, incarnation: Uuid) {
|
||||
let mut entries = self.entries.lock().expect("bucket fence registry poisoned");
|
||||
if let Some(entry) = entries.get_mut(bucket)
|
||||
&& entry.guards > 0
|
||||
&& !entry.guards.is_empty()
|
||||
{
|
||||
entry.validated = Some(incarnation);
|
||||
if entry.guards.iter().any(|guard| guard.loss_probe.is_lost()) {
|
||||
entry.validated = None;
|
||||
} else {
|
||||
entry.validated = Some(incarnation);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Deregister a guard. Clears the memo when the last guard leaves or when
|
||||
/// the leaving guard lost its lock (lost coverage means a lifecycle write
|
||||
/// lock may have been granted, so the memo can no longer be trusted).
|
||||
fn exit(&self, bucket: &str, lock_lost: bool) {
|
||||
fn exit(&self, bucket: &str, token: u64, lock_lost: bool) {
|
||||
let mut entries = self.entries.lock().expect("bucket fence registry poisoned");
|
||||
if let Some(entry) = entries.get_mut(bucket) {
|
||||
entry.guards = entry.guards.saturating_sub(1);
|
||||
entry.guards.retain(|guard| guard.token != token);
|
||||
if lock_lost {
|
||||
entry.validated = None;
|
||||
}
|
||||
if entry.guards == 0 {
|
||||
if entry.guards.is_empty() {
|
||||
entries.remove(bucket);
|
||||
}
|
||||
}
|
||||
@@ -104,6 +154,7 @@ pub(crate) struct BucketIncarnationFenceGuard {
|
||||
inner: Option<NamespaceLockGuard>,
|
||||
registry: Arc<BucketFenceRegistry>,
|
||||
bucket: String,
|
||||
token: u64,
|
||||
}
|
||||
|
||||
impl BucketIncarnationFenceGuard {
|
||||
@@ -115,7 +166,7 @@ impl BucketIncarnationFenceGuard {
|
||||
impl Drop for BucketIncarnationFenceGuard {
|
||||
fn drop(&mut self) {
|
||||
let lost = self.is_lock_lost();
|
||||
self.registry.exit(&self.bucket, lost);
|
||||
self.registry.exit(&self.bucket, self.token, lost);
|
||||
self.inner.take();
|
||||
}
|
||||
}
|
||||
@@ -128,8 +179,8 @@ pub(super) struct FencePieces {
|
||||
impl FencePieces {
|
||||
/// Register the freshly acquired read lock and return the memoized
|
||||
/// incarnation for the coverage window, if any.
|
||||
pub(super) fn enter(&self, bucket: &str) -> Option<Uuid> {
|
||||
self.registry.enter(bucket)
|
||||
pub(super) fn enter(&self, bucket: &str) -> FenceRegistration {
|
||||
self.registry.enter(bucket, LockLossProbe::from_guard(&self.inner))
|
||||
}
|
||||
|
||||
pub(super) fn memoize(&self, bucket: &str, incarnation: Uuid) {
|
||||
@@ -140,18 +191,19 @@ impl FencePieces {
|
||||
self.inner.is_lock_lost()
|
||||
}
|
||||
|
||||
pub(super) fn into_guard(self, bucket: &str) -> BucketIncarnationFenceGuard {
|
||||
pub(super) fn into_guard(self, bucket: &str, token: u64) -> BucketIncarnationFenceGuard {
|
||||
BucketIncarnationFenceGuard {
|
||||
inner: Some(self.inner),
|
||||
registry: self.registry,
|
||||
bucket: bucket.to_string(),
|
||||
token,
|
||||
}
|
||||
}
|
||||
|
||||
/// Abandon the acquisition (validation failed): deregister and release.
|
||||
pub(super) fn abandon(self, bucket: &str) {
|
||||
pub(super) fn abandon(self, bucket: &str, token: u64) {
|
||||
let lost = self.lock_lost();
|
||||
self.registry.exit(bucket, lost);
|
||||
self.registry.exit(bucket, token, lost);
|
||||
drop(self.inner);
|
||||
}
|
||||
}
|
||||
@@ -159,57 +211,155 @@ impl FencePieces {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::time::Duration;
|
||||
|
||||
use rustfs_lock::{LocalClient, LockRequest, LockType, NamespaceLock, ObjectKey};
|
||||
|
||||
fn uuid(n: u128) -> Uuid {
|
||||
Uuid::from_u128(n)
|
||||
}
|
||||
|
||||
fn live_probe() -> LockLossProbe {
|
||||
LockLossProbe::Test(Arc::new(std::sync::atomic::AtomicBool::new(false)))
|
||||
}
|
||||
|
||||
fn controllable_probe() -> (LockLossProbe, Arc<std::sync::atomic::AtomicBool>) {
|
||||
let lost = Arc::new(std::sync::atomic::AtomicBool::new(false));
|
||||
(LockLossProbe::Test(lost.clone()), lost)
|
||||
}
|
||||
|
||||
fn lock_request(owner: &str) -> LockRequest {
|
||||
LockRequest::new(ObjectKey::new("b", "lifecycle"), LockType::Shared, owner)
|
||||
.with_acquire_timeout(Duration::from_millis(100))
|
||||
.with_ttl(Duration::from_millis(20))
|
||||
.with_refresh_interval(Duration::from_millis(50))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn memo_valid_only_while_guards_overlap() {
|
||||
let reg = BucketFenceRegistry::default();
|
||||
|
||||
assert_eq!(reg.enter("b"), None, "first guard sees no memo");
|
||||
let first = reg.enter("b", live_probe());
|
||||
assert_eq!(first.memoized, None, "first guard sees no memo");
|
||||
reg.memoize("b", uuid(1));
|
||||
assert_eq!(reg.enter("b"), Some(uuid(1)), "overlapping guard reuses memo");
|
||||
reg.exit("b", false);
|
||||
reg.exit("b", false);
|
||||
let second = reg.enter("b", live_probe());
|
||||
assert_eq!(second.memoized, Some(uuid(1)), "overlapping guard reuses memo");
|
||||
reg.exit("b", first.token, false);
|
||||
reg.exit("b", second.token, false);
|
||||
|
||||
// Coverage gap: all guards gone, memo must be dropped.
|
||||
assert_eq!(reg.enter("b"), None, "post-gap guard must revalidate");
|
||||
reg.exit("b", false);
|
||||
let third = reg.enter("b", live_probe());
|
||||
assert_eq!(third.memoized, None, "post-gap guard must revalidate");
|
||||
reg.exit("b", third.token, false);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lost_lock_clears_memo_but_keeps_other_guards_registered() {
|
||||
let reg = BucketFenceRegistry::default();
|
||||
|
||||
assert_eq!(reg.enter("b"), None);
|
||||
let first = reg.enter("b", live_probe());
|
||||
assert_eq!(first.memoized, None);
|
||||
reg.memoize("b", uuid(7));
|
||||
assert_eq!(reg.enter("b"), Some(uuid(7)));
|
||||
let second = reg.enter("b", live_probe());
|
||||
assert_eq!(second.memoized, Some(uuid(7)));
|
||||
|
||||
// First guard exits reporting a lost lock: memo cleared even though
|
||||
// a second guard is still live.
|
||||
reg.exit("b", true);
|
||||
assert_eq!(reg.enter("b"), None, "memo not trusted after a lost lock");
|
||||
reg.exit("b", false);
|
||||
reg.exit("b", false);
|
||||
reg.exit("b", first.token, true);
|
||||
let third = reg.enter("b", live_probe());
|
||||
assert_eq!(third.memoized, None, "memo not trusted after a lost lock");
|
||||
reg.exit("b", second.token, false);
|
||||
reg.exit("b", third.token, false);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn live_lost_guard_blocks_memo_reuse_before_drop() {
|
||||
let reg = BucketFenceRegistry::default();
|
||||
|
||||
let (first_probe, first_lost) = controllable_probe();
|
||||
let first = reg.enter("b", first_probe);
|
||||
assert_eq!(first.memoized, None);
|
||||
reg.memoize("b", uuid(7));
|
||||
|
||||
first_lost.store(true, std::sync::atomic::Ordering::SeqCst);
|
||||
|
||||
let second = reg.enter("b", live_probe());
|
||||
assert_eq!(second.memoized, None, "live lost guard must force disk revalidation");
|
||||
reg.memoize("b", uuid(8));
|
||||
|
||||
let third = reg.enter("b", live_probe());
|
||||
assert_eq!(third.memoized, None, "memo remains blocked while the lost guard is live");
|
||||
|
||||
reg.exit("b", first.token, true);
|
||||
reg.memoize("b", uuid(8));
|
||||
let fourth = reg.enter("b", live_probe());
|
||||
assert_eq!(fourth.memoized, Some(uuid(8)), "memo resumes after lost coverage leaves");
|
||||
|
||||
reg.exit("b", second.token, false);
|
||||
reg.exit("b", third.token, false);
|
||||
reg.exit("b", fourth.token, false);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn fence_pieces_forwards_distributed_lock_loss_to_registry() {
|
||||
let registry = Arc::new(BucketFenceRegistry::default());
|
||||
let lock = NamespaceLock::new("bucket-fence-test".to_string(), Arc::new(LocalClient::new()));
|
||||
let first_guard = lock
|
||||
.acquire_guard(&lock_request("first"))
|
||||
.await
|
||||
.expect("distributed lock acquisition should not fail")
|
||||
.expect("distributed lock quorum should be reached");
|
||||
let first_pieces = FencePieces {
|
||||
registry: registry.clone(),
|
||||
inner: first_guard,
|
||||
};
|
||||
|
||||
let first = first_pieces.enter("b");
|
||||
assert_eq!(first.memoized, None);
|
||||
first_pieces.memoize("b", uuid(7));
|
||||
|
||||
tokio::time::timeout(Duration::from_secs(2), first_pieces.inner.lock_lost_notified())
|
||||
.await
|
||||
.expect("non-renewed distributed guard should lose its lease");
|
||||
assert!(first_pieces.lock_lost(), "test guard should observe lost refresh quorum");
|
||||
|
||||
let second_guard = lock
|
||||
.acquire_guard(&lock_request("second"))
|
||||
.await
|
||||
.expect("second distributed lock acquisition should not fail")
|
||||
.expect("second distributed lock quorum should be reached");
|
||||
let second_pieces = FencePieces {
|
||||
registry: registry.clone(),
|
||||
inner: second_guard,
|
||||
};
|
||||
let second = second_pieces.enter("b");
|
||||
assert_eq!(
|
||||
second.memoized, None,
|
||||
"a live distributed guard whose signal is lost must block memo reuse"
|
||||
);
|
||||
|
||||
second_pieces.abandon("b", second.token);
|
||||
first_pieces.abandon("b", first.token);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn buckets_are_isolated() {
|
||||
let reg = BucketFenceRegistry::default();
|
||||
assert_eq!(reg.enter("a"), None);
|
||||
let first = reg.enter("a", live_probe());
|
||||
assert_eq!(first.memoized, None);
|
||||
reg.memoize("a", uuid(1));
|
||||
assert_eq!(reg.enter("b"), None, "memo does not leak across buckets");
|
||||
reg.exit("b", false);
|
||||
reg.exit("a", false);
|
||||
let second = reg.enter("b", live_probe());
|
||||
assert_eq!(second.memoized, None, "memo does not leak across buckets");
|
||||
reg.exit("b", second.token, false);
|
||||
reg.exit("a", first.token, false);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn memoize_without_live_guard_is_ignored() {
|
||||
let reg = BucketFenceRegistry::default();
|
||||
reg.memoize("b", uuid(9));
|
||||
assert_eq!(reg.enter("b"), None);
|
||||
reg.exit("b", false);
|
||||
let first = reg.enter("b", live_probe());
|
||||
assert_eq!(first.memoized, None);
|
||||
reg.exit("b", first.token, false);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -21,7 +21,27 @@ const LOG_SUBSYSTEM_HEAL: &str = "heal";
|
||||
const EVENT_HEAL_FORMAT_COMPLETED: &str = "heal_format_completed";
|
||||
const EVENT_HEAL_OBJECT_STARTED: &str = "heal_object_started";
|
||||
|
||||
fn invalid_heal_pool_index(pool_idx: usize, pool_count: usize) -> Error {
|
||||
StorageError::InvalidArgument(
|
||||
"heal".to_string(),
|
||||
"pool".to_string(),
|
||||
format!("invalid heal pool index {pool_idx} for {pool_count} pools"),
|
||||
)
|
||||
}
|
||||
|
||||
impl ECStore {
|
||||
fn get_pools_for_heal_object(&self, opts: &HealOpts) -> Result<Vec<Arc<Sets>>> {
|
||||
match opts.pool {
|
||||
Some(pool_idx) => Ok(vec![
|
||||
self.pools
|
||||
.get(pool_idx)
|
||||
.cloned()
|
||||
.ok_or_else(|| invalid_heal_pool_index(pool_idx, self.pools.len()))?,
|
||||
]),
|
||||
None => Ok(self.pools.clone()),
|
||||
}
|
||||
}
|
||||
|
||||
#[instrument(skip(self))]
|
||||
pub(super) async fn handle_heal_format(&self, dry_run: bool) -> Result<(HealResultItem, Option<Error>)> {
|
||||
let mut r = HealResultItem {
|
||||
@@ -105,8 +125,10 @@ impl ECStore {
|
||||
);
|
||||
let object = encode_dir_object(object);
|
||||
|
||||
let mut futures = Vec::with_capacity(self.pools.len());
|
||||
for pool in self.pools.iter() {
|
||||
let pools = self.get_pools_for_heal_object(opts)?;
|
||||
|
||||
let mut futures = Vec::with_capacity(pools.len());
|
||||
for pool in pools.iter() {
|
||||
if self.is_suspended(pool.pool_idx).await {
|
||||
continue;
|
||||
}
|
||||
@@ -178,6 +200,82 @@ mod tests {
|
||||
use crate::layout::endpoints::{Endpoints, PoolEndpoints};
|
||||
use crate::store::init_format::{load_format_erasure, save_format_file};
|
||||
|
||||
async fn minimal_heal_pool(pool_idx: usize) -> Arc<Sets> {
|
||||
let format = FormatV3::new(1, 1);
|
||||
let endpoint_url = format!("http://127.0.0.1:{}/data", 19000 + pool_idx);
|
||||
let mut endpoint = Endpoint::try_from(endpoint_url.as_str()).expect("endpoint should parse");
|
||||
endpoint.set_pool_index(pool_idx);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(0);
|
||||
|
||||
Sets::new(
|
||||
vec![None],
|
||||
&PoolEndpoints {
|
||||
legacy: false,
|
||||
set_count: 1,
|
||||
drives_per_set: 1,
|
||||
endpoints: Endpoints::from(vec![endpoint]),
|
||||
cmd_line: String::new(),
|
||||
platform: String::new(),
|
||||
},
|
||||
&format,
|
||||
pool_idx,
|
||||
0,
|
||||
)
|
||||
.await
|
||||
.expect("minimal pool should build")
|
||||
}
|
||||
|
||||
async fn minimal_heal_store() -> ECStore {
|
||||
ECStore {
|
||||
id: Uuid::new_v4(),
|
||||
disk_map: HashMap::new(),
|
||||
pools: vec![minimal_heal_pool(0).await, minimal_heal_pool(1).await],
|
||||
peer_sys: S3PeerSys {
|
||||
clients: Vec::new(),
|
||||
pools_count: 2,
|
||||
},
|
||||
pool_meta: RwLock::new(PoolMeta::default()),
|
||||
rebalance_meta: RwLock::new(None),
|
||||
decommission_cancelers: RwLock::new(Vec::new()),
|
||||
start_gate: Mutex::new(()),
|
||||
pool_meta_save_gate: Mutex::new(()),
|
||||
ctx: crate::runtime::instance::bootstrap_ctx(),
|
||||
bucket_fence_registry: std::sync::Arc::default(),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn heal_object_pool_scope_selects_only_requested_pool() {
|
||||
let store = minimal_heal_store().await;
|
||||
let pools = store
|
||||
.get_pools_for_heal_object(&HealOpts {
|
||||
pool: Some(1),
|
||||
..Default::default()
|
||||
})
|
||||
.expect("requested pool should be selected");
|
||||
|
||||
assert_eq!(pools.len(), 1);
|
||||
assert!(Arc::ptr_eq(&pools[0], &store.pools[1]));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn heal_object_pool_scope_rejects_invalid_pool() {
|
||||
let store = minimal_heal_store().await;
|
||||
let err = store
|
||||
.get_pools_for_heal_object(&HealOpts {
|
||||
pool: Some(2),
|
||||
..Default::default()
|
||||
})
|
||||
.expect_err("out-of-range pool scope must fail closed");
|
||||
|
||||
assert!(
|
||||
matches!(err, StorageError::InvalidArgument(_, ref field, ref reason)
|
||||
if field == "pool" && reason.contains("invalid heal pool index 2 for 2 pools")),
|
||||
"unexpected invalid pool error: {err:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn handle_heal_format_continues_after_a_pool_error() {
|
||||
let canonical_format = FormatV3::new(1, 3);
|
||||
|
||||
@@ -602,7 +602,7 @@ mod tests {
|
||||
error::{Error, Result, StorageError},
|
||||
layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints},
|
||||
object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader},
|
||||
services::rebalance::RebalanceMeta,
|
||||
services::rebalance::{RebalStatus, RebalanceInfo, RebalanceMeta, RebalanceStats},
|
||||
storage_api_contracts::{
|
||||
bucket::{BucketOperations as _, MakeBucketOptions},
|
||||
multipart::MultipartOperations as _,
|
||||
@@ -1155,6 +1155,258 @@ mod tests {
|
||||
(instance_ctx, store, shutdown)
|
||||
}
|
||||
|
||||
fn active_rebalance_meta_for_pool(pool_count: usize, active_pool_idx: usize) -> RebalanceMeta {
|
||||
let now = OffsetDateTime::now_utc();
|
||||
let mut pool_stats = vec![RebalanceStats::default(); pool_count];
|
||||
pool_stats[active_pool_idx] = RebalanceStats {
|
||||
participating: true,
|
||||
info: RebalanceInfo {
|
||||
start_time: Some(now),
|
||||
status: RebalStatus::Started,
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
RebalanceMeta {
|
||||
id: uuid::Uuid::new_v4().to_string(),
|
||||
pool_stats,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn tag_updates_skip_active_rebalance_source_pool() {
|
||||
let temp_dir = tempfile::tempdir().expect("create writer-fencing store dir");
|
||||
let (_ctx, store, shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(temp_dir.path(), "writer-fencing-tags", &[4, 4])).await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
|
||||
let bucket = format!("writer-fencing-tags-{}", uuid::Uuid::new_v4());
|
||||
let object = "tagged-object.bin";
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create writer fencing bucket");
|
||||
|
||||
let old_time = OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("fixed timestamp should be valid");
|
||||
let newer_time = old_time + time::Duration::seconds(10);
|
||||
let mut source_reader = PutObjReader::from_vec(b"source-body".to_vec());
|
||||
store.pools[0]
|
||||
.put_object(
|
||||
&bucket,
|
||||
object,
|
||||
&mut source_reader,
|
||||
&ObjectOptions {
|
||||
mod_time: Some(newer_time),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("write newer source object");
|
||||
let mut target_reader = PutObjReader::from_vec(b"target-body".to_vec());
|
||||
store.pools[1]
|
||||
.put_object(
|
||||
&bucket,
|
||||
object,
|
||||
&mut target_reader,
|
||||
&ObjectOptions {
|
||||
mod_time: Some(old_time),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("write older target object");
|
||||
|
||||
*store.rebalance_meta.write().await = Some(active_rebalance_meta_for_pool(store.pools.len(), 0));
|
||||
assert!(store.is_pool_rebalancing(0).await, "pool 0 must be marked as an active rebalance source");
|
||||
|
||||
let tags = "rebalance=target";
|
||||
assert_ne!(
|
||||
store.pools[0]
|
||||
.get_object_tags(&bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("source object tags should be readable before update"),
|
||||
tags,
|
||||
"source object must start without the target tag"
|
||||
);
|
||||
assert_ne!(
|
||||
store.pools[1]
|
||||
.get_object_tags(&bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("target object tags should be readable before update"),
|
||||
tags,
|
||||
"target object must start without the target tag"
|
||||
);
|
||||
let selected_pool = store
|
||||
.get_pool_idx_existing_with_opts(
|
||||
&bucket,
|
||||
object,
|
||||
&ObjectOptions {
|
||||
no_lock: true,
|
||||
metadata_chg: true,
|
||||
skip_decommissioned: true,
|
||||
skip_rebalancing: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("writer lookup should select an existing non-rebalancing pool");
|
||||
assert_eq!(selected_pool, 1, "writer lookup must skip active rebalance pool 0");
|
||||
|
||||
let updated = store
|
||||
.put_object_tags(&bucket, object, tags, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("tag update should use the non-rebalancing target pool");
|
||||
assert_eq!(
|
||||
updated.mod_time,
|
||||
Some(old_time),
|
||||
"tag update must return the non-rebalancing pool object rather than the newer active source"
|
||||
);
|
||||
|
||||
let target_tags = store.pools[1]
|
||||
.get_object_tags(&bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("target object tags should be readable");
|
||||
assert_eq!(target_tags, tags, "non-rebalancing pool must receive writer tag updates");
|
||||
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn multipart_listing_skips_active_rebalance_source_pool() {
|
||||
let temp_dir = tempfile::tempdir().expect("create multipart writer-fencing store dir");
|
||||
let (_ctx, store, shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(temp_dir.path(), "writer-fencing-multipart", &[4, 4])).await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
|
||||
let bucket = format!("writer-fencing-multipart-{}", uuid::Uuid::new_v4());
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create multipart writer fencing bucket");
|
||||
|
||||
let incarnation = store.bucket_incarnation_id(&bucket).await.expect("read bucket incarnation");
|
||||
let lifecycle_guard = store
|
||||
.acquire_bucket_lifecycle_read_lock(&bucket)
|
||||
.await
|
||||
.expect("acquire multipart test lifecycle fence");
|
||||
let mut upload_opts = ObjectOptions {
|
||||
expected_bucket_incarnation_id: Some(incarnation),
|
||||
..Default::default()
|
||||
};
|
||||
upload_opts.add_bucket_lifecycle_lock_guard(&lifecycle_guard);
|
||||
let source_upload = store.pools[0]
|
||||
.new_multipart_upload(&bucket, "source-only.bin", &upload_opts)
|
||||
.await
|
||||
.expect("create source upload");
|
||||
let target_upload = store.pools[1]
|
||||
.new_multipart_upload(&bucket, "target-visible.bin", &upload_opts)
|
||||
.await
|
||||
.expect("create target upload");
|
||||
|
||||
*store.rebalance_meta.write().await = Some(active_rebalance_meta_for_pool(store.pools.len(), 0));
|
||||
assert!(store.is_pool_rebalancing(0).await, "pool 0 must be marked as an active rebalance source");
|
||||
|
||||
let listed = store
|
||||
.list_multipart_uploads(&bucket, "", None, None, None, 100)
|
||||
.await
|
||||
.expect("list multipart uploads");
|
||||
let listed_uploads: Vec<(&str, &str)> = listed
|
||||
.uploads
|
||||
.iter()
|
||||
.map(|upload| (upload.object.as_str(), upload.upload_id.as_str()))
|
||||
.collect();
|
||||
|
||||
assert!(
|
||||
!listed_uploads.contains(&("source-only.bin", source_upload.upload_id.as_str())),
|
||||
"active source pool upload must be hidden from multipart listing"
|
||||
);
|
||||
assert!(
|
||||
listed_uploads.contains(&("target-visible.bin", target_upload.upload_id.as_str())),
|
||||
"non-rebalancing pool upload must remain visible"
|
||||
);
|
||||
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn delete_objects_skips_active_rebalance_source_pool() {
|
||||
let temp_dir = tempfile::tempdir().expect("create batch-delete writer-fencing store dir");
|
||||
let (_ctx, store, shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(temp_dir.path(), "batch-delete-rebalance", &[4, 4])).await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
|
||||
let bucket = format!("batch-delete-rebalance-{}", uuid::Uuid::new_v4());
|
||||
let object = "delete-me.bin";
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create batch delete rebalance bucket");
|
||||
|
||||
let mut source_reader = PutObjReader::from_vec(b"source-body".to_vec());
|
||||
store.pools[0]
|
||||
.put_object(&bucket, object, &mut source_reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("write object on active source pool");
|
||||
let mut target_reader = PutObjReader::from_vec(b"target-body".to_vec());
|
||||
store.pools[1]
|
||||
.put_object(&bucket, object, &mut target_reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("write object on non-rebalancing target pool");
|
||||
|
||||
let mut pool_stats = vec![RebalanceStats::default(); store.pools.len()];
|
||||
pool_stats[0] = RebalanceStats {
|
||||
participating: true,
|
||||
info: RebalanceInfo {
|
||||
start_time: Some(OffsetDateTime::now_utc()),
|
||||
status: RebalStatus::Started,
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
*store.rebalance_meta.write().await = Some(RebalanceMeta {
|
||||
id: uuid::Uuid::new_v4().to_string(),
|
||||
pool_stats,
|
||||
..Default::default()
|
||||
});
|
||||
assert!(store.is_pool_rebalancing(0).await, "pool 0 must be marked as an active rebalance source");
|
||||
|
||||
let (deleted, errs) = store
|
||||
.delete_objects(
|
||||
&bucket,
|
||||
vec![crate::storage_api_contracts::object::ObjectToDelete {
|
||||
object_name: object.to_string(),
|
||||
..Default::default()
|
||||
}],
|
||||
ObjectOptions::default(),
|
||||
)
|
||||
.await;
|
||||
assert!(matches!(errs.as_slice(), [None]), "batch delete must not fail: {errs:?}");
|
||||
assert!(
|
||||
matches!(deleted.as_slice(), [deleted] if deleted.found && deleted.object_name == object),
|
||||
"batch delete must report the non-rebalancing pool deletion"
|
||||
);
|
||||
|
||||
store.pools[0]
|
||||
.get_object_info(&bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("active source pool object must not be deleted by DeleteObjects");
|
||||
let target_err = store.pools[1]
|
||||
.get_object_info(&bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect_err("non-rebalancing target pool object must be deleted");
|
||||
assert!(
|
||||
matches!(target_err, StorageError::ObjectNotFound(_, _)),
|
||||
"target pool should report object not found after DeleteObjects, got {target_err:?}"
|
||||
);
|
||||
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn data_movement_conflicts_preserve_newer_target_and_abort_staging() {
|
||||
|
||||
@@ -221,7 +221,7 @@ impl ECStore {
|
||||
}
|
||||
|
||||
for pool in self.pools.iter() {
|
||||
if self.is_suspended(pool.pool_idx).await {
|
||||
if self.is_suspended(pool.pool_idx).await || self.is_pool_rebalancing(pool.pool_idx).await {
|
||||
continue;
|
||||
}
|
||||
return match pool
|
||||
@@ -284,7 +284,7 @@ impl ECStore {
|
||||
let mut source_truncated = false;
|
||||
|
||||
for pool in self.pools.iter() {
|
||||
if self.is_suspended(pool.pool_idx).await {
|
||||
if self.is_suspended(pool.pool_idx).await || self.is_pool_rebalancing(pool.pool_idx).await {
|
||||
continue;
|
||||
}
|
||||
let res = list_pool_multipart_uploads_for_incarnation(
|
||||
@@ -433,7 +433,7 @@ impl ECStore {
|
||||
}
|
||||
|
||||
for pool in self.pools.iter() {
|
||||
if self.is_suspended(pool.pool_idx).await {
|
||||
if self.is_suspended(pool.pool_idx).await || self.is_pool_rebalancing(pool.pool_idx).await {
|
||||
continue;
|
||||
}
|
||||
let err = match pool.put_object_part(bucket, object, upload_id, part_id, data, opts).await {
|
||||
@@ -472,7 +472,7 @@ impl ECStore {
|
||||
}
|
||||
|
||||
for pool in self.pools.iter() {
|
||||
if self.is_suspended(pool.pool_idx).await {
|
||||
if self.is_suspended(pool.pool_idx).await || self.is_pool_rebalancing(pool.pool_idx).await {
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -510,7 +510,7 @@ impl ECStore {
|
||||
}
|
||||
|
||||
for pool in self.pools.iter() {
|
||||
if self.is_suspended(pool.pool_idx).await {
|
||||
if self.is_suspended(pool.pool_idx).await || self.is_pool_rebalancing(pool.pool_idx).await {
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -551,7 +551,7 @@ impl ECStore {
|
||||
}
|
||||
|
||||
for pool in self.pools.iter() {
|
||||
if self.is_suspended(pool.pool_idx).await {
|
||||
if self.is_suspended(pool.pool_idx).await || self.is_pool_rebalancing(pool.pool_idx).await {
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
@@ -597,6 +597,10 @@ fn version_aware_lookup_opts(opts: &ObjectOptions, no_lock: bool) -> ObjectOptio
|
||||
}
|
||||
|
||||
fn data_movement_pool_lookup_opts(opts: &ObjectOptions, no_lock: bool) -> ObjectOptions {
|
||||
writer_pool_lookup_opts(opts, no_lock)
|
||||
}
|
||||
|
||||
fn writer_pool_lookup_opts(opts: &ObjectOptions, no_lock: bool) -> ObjectOptions {
|
||||
let mut lookup_opts = version_aware_lookup_opts(opts, no_lock);
|
||||
lookup_opts.skip_decommissioned = true;
|
||||
lookup_opts.skip_rebalancing = true;
|
||||
@@ -607,6 +611,7 @@ fn data_movement_pool_lookup_opts(opts: &ObjectOptions, no_lock: bool) -> Object
|
||||
fn transition_restore_pool_opts(opts: &ObjectOptions) -> ObjectOptions {
|
||||
let mut lookup_opts = opts.clone();
|
||||
lookup_opts.skip_decommissioned = true;
|
||||
lookup_opts.skip_rebalancing = true;
|
||||
lookup_opts
|
||||
}
|
||||
|
||||
@@ -1358,7 +1363,7 @@ impl ECStore {
|
||||
|
||||
if cp_src_dst_same {
|
||||
let pool_idx = self
|
||||
.get_pool_info_existing_with_opts(src_bucket, &src_object, &version_aware_lookup_opts(src_opts, true))
|
||||
.get_pool_info_existing_with_opts(src_bucket, &src_object, &writer_pool_lookup_opts(src_opts, true))
|
||||
.await?
|
||||
.0
|
||||
.index;
|
||||
@@ -1671,7 +1676,7 @@ impl ECStore {
|
||||
return Ok(ObjectInfo::default());
|
||||
}
|
||||
|
||||
let gopts = version_aware_lookup_opts(&opts, true);
|
||||
let gopts = writer_pool_lookup_opts(&opts, true);
|
||||
|
||||
if opts.data_movement {
|
||||
let existing_pool_info = self.get_pool_info_existing_with_opts(bucket, object, &gopts).await;
|
||||
@@ -1787,6 +1792,10 @@ impl ECStore {
|
||||
}
|
||||
|
||||
for pool in self.pools.iter() {
|
||||
if self.is_suspended(pool.pool_idx).await || self.is_pool_rebalancing(pool.pool_idx).await {
|
||||
continue;
|
||||
}
|
||||
|
||||
match pool.delete_object(bucket, object, opts.clone()).await {
|
||||
Ok(res) => {
|
||||
if let (Some(api), Some(je)) = (tier_journal_api.as_ref(), journal_entry.as_ref()) {
|
||||
@@ -1923,6 +1932,9 @@ impl ECStore {
|
||||
let mut futures = Vec::with_capacity(self.pools.len());
|
||||
|
||||
for pool in self.pools.iter() {
|
||||
if self.is_pool_rebalancing(pool.pool_idx).await {
|
||||
continue;
|
||||
}
|
||||
futures.push(pool.delete_objects(bucket, objects.clone(), opts.clone()));
|
||||
}
|
||||
|
||||
@@ -2090,7 +2102,7 @@ impl ECStore {
|
||||
..Default::default()
|
||||
};
|
||||
let (_, idx) = self
|
||||
.get_latest_accessible_object_info_with_idx(bucket, object.as_str(), &opts)
|
||||
.get_latest_accessible_object_info_with_idx(bucket, object.as_str(), &writer_pool_lookup_opts(&opts, opts.no_lock))
|
||||
.await?;
|
||||
|
||||
let _ = self.pools[idx].add_partial(bucket, object.as_str(), version_id).await;
|
||||
@@ -2181,7 +2193,7 @@ impl ECStore {
|
||||
}
|
||||
|
||||
let (_, idx) = self
|
||||
.get_latest_accessible_object_info_with_idx(bucket, object.as_str(), &opts)
|
||||
.get_latest_accessible_object_info_with_idx(bucket, object.as_str(), &writer_pool_lookup_opts(&opts, opts.no_lock))
|
||||
.await?;
|
||||
|
||||
self.pools[idx]
|
||||
@@ -2224,7 +2236,7 @@ impl ECStore {
|
||||
}
|
||||
|
||||
let (_, idx) = self
|
||||
.get_latest_accessible_object_info_with_idx(bucket, object.as_str(), &opts)
|
||||
.get_latest_accessible_object_info_with_idx(bucket, object.as_str(), &writer_pool_lookup_opts(&opts, opts.no_lock))
|
||||
.await?;
|
||||
|
||||
let result = self.pools[idx].put_object_metadata(bucket, object.as_str(), &opts).await;
|
||||
@@ -2259,7 +2271,7 @@ impl ECStore {
|
||||
}
|
||||
|
||||
let (_, idx) = self
|
||||
.get_latest_accessible_object_info_with_idx(bucket, object.as_str(), opts)
|
||||
.get_latest_accessible_object_info_with_idx(bucket, object.as_str(), &writer_pool_lookup_opts(opts, opts.no_lock))
|
||||
.await?;
|
||||
|
||||
self.pools[idx].put_object_tags(bucket, object.as_str(), tags, opts).await
|
||||
@@ -2294,7 +2306,7 @@ impl ECStore {
|
||||
}
|
||||
|
||||
let (_, idx) = self
|
||||
.get_latest_accessible_object_info_with_idx(bucket, object.as_str(), opts)
|
||||
.get_latest_accessible_object_info_with_idx(bucket, object.as_str(), &writer_pool_lookup_opts(opts, opts.no_lock))
|
||||
.await?;
|
||||
|
||||
self.pools[idx].delete_object_tags(bucket, object.as_str(), opts).await
|
||||
@@ -2923,6 +2935,23 @@ mod tests {
|
||||
assert_eq!(lookup_opts.version_id.as_deref(), Some("vid-1"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn writer_pool_lookup_opts_skips_rebalance_sources() {
|
||||
let lookup_opts = writer_pool_lookup_opts(
|
||||
&ObjectOptions {
|
||||
version_id: Some("vid-1".to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
true,
|
||||
);
|
||||
|
||||
assert!(lookup_opts.no_lock);
|
||||
assert!(lookup_opts.metadata_chg);
|
||||
assert!(lookup_opts.skip_decommissioned);
|
||||
assert!(lookup_opts.skip_rebalancing);
|
||||
assert_eq!(lookup_opts.version_id.as_deref(), Some("vid-1"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn data_movement_pool_lookup_opts_keeps_no_lock_for_tiered_moves() {
|
||||
let lookup_opts = data_movement_pool_lookup_opts(
|
||||
@@ -2948,6 +2977,7 @@ mod tests {
|
||||
});
|
||||
|
||||
assert!(lookup_opts.skip_decommissioned);
|
||||
assert!(lookup_opts.skip_rebalancing);
|
||||
assert!(!lookup_opts.no_lock);
|
||||
}
|
||||
|
||||
@@ -2959,6 +2989,7 @@ mod tests {
|
||||
});
|
||||
|
||||
assert!(lookup_opts.skip_decommissioned);
|
||||
assert!(lookup_opts.skip_rebalancing);
|
||||
assert!(lookup_opts.no_lock);
|
||||
}
|
||||
|
||||
|
||||
@@ -541,14 +541,22 @@ impl ECStore {
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<(ObjectInfo, usize)> {
|
||||
let mut futures = Vec::with_capacity(self.pools.len());
|
||||
for pool in self.pools.iter() {
|
||||
futures.push(pool.get_object_info(bucket, object, opts));
|
||||
for (idx, pool) in self.pools.iter().enumerate() {
|
||||
if opts.skip_decommissioned && self.is_suspended(idx).await {
|
||||
continue;
|
||||
}
|
||||
|
||||
if opts.skip_rebalancing && self.is_pool_rebalancing(idx).await {
|
||||
continue;
|
||||
}
|
||||
|
||||
futures.push(async move { (idx, pool.get_object_info(bucket, object, opts).await) });
|
||||
}
|
||||
|
||||
let results = join_all(futures).await;
|
||||
let mut candidates = Vec::with_capacity(self.pools.len());
|
||||
|
||||
for (idx, result) in results.into_iter().enumerate() {
|
||||
for (idx, result) in results {
|
||||
match result {
|
||||
Ok(res) => {
|
||||
candidates.push(LatestObjectInfoCandidate {
|
||||
|
||||
@@ -1463,6 +1463,35 @@ mod test {
|
||||
/// Regression test for rustfs/rustfs#2715: a corrupted version count in
|
||||
/// xl.meta must yield a decode error instead of sizing a huge allocation
|
||||
/// from the bogus count (which aborts the whole process).
|
||||
/// A CRC mismatch means the bytes on disk are not the bytes that were
|
||||
/// written — bitrot. It must surface as `Error::FileCorrupt` specifically:
|
||||
/// that variant converts to `DiskError::FileCorrupt`, which is the only
|
||||
/// corruption signal `should_heal_object_on_disk` recognises. As a generic
|
||||
/// error the drive is skipped, the heal reports success, and the damaged
|
||||
/// `xl.meta` is never rewritten.
|
||||
#[test]
|
||||
fn test_unmarshal_reports_file_corrupt_on_crc_mismatch() {
|
||||
let mut fm = FileMeta::default();
|
||||
let mut buf = fm.marshal_msg().expect("serialize default FileMeta");
|
||||
// Flip one byte inside the meta blob: past the 8-byte XL2 header and
|
||||
// the 5-byte bin32 length prefix, before the CRC trailer.
|
||||
let idx = 8 + 5;
|
||||
buf[idx] ^= 0xff;
|
||||
let err = fm.unmarshal_msg(&buf).expect_err("corrupted meta must fail to decode");
|
||||
assert_eq!(err, Error::FileCorrupt, "CRC mismatch must classify as FileCorrupt, got: {err}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_is_indexed_meta_reports_file_corrupt_on_crc_mismatch() {
|
||||
let fm = FileMeta::default();
|
||||
let mut buf = fm.marshal_msg().expect("serialize default FileMeta");
|
||||
let idx = 8 + 5;
|
||||
buf[idx] ^= 0xff;
|
||||
|
||||
let err = FileMeta::is_indexed_meta(&buf).expect_err("corrupted indexed metadata must fail");
|
||||
assert_eq!(err, Error::FileCorrupt, "indexed CRC mismatch must classify as FileCorrupt, got: {err}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_unmarshal_rejects_absurd_version_count() {
|
||||
let mut meta = Vec::new();
|
||||
|
||||
@@ -97,7 +97,20 @@ impl FileMeta {
|
||||
let meta_crc = xxh64::xxh64(meta, XXHASH_SEED) as u32;
|
||||
|
||||
if crc != meta_crc {
|
||||
return Err(Error::other("xl file crc check failed"));
|
||||
error!(
|
||||
event = "filemeta_xl_crc_mismatch",
|
||||
component = "filemeta",
|
||||
expected_crc = meta_crc,
|
||||
actual_crc = crc,
|
||||
"xl.meta payload failed its CRC check"
|
||||
);
|
||||
// Error::FileCorrupt, not a generic error, for the same reason
|
||||
// check_xl2_v1 classifies a bad magic as FileCorrupt: heal
|
||||
// classification (should_heal_object_on_disk) recognises
|
||||
// corruption only by the DiskError::FileCorrupt variant this
|
||||
// converts to. As a generic error the drive is skipped,
|
||||
// heal_object reports ok, and on-disk bitrot is never repaired.
|
||||
return Err(Error::FileCorrupt);
|
||||
}
|
||||
|
||||
Ok((meta, inline_data))
|
||||
@@ -163,8 +176,16 @@ impl FileMeta {
|
||||
let meta_crc = xxh64::xxh64(meta, XXHASH_SEED) as u32;
|
||||
|
||||
if crc != meta_crc {
|
||||
error!("xl file crc check failed: expected CRC {:#x}, got {:#x}", meta_crc, crc);
|
||||
return Err(Error::other("xl file crc check failed"));
|
||||
error!(
|
||||
event = "filemeta_xl_crc_mismatch",
|
||||
component = "filemeta",
|
||||
expected_crc = meta_crc,
|
||||
actual_crc = crc,
|
||||
"xl.meta payload failed its CRC check"
|
||||
);
|
||||
// See is_indexed_meta: the FileCorrupt variant is what makes heal
|
||||
// classify this drive as needing metadata repair.
|
||||
return Err(Error::FileCorrupt);
|
||||
}
|
||||
|
||||
if !buf.is_empty() {
|
||||
|
||||
@@ -93,6 +93,8 @@ fn read_msgp_bin<R: std::io::Read>(rd: &mut R) -> Result<Vec<u8>> {
|
||||
read_exact_vec(rd, len)
|
||||
}
|
||||
|
||||
const MSGP_OBJECT_KEY_STACK_CAP: usize = 16;
|
||||
|
||||
/// Writes an `OffsetDateTime` as the ext8 / legacy (type 5, 12-byte
|
||||
/// seconds+nanos) msgpack time encoding used by the V1 (Legacy) object body.
|
||||
/// `read_msgp_time` decodes exactly this shape via `MSGPACK_TIME_EXT_LEGACY`.
|
||||
@@ -2060,16 +2062,33 @@ impl MetaObject {
|
||||
tracing::error!(error = %e, "decode_from: read_str_len key failed");
|
||||
e
|
||||
})?;
|
||||
let key_buf = read_exact_vec(rd, key_len as usize).map_err(|e| {
|
||||
tracing::error!(error = %e, "decode_from: read key_buf failed");
|
||||
e
|
||||
})?;
|
||||
let key = String::from_utf8(key_buf).map_err(|e| {
|
||||
tracing::error!(error = %e, "decode_from: from_utf8 key failed");
|
||||
e
|
||||
let key_len = usize::try_from(key_len).map_err(|e| {
|
||||
tracing::error!(error = %e, "decode_from: key length conversion failed");
|
||||
Error::other(e)
|
||||
})?;
|
||||
|
||||
match key.as_str() {
|
||||
let mut inline_key = [0u8; MSGP_OBJECT_KEY_STACK_CAP];
|
||||
let heap_key;
|
||||
let key_buf = if key_len <= MSGP_OBJECT_KEY_STACK_CAP {
|
||||
rd.read_exact(&mut inline_key[..key_len]).map_err(|e| {
|
||||
tracing::error!(error = %e, "decode_from: read key_buf failed");
|
||||
Error::from(e)
|
||||
})?;
|
||||
&inline_key[..key_len]
|
||||
} else {
|
||||
heap_key = read_exact_vec(rd, key_len).map_err(|e| {
|
||||
tracing::error!(error = %e, "decode_from: read key_buf failed");
|
||||
e
|
||||
})?;
|
||||
heap_key.as_slice()
|
||||
};
|
||||
|
||||
let key = std::str::from_utf8(key_buf).map_err(|e| {
|
||||
tracing::error!(error = %e, "decode_from: from_utf8 key failed");
|
||||
Error::FromUtf8(e.to_string())
|
||||
})?;
|
||||
|
||||
match key {
|
||||
"ID" => {
|
||||
let _ = rmp::decode::read_bin_len(rd).map_err(|e| {
|
||||
tracing::error!(error = %e, "decode_from: read_bin_len ID failed");
|
||||
@@ -2285,6 +2304,7 @@ impl MetaObject {
|
||||
Some(n) => n,
|
||||
};
|
||||
self.meta_sys.clear();
|
||||
self.meta_sys.reserve(prealloc_hint(len));
|
||||
for _ in 0..len {
|
||||
let k_len = rmp::decode::read_str_len(rd).map_err(|e| {
|
||||
tracing::error!(error = %e, "decode_from: read_str_len MetaSys key failed");
|
||||
@@ -2323,6 +2343,7 @@ impl MetaObject {
|
||||
Some(n) => n,
|
||||
};
|
||||
self.meta_user.clear();
|
||||
self.meta_user.reserve(prealloc_hint(len));
|
||||
for _ in 0..len {
|
||||
let k_len = rmp::decode::read_str_len(rd).map_err(|e| {
|
||||
tracing::error!(error = %e, "decode_from: read_str_len MetaUsr key failed");
|
||||
@@ -4824,6 +4845,46 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn meta_object_decode_round_trips_stack_sized_keys() {
|
||||
let object = signed_object();
|
||||
let encoded = object.marshal_msg().expect("object marshal should succeed");
|
||||
|
||||
let mut decoded = MetaObject::default();
|
||||
decoded
|
||||
.decode_from(&mut std::io::Cursor::new(&encoded))
|
||||
.expect("object decode should succeed");
|
||||
|
||||
assert_eq!(decoded, object);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn meta_object_decode_skips_unknown_valid_utf8_field() {
|
||||
let mut encoded = Vec::new();
|
||||
rmp::encode::write_map_len(&mut encoded, 1).expect("map header should encode");
|
||||
rmp::encode::write_str(&mut encoded, "UnknownFutureField").expect("field key should encode");
|
||||
rmp::encode::write_nil(&mut encoded).expect("nil payload should encode");
|
||||
|
||||
let mut decoded = MetaObject::default();
|
||||
decoded
|
||||
.decode_from(&mut std::io::Cursor::new(&encoded))
|
||||
.expect("unknown valid UTF-8 field should be skipped");
|
||||
|
||||
assert_eq!(decoded, MetaObject::default());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn meta_object_decode_rejects_invalid_utf8_field_name() {
|
||||
let encoded = [0x81, 0xa1, 0xff, 0xc0];
|
||||
let mut decoded = MetaObject::default();
|
||||
|
||||
let err = decoded
|
||||
.decode_from(&mut std::io::Cursor::new(encoded))
|
||||
.expect_err("invalid UTF-8 field name should fail");
|
||||
|
||||
assert!(matches!(err, Error::FromUtf8(_)), "unexpected error: {err}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn signature_is_no_longer_hardcoded_zero() {
|
||||
// Regression for B12: the write-path header must carry a real signature.
|
||||
|
||||
@@ -2475,7 +2475,7 @@ impl HealManager {
|
||||
return;
|
||||
}
|
||||
|
||||
let mut running_per_set = running_erasure_set_counts(&active_heals_guard);
|
||||
let mut running_per_set = running_heal_set_counts(&active_heals_guard);
|
||||
let mut tasks_started = 0usize;
|
||||
let mut delayed_by_mainline_throttle = false;
|
||||
|
||||
@@ -2856,6 +2856,14 @@ impl std::fmt::Debug for HealManager {
|
||||
fn heal_request_set_key(request: &HealRequest) -> Option<String> {
|
||||
match &request.heal_type {
|
||||
HealType::ErasureSet { set_disk_id, .. } => Some(set_disk_id.clone()),
|
||||
HealType::Object { .. } => heal_options_set_key(&request.options),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn heal_options_set_key(options: &HealOptions) -> Option<String> {
|
||||
match (options.pool_index, options.set_index) {
|
||||
(Some(pool), Some(set)) => Some(format!("pool_{pool}_set_{set}")),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
@@ -2905,16 +2913,24 @@ fn update_task_running_metric_for_task(active_heals: &HashMap<String, Arc<HealTa
|
||||
.set(count as f64);
|
||||
}
|
||||
|
||||
fn running_erasure_set_counts(active_heals: &HashMap<String, Arc<HealTask>>) -> HashMap<String, usize> {
|
||||
fn running_heal_set_counts(active_heals: &HashMap<String, Arc<HealTask>>) -> HashMap<String, usize> {
|
||||
let mut running = HashMap::new();
|
||||
for task in active_heals.values() {
|
||||
if let HealType::ErasureSet { set_disk_id, .. } = &task.heal_type {
|
||||
*running.entry(set_disk_id.clone()).or_insert(0) += 1;
|
||||
if let Some(set_key) = heal_request_set_key_for_task(task) {
|
||||
*running.entry(set_key).or_insert(0) += 1;
|
||||
}
|
||||
}
|
||||
running
|
||||
}
|
||||
|
||||
fn heal_request_set_key_for_task(task: &HealTask) -> Option<String> {
|
||||
match &task.heal_type {
|
||||
HealType::ErasureSet { set_disk_id, .. } => Some(set_disk_id.clone()),
|
||||
HealType::Object { .. } => heal_options_set_key(&task.options),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn prune_completed_heal_statuses(completed_heals: &mut HashMap<String, CompletedHealStatus>) {
|
||||
let Ok(now) = SystemTime::now().duration_since(SystemTime::UNIX_EPOCH) else {
|
||||
return;
|
||||
@@ -3523,6 +3539,30 @@ mod tests {
|
||||
assert!(can_schedule_request(&request, &running, 2));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_can_schedule_scoped_object_request_respects_per_set_limit() {
|
||||
let options = HealOptions {
|
||||
pool_index: Some(0),
|
||||
set_index: Some(1),
|
||||
..Default::default()
|
||||
};
|
||||
let request = HealRequest::new(
|
||||
HealType::Object {
|
||||
bucket: "bucket".to_string(),
|
||||
object: "object".to_string(),
|
||||
version_id: None,
|
||||
},
|
||||
options,
|
||||
HealPriority::Normal,
|
||||
);
|
||||
|
||||
let mut running = HashMap::new();
|
||||
running.insert("pool_0_set_1".to_string(), 1);
|
||||
|
||||
assert!(!can_schedule_request(&request, &running, 1));
|
||||
assert!(can_schedule_request(&request, &running, 2));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_submit_heal_request_returns_merged_for_duplicate() {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
@@ -5136,7 +5176,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_running_erasure_set_counts_groups_only_erasure_tasks() {
|
||||
fn test_running_heal_set_counts_groups_set_scoped_tasks() {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
let erasure_task = Arc::new(HealTask::from_request(
|
||||
HealRequest::new(
|
||||
@@ -5149,6 +5189,23 @@ mod tests {
|
||||
),
|
||||
storage.clone(),
|
||||
));
|
||||
let scoped_options = HealOptions {
|
||||
pool_index: Some(0),
|
||||
set_index: Some(1),
|
||||
..Default::default()
|
||||
};
|
||||
let scoped_object_task = Arc::new(HealTask::from_request(
|
||||
HealRequest::new(
|
||||
HealType::Object {
|
||||
bucket: "bucket".to_string(),
|
||||
object: "scoped-object".to_string(),
|
||||
version_id: None,
|
||||
},
|
||||
scoped_options,
|
||||
HealPriority::Normal,
|
||||
),
|
||||
storage.clone(),
|
||||
));
|
||||
let object_task = Arc::new(HealTask::from_request(
|
||||
HealRequest::new(
|
||||
HealType::Object {
|
||||
@@ -5164,10 +5221,11 @@ mod tests {
|
||||
|
||||
let mut active = HashMap::new();
|
||||
active.insert(erasure_task.id.clone(), erasure_task);
|
||||
active.insert(scoped_object_task.id.clone(), scoped_object_task);
|
||||
active.insert(object_task.id.clone(), object_task);
|
||||
|
||||
let counts = running_erasure_set_counts(&active);
|
||||
assert_eq!(counts.get("pool_0_set_1"), Some(&1));
|
||||
let counts = running_heal_set_counts(&active);
|
||||
assert_eq!(counts.get("pool_0_set_1"), Some(&2));
|
||||
assert_eq!(counts.len(), 1);
|
||||
}
|
||||
|
||||
|
||||
@@ -18,6 +18,17 @@ use super::{
|
||||
};
|
||||
use crate::oidc::{OidcProviderConfig, OidcProviderSummary};
|
||||
|
||||
const DEFAULT_OIDC_PROVIDER_ID: &str = "default";
|
||||
|
||||
fn sorted_provider_summaries(mut providers: Vec<OidcProviderSummary>) -> Vec<OidcProviderSummary> {
|
||||
providers.sort_by(|left, right| {
|
||||
(left.provider_id != DEFAULT_OIDC_PROVIDER_ID)
|
||||
.cmp(&(right.provider_id != DEFAULT_OIDC_PROVIDER_ID))
|
||||
.then_with(|| left.provider_id.cmp(&right.provider_id))
|
||||
});
|
||||
providers
|
||||
}
|
||||
|
||||
pub struct FederatedIdentityService {
|
||||
registry: FederatedIdentityRegistry,
|
||||
}
|
||||
@@ -32,11 +43,11 @@ impl FederatedIdentityService {
|
||||
}
|
||||
|
||||
pub fn list_providers(&self) -> Vec<OidcProviderSummary> {
|
||||
self.registry.standard_oidc().list_providers()
|
||||
sorted_provider_summaries(self.registry.standard_oidc().list_providers())
|
||||
}
|
||||
|
||||
pub fn list_visible_providers(&self) -> Vec<OidcProviderSummary> {
|
||||
self.registry.standard_oidc().list_visible_providers()
|
||||
sorted_provider_summaries(self.registry.standard_oidc().list_visible_providers())
|
||||
}
|
||||
|
||||
pub fn get_provider_config(&self, id: &str) -> Option<&OidcProviderConfig> {
|
||||
@@ -158,6 +169,8 @@ mod tests {
|
||||
failure: ProviderFailure,
|
||||
events: Arc<Mutex<Vec<&'static str>>>,
|
||||
expected_logout: (&'static str, &'static str),
|
||||
listed_provider_ids: Vec<&'static str>,
|
||||
visible_provider_ids: Vec<&'static str>,
|
||||
}
|
||||
|
||||
impl TestProvider {
|
||||
@@ -165,11 +178,13 @@ mod tests {
|
||||
Self {
|
||||
with_policy: true,
|
||||
with_group: false,
|
||||
browser_provider_id: "default",
|
||||
web_provider_id: "default",
|
||||
browser_provider_id: DEFAULT_OIDC_PROVIDER_ID,
|
||||
web_provider_id: DEFAULT_OIDC_PROVIDER_ID,
|
||||
failure: ProviderFailure::None,
|
||||
events,
|
||||
expected_logout: ("default", "id-token"),
|
||||
expected_logout: (DEFAULT_OIDC_PROVIDER_ID, "id-token"),
|
||||
listed_provider_ids: Vec::new(),
|
||||
visible_provider_ids: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -203,6 +218,16 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
fn provider_summaries(provider_ids: &[&str]) -> Vec<OidcProviderSummary> {
|
||||
provider_ids
|
||||
.iter()
|
||||
.map(|provider_id| OidcProviderSummary {
|
||||
provider_id: (*provider_id).to_string(),
|
||||
display_name: (*provider_id).to_string(),
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl FederatedIdentityProvider for TestProvider {
|
||||
fn has_providers(&self) -> bool {
|
||||
@@ -210,11 +235,11 @@ mod tests {
|
||||
}
|
||||
|
||||
fn list_providers(&self) -> Vec<OidcProviderSummary> {
|
||||
Vec::new()
|
||||
provider_summaries(&self.listed_provider_ids)
|
||||
}
|
||||
|
||||
fn list_visible_providers(&self) -> Vec<OidcProviderSummary> {
|
||||
Vec::new()
|
||||
provider_summaries(&self.visible_provider_ids)
|
||||
}
|
||||
|
||||
fn provider_config(&self, _id: &str) -> Option<&OidcProviderConfig> {
|
||||
@@ -302,6 +327,39 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn provider_listing_puts_default_first_and_sorts_named_providers() {
|
||||
let events = Arc::new(Mutex::new(Vec::new()));
|
||||
let mut provider = TestProvider::new(events);
|
||||
provider.listed_provider_ids = vec!["zeta", "hidden", DEFAULT_OIDC_PROVIDER_ID, "alpha"];
|
||||
provider.visible_provider_ids = vec!["zeta", DEFAULT_OIDC_PROVIDER_ID, "alpha"];
|
||||
let service = FederatedIdentityService::new(FederatedIdentityRegistry::new(Arc::new(provider)));
|
||||
|
||||
assert_eq!(
|
||||
service
|
||||
.list_providers()
|
||||
.into_iter()
|
||||
.map(|provider| provider.provider_id)
|
||||
.collect::<Vec<_>>(),
|
||||
[DEFAULT_OIDC_PROVIDER_ID, "alpha", "hidden", "zeta"]
|
||||
);
|
||||
assert_eq!(
|
||||
service
|
||||
.list_visible_providers()
|
||||
.into_iter()
|
||||
.map(|provider| provider.provider_id)
|
||||
.collect::<Vec<_>>(),
|
||||
[DEFAULT_OIDC_PROVIDER_ID, "alpha", "zeta"]
|
||||
);
|
||||
assert_eq!(
|
||||
sorted_provider_summaries(provider_summaries(&["zeta", "alpha"]))
|
||||
.into_iter()
|
||||
.map(|provider| provider.provider_id)
|
||||
.collect::<Vec<_>>(),
|
||||
["alpha", "zeta"]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn callback_and_web_identity_preserve_provider_and_transaction_boundaries() {
|
||||
let events = Arc::new(Mutex::new(Vec::new()));
|
||||
|
||||
+345
-54
@@ -21,16 +21,16 @@
|
||||
use crate::oidc_state::{OidcAuthSession, OidcLogoutSession, OidcStateStore};
|
||||
use openidconnect::core::{CoreAuthenticationFlow, CoreClient, CoreIdToken, CoreJsonWebKeySet};
|
||||
use openidconnect::{
|
||||
AsyncHttpClient, Audience, AuthType, AuthorizationCode, ClientId, ClientSecret, CsrfToken, IssuerUrl, JsonWebKeySetUrl,
|
||||
LogoutRequest, Nonce, PkceCodeChallenge, PkceCodeVerifier, PostLogoutRedirectUrl, ProviderMetadataWithLogout, RedirectUrl,
|
||||
RequestTokenError, Scope,
|
||||
AsyncHttpClient, Audience, AuthType, AuthorizationCode, ClientId, ClientSecret, CsrfToken, DiscoveryError, IssuerUrl,
|
||||
JsonWebKeySetUrl, LogoutRequest, Nonce, PkceCodeChallenge, PkceCodeVerifier, PostLogoutRedirectUrl,
|
||||
ProviderMetadataWithLogout, RedirectUrl, RequestTokenError, Scope,
|
||||
};
|
||||
use reqwest::Client;
|
||||
use rustfs_config::oidc::*;
|
||||
use rustfs_config::server_config::{Config as ServerConfig, KVS};
|
||||
use rustfs_config::{DEFAULT_DELIMITER, ENABLE_KEY, EnableState, MAX_OIDC_RESPONSE_SIZE};
|
||||
use rustfs_policy::policy::{ClaimLookup, get_claim_case_insensitive};
|
||||
use rustfs_utils::egress::OutboundPolicy;
|
||||
use rustfs_utils::egress::{ENV_OUTBOUND_ALLOW_ORIGINS, OutboundPolicy, find_outbound_dns_policy_rejection};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::borrow::Cow;
|
||||
use std::collections::{HashMap, VecDeque};
|
||||
@@ -38,6 +38,8 @@ use std::fmt;
|
||||
use std::future::Future;
|
||||
use std::net::IpAddr;
|
||||
use std::pin::Pin;
|
||||
#[cfg(test)]
|
||||
use std::sync::Arc;
|
||||
use std::sync::{LazyLock, Mutex, MutexGuard, RwLock};
|
||||
use std::time::{Duration as StdDuration, Instant};
|
||||
use tokio::time::sleep;
|
||||
@@ -51,6 +53,7 @@ const EVENT_OIDC_HTTP: &str = "oidc_http";
|
||||
const OIDC_JWKS_REFRESH_INTERVAL: StdDuration = StdDuration::from_secs(24 * 60 * 60);
|
||||
const OIDC_DISCOVERY_TRANSPORT_RETRIES: usize = 3;
|
||||
const OIDC_DISCOVERY_TRANSPORT_RETRY_DELAY: StdDuration = StdDuration::from_millis(50);
|
||||
const OIDC_DISCOVERY_BLOCKED_BY_OUTBOUND_POLICY: &str = "OIDC provider discovery blocked by outbound policy";
|
||||
const OIDC_HTTP_REQUEST_TIMEOUT: StdDuration = StdDuration::from_secs(10);
|
||||
const OIDC_HTTP_CONNECT_TIMEOUT: StdDuration = StdDuration::from_secs(3);
|
||||
const OIDC_PLUGIN_AUTHN_WINDOW: StdDuration = StdDuration::from_secs(60);
|
||||
@@ -308,6 +311,8 @@ pub(crate) struct ReqwestHttpClient {
|
||||
/// `None` in production: the process-cached outbound policy from the environment is used.
|
||||
/// `Some(..)` only in tests, to explicitly allow a loopback mock endpoint.
|
||||
policy_override: Option<OutboundPolicy>,
|
||||
#[cfg(test)]
|
||||
dns_resolver_override: Option<Arc<dyn reqwest::dns::Resolve>>,
|
||||
}
|
||||
|
||||
/// Build a reqwest client pinned to the shared outbound egress policy for a single request.
|
||||
@@ -318,7 +323,11 @@ pub(crate) struct ReqwestHttpClient {
|
||||
/// connection so DNS rebinding fails closed. Redirects are not followed: a redirect target
|
||||
/// would otherwise skip URL-shape re-validation. The timeouts bound how long a slow or
|
||||
/// stalled provider can pin the calling task.
|
||||
fn build_oidc_http_client(uri: &str, policy_override: Option<&OutboundPolicy>) -> Result<Client, OidcHttpError> {
|
||||
fn build_oidc_http_client(
|
||||
uri: &str,
|
||||
policy_override: Option<&OutboundPolicy>,
|
||||
#[cfg(test)] dns_resolver_override: Option<Arc<dyn reqwest::dns::Resolve>>,
|
||||
) -> Result<(Client, Url), OidcHttpError> {
|
||||
let url = Url::parse(uri).map_err(|_| OidcHttpError::ForbiddenOutbound("invalid outbound OIDC URL".to_string()))?;
|
||||
let resolver = match policy_override {
|
||||
Some(policy) => policy.resolver_for(&url),
|
||||
@@ -326,17 +335,48 @@ fn build_oidc_http_client(uri: &str, policy_override: Option<&OutboundPolicy>) -
|
||||
.map_err(|err| OidcHttpError::ForbiddenOutbound(err.to_string()))?
|
||||
.resolver_for(&url),
|
||||
}
|
||||
.map_err(|err| OidcHttpError::ForbiddenOutbound(err.to_string()))?;
|
||||
.map_err(|err| {
|
||||
let base = err.to_string();
|
||||
let origin = url.origin().ascii_serialization();
|
||||
let can_allow_origin =
|
||||
OutboundPolicy::from_allowed_origins(&origin).is_ok_and(|allowlisted| allowlisted.validate_url(&url).is_ok());
|
||||
oidc_forbidden_outbound_error(&url, base, can_allow_origin)
|
||||
})?;
|
||||
let bypass_proxy = should_bypass_proxy_for_oidc_uri(uri);
|
||||
#[cfg(test)]
|
||||
let bypass_proxy = bypass_proxy || dns_resolver_override.is_some();
|
||||
#[cfg(test)]
|
||||
let resolver: Arc<dyn reqwest::dns::Resolve> = dns_resolver_override.unwrap_or_else(|| Arc::new(resolver));
|
||||
|
||||
let mut builder = reqwest::Client::builder()
|
||||
.dns_resolver(resolver)
|
||||
.redirect(reqwest::redirect::Policy::none())
|
||||
.timeout(OIDC_HTTP_REQUEST_TIMEOUT)
|
||||
.connect_timeout(OIDC_HTTP_CONNECT_TIMEOUT);
|
||||
if should_bypass_proxy_for_oidc_uri(uri) {
|
||||
if bypass_proxy {
|
||||
builder = builder.no_proxy();
|
||||
}
|
||||
builder.build().map_err(OidcHttpError::Reqwest)
|
||||
builder.build().map(|client| (client, url)).map_err(OidcHttpError::Reqwest)
|
||||
}
|
||||
|
||||
fn oidc_forbidden_outbound_error(url: &Url, base: String, can_allow_origin: bool) -> OidcHttpError {
|
||||
let reason = if can_allow_origin {
|
||||
let origin = url.origin().ascii_serialization();
|
||||
format!(
|
||||
"{base}; add {origin} to {ENV_OUTBOUND_ALLOW_ORIGINS} (comma-separated) and restart RustFS to allow this operator-owned OIDC provider (origin only, no path)"
|
||||
)
|
||||
} else {
|
||||
base
|
||||
};
|
||||
OidcHttpError::ForbiddenOutbound(reason)
|
||||
}
|
||||
|
||||
fn oidc_http_error_from_reqwest(url: &Url, error: reqwest::Error) -> OidcHttpError {
|
||||
if let Some(rejection) = find_outbound_dns_policy_rejection(&error) {
|
||||
let base = rejection.to_string();
|
||||
return oidc_forbidden_outbound_error(url, base, rejection.allow_origin_can_recover());
|
||||
}
|
||||
OidcHttpError::Reqwest(error)
|
||||
}
|
||||
|
||||
/// Buffer a provider response body, failing closed once `limit` bytes have been seen.
|
||||
@@ -370,7 +410,11 @@ fn should_bypass_proxy_for_oidc_uri(uri: &str) -> bool {
|
||||
|
||||
impl ReqwestHttpClient {
|
||||
fn new() -> Result<Self, String> {
|
||||
Ok(Self { policy_override: None })
|
||||
Ok(Self {
|
||||
policy_override: None,
|
||||
#[cfg(test)]
|
||||
dns_resolver_override: None,
|
||||
})
|
||||
}
|
||||
|
||||
/// Test-only constructor that pins outbound requests to an explicit policy, so a
|
||||
@@ -379,6 +423,15 @@ impl ReqwestHttpClient {
|
||||
fn with_policy(policy: OutboundPolicy) -> Self {
|
||||
Self {
|
||||
policy_override: Some(policy),
|
||||
dns_resolver_override: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn with_policy_and_dns_resolver(policy: OutboundPolicy, resolver: Arc<dyn reqwest::dns::Resolve>) -> Self {
|
||||
Self {
|
||||
policy_override: Some(policy),
|
||||
dns_resolver_override: Some(resolver),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -408,7 +461,12 @@ impl<'c> AsyncHttpClient<'c> for ReqwestHttpClient {
|
||||
);
|
||||
}
|
||||
|
||||
let client = build_oidc_http_client(&uri, self.policy_override.as_ref())?;
|
||||
let (client, url) = build_oidc_http_client(
|
||||
&uri,
|
||||
self.policy_override.as_ref(),
|
||||
#[cfg(test)]
|
||||
self.dns_resolver_override.clone(),
|
||||
)?;
|
||||
let response = client
|
||||
.request(parts.method, uri.clone())
|
||||
.headers(parts.headers)
|
||||
@@ -421,6 +479,7 @@ impl<'c> AsyncHttpClient<'c> for ReqwestHttpClient {
|
||||
OIDC_PLUGIN_AUTHN_METRICS.record(elapsed_ms, succeeded);
|
||||
|
||||
let response = response.map_err(|err| {
|
||||
let error = oidc_http_error_from_reqwest(&url, err);
|
||||
error!(
|
||||
event = EVENT_OIDC_HTTP,
|
||||
component = LOG_COMPONENT_IAM,
|
||||
@@ -429,10 +488,10 @@ impl<'c> AsyncHttpClient<'c> for ReqwestHttpClient {
|
||||
method = %method,
|
||||
uri = %uri,
|
||||
elapsed_ms,
|
||||
error = %err,
|
||||
error = %error,
|
||||
"oidc outbound http"
|
||||
);
|
||||
OidcHttpError::Reqwest(err)
|
||||
error
|
||||
})?;
|
||||
|
||||
let status = response.status();
|
||||
@@ -866,11 +925,11 @@ impl OidcSys {
|
||||
redirect_uri = %redirect_uri,
|
||||
request_error_kind = %request_error_kind,
|
||||
request_error_status = %request_error_status,
|
||||
error = %e,
|
||||
error = %err,
|
||||
"oidc token exchange failed"
|
||||
);
|
||||
format!(
|
||||
"token exchange failed: {e}: stage=token_request_failed, provider_id={}, config_url={}, issuer={}, token_endpoint={}, redirect_uri={}, client_id={}, request_error_kind={}, request_error_status={}",
|
||||
"token exchange failed: stage=token_request_failed, provider_id={}, config_url={}, issuer={}, token_endpoint={}, redirect_uri={}, client_id={}, request_error_kind={}, request_error_status={}, request_error={}",
|
||||
session.provider_id,
|
||||
config.config_url,
|
||||
issuer,
|
||||
@@ -878,7 +937,8 @@ impl OidcSys {
|
||||
redirect_uri,
|
||||
config.client_id,
|
||||
request_error_kind,
|
||||
request_error_status
|
||||
request_error_status,
|
||||
err
|
||||
)
|
||||
}
|
||||
RequestTokenError::Parse(parse_err, body) => {
|
||||
@@ -1656,17 +1716,18 @@ impl OidcSys {
|
||||
let issuer_url = IssuerUrl::new(candidate_issuer.clone()).map_err(|e| format!("invalid issuer URL: {e}"))?;
|
||||
|
||||
for attempt in 0..OIDC_DISCOVERY_TRANSPORT_RETRIES {
|
||||
match ProviderMetadataWithLogout::discover_async(issuer_url.clone(), http_client)
|
||||
.await
|
||||
.map_err(|e| format!("discovery failed: {e}"))
|
||||
{
|
||||
match ProviderMetadataWithLogout::discover_async(issuer_url.clone(), http_client).await {
|
||||
Ok(metadata) => {
|
||||
return Ok(ProviderState {
|
||||
metadata,
|
||||
discovered_at: Instant::now(),
|
||||
});
|
||||
}
|
||||
Err(DiscoveryError::Request(OidcHttpError::ForbiddenOutbound(reason))) => {
|
||||
return Err(format!("{OIDC_DISCOVERY_BLOCKED_BY_OUTBOUND_POLICY}: {reason}"));
|
||||
}
|
||||
Err(error) => {
|
||||
let error = format!("discovery failed: {error}");
|
||||
let is_transient_transport = error.contains("Request failed");
|
||||
let should_retry = is_transient_transport && attempt + 1 < OIDC_DISCOVERY_TRANSPORT_RETRIES;
|
||||
if should_retry {
|
||||
@@ -1728,10 +1789,13 @@ impl OidcSys {
|
||||
.body(Vec::new())
|
||||
.map_err(|err| format!("failed to prepare discovery request: {err}"))?;
|
||||
|
||||
let response = http_client
|
||||
.call(request)
|
||||
.await
|
||||
.map_err(|err| format!("discovery request failed: {err}"))?;
|
||||
let response = match http_client.call(request).await {
|
||||
Ok(response) => response,
|
||||
Err(OidcHttpError::ForbiddenOutbound(reason)) => {
|
||||
return Err(format!("{OIDC_DISCOVERY_BLOCKED_BY_OUTBOUND_POLICY}: {reason}"));
|
||||
}
|
||||
Err(err) => return Err(format!("discovery request failed: {err}")),
|
||||
};
|
||||
if response.status() != http::StatusCode::OK {
|
||||
return Err(format!("discovery failed: HTTP status code {} at {}", response.status(), discovery_url));
|
||||
}
|
||||
@@ -1747,9 +1811,13 @@ impl OidcSys {
|
||||
}
|
||||
|
||||
let jwks_url = jwks_url_from_config_url(&config.config_url, &issuer_url, provider_metadata.jwks_uri())?;
|
||||
let jwks = CoreJsonWebKeySet::fetch_async(&jwks_url, http_client)
|
||||
.await
|
||||
.map_err(|err| format!("failed to fetch JWKS: {err}"))?;
|
||||
let jwks = match CoreJsonWebKeySet::fetch_async(&jwks_url, http_client).await {
|
||||
Ok(jwks) => jwks,
|
||||
Err(DiscoveryError::Request(OidcHttpError::ForbiddenOutbound(reason))) => {
|
||||
return Err(format!("JWKS request blocked by outbound policy: {reason}"));
|
||||
}
|
||||
Err(err) => return Err(format!("failed to fetch JWKS: {err}")),
|
||||
};
|
||||
|
||||
Ok(ProviderState {
|
||||
metadata: provider_metadata.set_jwks(jwks),
|
||||
@@ -2046,6 +2114,25 @@ pub(crate) fn test_config(id: &str) -> OidcProviderConfig {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use rustfs_utils::egress::OutboundDnsPolicyRejection;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
||||
#[derive(Clone)]
|
||||
struct RejectingDnsResolver {
|
||||
allow_origin_can_recover: bool,
|
||||
calls: Option<Arc<AtomicUsize>>,
|
||||
}
|
||||
|
||||
impl reqwest::dns::Resolve for RejectingDnsResolver {
|
||||
fn resolve(&self, name: reqwest::dns::Name) -> reqwest::dns::Resolving {
|
||||
if let Some(calls) = &self.calls {
|
||||
calls.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
let host = name.as_str().to_string();
|
||||
let rejection = OutboundDnsPolicyRejection::new(host, self.allow_origin_can_recover);
|
||||
Box::pin(async move { Err(std::io::Error::new(std::io::ErrorKind::PermissionDenied, rejection).into()) })
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_string_claim() {
|
||||
@@ -2853,6 +2940,32 @@ mod tests {
|
||||
assert!(sys.list_providers().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_oidc_http_client_rejects_forbidden_targets_without_allowlist() {
|
||||
// Cloud metadata endpoint is never allowed.
|
||||
assert!(
|
||||
matches!(
|
||||
build_oidc_http_client("http://169.254.169.254/latest/meta-data/", None, None),
|
||||
Err(OidcHttpError::ForbiddenOutbound(_))
|
||||
),
|
||||
"metadata endpoint must be rejected"
|
||||
);
|
||||
// Loopback is rejected by default (no allow-origins configured).
|
||||
assert!(
|
||||
matches!(
|
||||
build_oidc_http_client("http://127.0.0.1:8080/.well-known/openid-configuration", None, None),
|
||||
Err(OidcHttpError::ForbiddenOutbound(_))
|
||||
),
|
||||
"loopback must be rejected by default"
|
||||
);
|
||||
// A public hostname passes the up-front shape/host check; the resolved IP is still
|
||||
// re-classified at connection time by the pinned resolver.
|
||||
assert!(
|
||||
build_oidc_http_client("https://accounts.example.com/.well-known/openid-configuration", None, None).is_ok(),
|
||||
"public https endpoint should build"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_should_bypass_proxy_for_oidc_uri_loopback_only() {
|
||||
assert!(should_bypass_proxy_for_oidc_uri("http://127.0.0.1:9000/.well-known/openid-configuration"));
|
||||
@@ -2864,49 +2977,227 @@ mod tests {
|
||||
assert!(!should_bypass_proxy_for_oidc_uri("not-a-url"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_oidc_http_client_rejects_forbidden_targets_without_allowlist() {
|
||||
// Cloud metadata endpoint is never allowed.
|
||||
assert!(
|
||||
matches!(
|
||||
build_oidc_http_client("http://169.254.169.254/latest/meta-data/", None),
|
||||
Err(OidcHttpError::ForbiddenOutbound(_))
|
||||
),
|
||||
"metadata endpoint must be rejected"
|
||||
);
|
||||
// Loopback is rejected by default (no allow-origins configured).
|
||||
assert!(
|
||||
matches!(
|
||||
build_oidc_http_client("http://127.0.0.1:8080/.well-known/openid-configuration", None),
|
||||
Err(OidcHttpError::ForbiddenOutbound(_))
|
||||
),
|
||||
"loopback must be rejected by default"
|
||||
);
|
||||
// A public hostname passes the up-front shape/host check; the resolved IP is still
|
||||
// re-classified at connection time by the pinned resolver.
|
||||
assert!(
|
||||
build_oidc_http_client("https://accounts.example.com/.well-known/openid-configuration", None).is_ok(),
|
||||
"public https endpoint should build"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_oidc_http_client_honors_explicit_allowlist_for_loopback() {
|
||||
let policy = OutboundPolicy::from_allowed_origins("http://127.0.0.1:8080").expect("origin should parse");
|
||||
assert!(
|
||||
build_oidc_http_client("http://127.0.0.1:8080/.well-known/openid-configuration", Some(&policy)).is_ok(),
|
||||
build_oidc_http_client("http://127.0.0.1:8080/.well-known/openid-configuration", Some(&policy), None).is_ok(),
|
||||
"explicitly allow-listed loopback origin should build"
|
||||
);
|
||||
// A metadata endpoint stays forbidden even when a loopback origin is allow-listed.
|
||||
assert!(
|
||||
matches!(
|
||||
build_oidc_http_client("http://169.254.169.254/", Some(&policy)),
|
||||
build_oidc_http_client("http://169.254.169.254/", Some(&policy), None),
|
||||
Err(OidcHttpError::ForbiddenOutbound(_))
|
||||
),
|
||||
"metadata endpoint stays forbidden despite an unrelated allow-list entry"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn oidc_discovery_reports_forbidden_outbound_without_retrying() {
|
||||
let config_url = "http://192.168.65.254:8080/realms/rustfs/.well-known/openid-configuration";
|
||||
let config = build_mocked_oidc_provider_config("default", config_url);
|
||||
let http_client = ReqwestHttpClient::with_policy(OutboundPolicy::default());
|
||||
|
||||
let error = match OidcSys::discover_provider(&config, &http_client).await {
|
||||
Ok(_) => panic!("private OIDC provider should require an explicit allowlist origin"),
|
||||
Err(error) => error,
|
||||
};
|
||||
|
||||
assert!(error.contains("OIDC provider discovery blocked by outbound policy"));
|
||||
assert!(error.contains(&format!("add http://192.168.65.254:8080 to {ENV_OUTBOUND_ALLOW_ORIGINS}")));
|
||||
assert!(!error.contains("discovery failed for all issuer variants"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn oidc_explicit_issuer_reports_forbidden_discovery_endpoint() {
|
||||
let config_url = "http://192.168.65.254:8080/realms/rustfs/.well-known/openid-configuration";
|
||||
let mut config = build_mocked_oidc_provider_config("default", config_url);
|
||||
config.issuer = Some("https://idp.example.com/realms/rustfs".to_string());
|
||||
let http_client = ReqwestHttpClient::with_policy(OutboundPolicy::default());
|
||||
|
||||
let error = match OidcSys::discover_provider(&config, &http_client).await {
|
||||
Ok(_) => panic!("private OIDC provider should require an explicit allowlist origin"),
|
||||
Err(error) => error,
|
||||
};
|
||||
|
||||
assert!(error.contains("OIDC provider discovery blocked by outbound policy"));
|
||||
assert!(error.contains(&format!("add http://192.168.65.254:8080 to {ENV_OUTBOUND_ALLOW_ORIGINS}")));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn oidc_explicit_issuer_reports_forbidden_jwks_endpoint() {
|
||||
let Some((base, handle)) = start_mock_oidc_discovery_server(
|
||||
|base| {
|
||||
(
|
||||
format!("{base}/realms/rustfs"),
|
||||
"http://192.168.65.254:8080/realms/rustfs/protocol/openid-connect/certs".to_string(),
|
||||
"/unused".to_string(),
|
||||
)
|
||||
},
|
||||
1,
|
||||
) else {
|
||||
return;
|
||||
};
|
||||
let mut config =
|
||||
build_mocked_oidc_provider_config("default", &format!("{base}/realms/rustfs/.well-known/openid-configuration"));
|
||||
config.issuer = Some(format!("{base}/realms/rustfs"));
|
||||
let policy = OutboundPolicy::from_allowed_origins(&base).expect("loopback discovery origin should be allowed");
|
||||
let http_client = ReqwestHttpClient::with_policy(policy);
|
||||
|
||||
let error = match OidcSys::discover_provider(&config, &http_client).await {
|
||||
Ok(_) => panic!("private JWKS endpoint should require an explicit allowlist origin"),
|
||||
Err(error) => error,
|
||||
};
|
||||
|
||||
assert!(error.contains("JWKS request blocked by outbound policy"));
|
||||
assert!(error.contains(&format!("add http://192.168.65.254:8080 to {ENV_OUTBOUND_ALLOW_ORIGINS}")));
|
||||
assert!(handle.join().is_ok());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn oidc_token_exchange_reports_forbidden_token_endpoint() {
|
||||
let provider_id = "default";
|
||||
let token_endpoint = "http://192.168.65.254:8080/realms/rustfs/protocol/openid-connect/token";
|
||||
let metadata = serde_json::from_value::<ProviderMetadataWithLogout>(serde_json::json!({
|
||||
"issuer": "https://idp.example.com/realms/rustfs",
|
||||
"authorization_endpoint": "https://idp.example.com/realms/rustfs/protocol/openid-connect/auth",
|
||||
"token_endpoint": token_endpoint,
|
||||
"jwks_uri": "https://idp.example.com/realms/rustfs/protocol/openid-connect/certs",
|
||||
"response_types_supported": ["code"],
|
||||
"subject_types_supported": ["public"],
|
||||
"id_token_signing_alg_values_supported": ["RS256"]
|
||||
}))
|
||||
.expect("provider metadata should parse");
|
||||
let config = build_mocked_oidc_provider_config(
|
||||
provider_id,
|
||||
"https://idp.example.com/realms/rustfs/.well-known/openid-configuration",
|
||||
);
|
||||
let state_store = OidcStateStore::new();
|
||||
state_store
|
||||
.insert(
|
||||
"test-state".to_string(),
|
||||
OidcAuthSession {
|
||||
provider_id: provider_id.to_string(),
|
||||
pkce_verifier: "test-pkce-verifier".to_string(),
|
||||
nonce: "test-nonce".to_string(),
|
||||
redirect_after: None,
|
||||
},
|
||||
)
|
||||
.await;
|
||||
let sys = OidcSys {
|
||||
configs: HashMap::from([(provider_id.to_string(), config)]),
|
||||
provider_states: RwLock::new(HashMap::from([(
|
||||
provider_id.to_string(),
|
||||
ProviderState {
|
||||
metadata,
|
||||
discovered_at: Instant::now(),
|
||||
},
|
||||
)])),
|
||||
state_store,
|
||||
http_client: ReqwestHttpClient::with_policy(OutboundPolicy::default()),
|
||||
};
|
||||
|
||||
let error = match sys
|
||||
.exchange_code("test-state", "test-code", "https://console.example.com/oauth_callback")
|
||||
.await
|
||||
{
|
||||
Ok(_) => panic!("private token endpoint should require an explicit allowlist origin"),
|
||||
Err(error) => error,
|
||||
};
|
||||
|
||||
assert!(error.contains("request_error_kind=forbidden_outbound"));
|
||||
assert!(error.contains(&format!("add http://192.168.65.254:8080 to {ENV_OUTBOUND_ALLOW_ORIGINS}")));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn oidc_reqwest_dns_policy_rejection_stays_typed() {
|
||||
let calls = Arc::new(AtomicUsize::new(0));
|
||||
let config = build_mocked_oidc_provider_config(
|
||||
"default",
|
||||
"http://keycloak.internal:8080/realms/rustfs/.well-known/openid-configuration",
|
||||
);
|
||||
let http_client = ReqwestHttpClient::with_policy_and_dns_resolver(
|
||||
OutboundPolicy::default(),
|
||||
Arc::new(RejectingDnsResolver {
|
||||
allow_origin_can_recover: true,
|
||||
calls: Some(calls.clone()),
|
||||
}),
|
||||
);
|
||||
|
||||
let error = match OidcSys::discover_provider(&config, &http_client).await {
|
||||
Ok(_) => panic!("private DNS answer should fail discovery"),
|
||||
Err(error) => error,
|
||||
};
|
||||
|
||||
assert!(error.contains("OIDC provider discovery blocked by outbound policy"));
|
||||
assert!(error.contains(&format!("add http://keycloak.internal:8080 to {ENV_OUTBOUND_ALLOW_ORIGINS}")));
|
||||
assert_eq!(calls.load(Ordering::Relaxed), 1, "policy rejection must not be retried");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn oidc_explicit_issuer_preserves_dns_policy_rejection() {
|
||||
let calls = Arc::new(AtomicUsize::new(0));
|
||||
let mut config = build_mocked_oidc_provider_config(
|
||||
"default",
|
||||
"http://keycloak.internal:8080/realms/rustfs/.well-known/openid-configuration",
|
||||
);
|
||||
config.issuer = Some("https://idp.example.com/realms/rustfs".to_string());
|
||||
let http_client = ReqwestHttpClient::with_policy_and_dns_resolver(
|
||||
OutboundPolicy::default(),
|
||||
Arc::new(RejectingDnsResolver {
|
||||
allow_origin_can_recover: true,
|
||||
calls: Some(calls.clone()),
|
||||
}),
|
||||
);
|
||||
|
||||
let error = match OidcSys::discover_provider(&config, &http_client).await {
|
||||
Ok(_) => panic!("private DNS answer should fail discovery"),
|
||||
Err(error) => error,
|
||||
};
|
||||
|
||||
assert!(error.contains("OIDC provider discovery blocked by outbound policy"));
|
||||
assert!(error.contains(&format!("add http://keycloak.internal:8080 to {ENV_OUTBOUND_ALLOW_ORIGINS}")));
|
||||
assert_eq!(calls.load(Ordering::Relaxed), 1, "policy rejection must not be retried");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn oidc_nonrecoverable_dns_policy_rejection_omits_allowlist_hint() {
|
||||
let uri = "http://metadata.internal/latest/meta-data";
|
||||
let http_client = ReqwestHttpClient::with_policy_and_dns_resolver(
|
||||
OutboundPolicy::default(),
|
||||
Arc::new(RejectingDnsResolver {
|
||||
allow_origin_can_recover: false,
|
||||
calls: None,
|
||||
}),
|
||||
);
|
||||
let request = http::Request::builder()
|
||||
.uri(uri)
|
||||
.body(Vec::new())
|
||||
.expect("request should build");
|
||||
|
||||
let error = http_client
|
||||
.call(request)
|
||||
.await
|
||||
.expect_err("metadata DNS answer should be rejected");
|
||||
let message = error.to_string();
|
||||
|
||||
assert!(matches!(error, OidcHttpError::ForbiddenOutbound(_)));
|
||||
assert!(message.contains("metadata.internal"));
|
||||
assert!(!message.contains(&format!("add http://metadata.internal to {ENV_OUTBOUND_ALLOW_ORIGINS}")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn oidc_metadata_endpoint_rejection_does_not_offer_allowlist_bypass() {
|
||||
let error = build_oidc_http_client("http://169.254.169.254/latest/meta-data/", Some(&OutboundPolicy::default()), None)
|
||||
.expect_err("metadata endpoint must remain forbidden");
|
||||
let message = error.to_string();
|
||||
|
||||
assert!(message.contains("metadata endpoint"));
|
||||
assert!(!message.contains(&format!("add http://169.254.169.254 to {ENV_OUTBOUND_ALLOW_ORIGINS}")));
|
||||
}
|
||||
|
||||
/// Serve exactly `body_len` bytes with no `Content-Length`, so the body ends only at EOF
|
||||
/// and the size guard cannot rely on an advertised length.
|
||||
fn start_unbounded_body_server(body_len: usize) -> Option<(String, std::thread::JoinHandle<()>)> {
|
||||
|
||||
@@ -97,7 +97,6 @@ swift = [
|
||||
"dep:serde",
|
||||
"dep:urlencoding",
|
||||
"dep:md-5",
|
||||
"dep:quick-xml",
|
||||
"dep:hmac",
|
||||
"dep:sha1",
|
||||
"dep:hex",
|
||||
@@ -162,7 +161,6 @@ tokio-util = { workspace = true, optional = true, features = ["rt", "io", "compa
|
||||
serde = { workspace = true, optional = true, features = ["derive"] }
|
||||
urlencoding = { workspace = true, optional = true }
|
||||
md-5 = { workspace = true, optional = true }
|
||||
quick-xml = { workspace = true, optional = true, features = ["serialize"] }
|
||||
hmac = { workspace = true, optional = true }
|
||||
sha1 = { workspace = true, optional = true }
|
||||
hex = { workspace = true, optional = true }
|
||||
|
||||
@@ -17,6 +17,14 @@
|
||||
pub const WALK_DIR_STREAM_COMPLETION_QUERY: &str = "walk_dir_stream_completion";
|
||||
pub const WALK_DIR_STREAM_COMPLETION_V1: &str = "error-v1";
|
||||
pub const WALK_DIR_BODY_SHA256_QUERY: &str = "walk_dir_body_sha256";
|
||||
pub const PUT_FILE_AUTH_QUERY: &str = "put_file_auth";
|
||||
pub const PUT_FILE_AUTH_V1: &str = "digest-trailer-v1";
|
||||
pub const PUT_FILE_NONCE_QUERY: &str = "put_file_nonce";
|
||||
pub const PUT_FILE_AUTH_TRAILER_MAGIC: &[u8; 16] = b"RFS-PUT-AUTH-V1\0";
|
||||
pub const PUT_FILE_AUTH_TRAILER_DIGEST_LEN: usize = 64;
|
||||
pub const PUT_FILE_AUTH_TRAILER_MAC_LEN: usize = 32;
|
||||
pub const PUT_FILE_AUTH_TRAILER_LEN: usize =
|
||||
PUT_FILE_AUTH_TRAILER_MAGIC.len() + PUT_FILE_AUTH_TRAILER_DIGEST_LEN + PUT_FILE_AUTH_TRAILER_MAC_LEN;
|
||||
pub const NS_SCANNER_BODY_SHA256_QUERY: &str = "ns_scanner_body_sha256";
|
||||
pub const NS_SCANNER_CAPABILITY_CHALLENGE_QUERY: &str = "ns_scanner_challenge";
|
||||
pub const NS_SCANNER_CYCLE_QUERY: &str = "ns_scanner_cycle";
|
||||
|
||||
+172
-16
@@ -82,6 +82,59 @@ impl fmt::Display for OutboundPolicyError {
|
||||
|
||||
impl std::error::Error for OutboundPolicyError {}
|
||||
|
||||
/// A DNS answer was rejected at the connection boundary because every resolved
|
||||
/// address was forbidden by the outbound policy.
|
||||
#[derive(Debug)]
|
||||
pub struct OutboundDnsPolicyRejection {
|
||||
host: String,
|
||||
allow_origin_can_recover: bool,
|
||||
}
|
||||
|
||||
impl OutboundDnsPolicyRejection {
|
||||
/// Records the rejected host and whether an exact operator allowlist origin
|
||||
/// could permit at least one of its resolved addresses.
|
||||
pub fn new(host: String, allow_origin_can_recover: bool) -> Self {
|
||||
Self {
|
||||
host,
|
||||
allow_origin_can_recover,
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether an exact allowlist origin could permit at least one rejected address.
|
||||
pub fn allow_origin_can_recover(&self) -> bool {
|
||||
self.allow_origin_can_recover
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for OutboundDnsPolicyRejection {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
write!(f, "outbound DNS resolution for '{}' returned no allowed addresses", self.host)
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for OutboundDnsPolicyRejection {}
|
||||
|
||||
/// Finds a typed DNS policy rejection through transport error wrappers.
|
||||
pub fn find_outbound_dns_policy_rejection<'a>(
|
||||
error: &'a (dyn std::error::Error + 'static),
|
||||
) -> Option<&'a OutboundDnsPolicyRejection> {
|
||||
let mut current = Some(error);
|
||||
while let Some(error) = current {
|
||||
if let Some(rejection) = error.downcast_ref::<OutboundDnsPolicyRejection>() {
|
||||
return Some(rejection);
|
||||
}
|
||||
if let Some(rejection) = error
|
||||
.downcast_ref::<std::io::Error>()
|
||||
.and_then(std::io::Error::get_ref)
|
||||
.and_then(|inner| inner.downcast_ref::<OutboundDnsPolicyRejection>())
|
||||
{
|
||||
return Some(rejection);
|
||||
}
|
||||
current = error.source();
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, PartialEq, Eq)]
|
||||
pub struct OutboundPolicy {
|
||||
allowed_restricted_origins: HashSet<String>,
|
||||
@@ -226,14 +279,29 @@ impl reqwest::dns::Resolve for OutboundDnsResolver {
|
||||
.map_err(|err| std::io::Error::new(std::io::ErrorKind::NotFound, err))?
|
||||
.collect()
|
||||
};
|
||||
if addresses.is_empty() {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::NotFound,
|
||||
format!("DNS resolution for '{host}' returned no addresses"),
|
||||
)
|
||||
.into());
|
||||
}
|
||||
let mut allow_origin_can_recover = false;
|
||||
let addrs = addresses
|
||||
.into_iter()
|
||||
.filter(|address| resolved_ip_allowed(address.ip(), allow_restricted))
|
||||
.filter(|address| match validate_policy_ip(address.ip()) {
|
||||
Ok(()) => true,
|
||||
Err(reason) => {
|
||||
let recoverable = restricted_reason_can_be_overridden(reason);
|
||||
allow_origin_can_recover |= recoverable;
|
||||
allow_restricted && recoverable
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
if addrs.is_empty() {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::PermissionDenied,
|
||||
format!("outbound DNS resolution for '{host}' returned no allowed addresses"),
|
||||
OutboundDnsPolicyRejection::new(host, allow_origin_can_recover),
|
||||
)
|
||||
.into());
|
||||
}
|
||||
@@ -302,13 +370,6 @@ fn restricted_reason_can_be_overridden(reason: &str) -> bool {
|
||||
)
|
||||
}
|
||||
|
||||
fn resolved_ip_allowed(ip: IpAddr, allow_restricted: bool) -> bool {
|
||||
match validate_policy_ip(ip) {
|
||||
Ok(()) => true,
|
||||
Err(reason) => allow_restricted && restricted_reason_can_be_overridden(reason),
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_policy_ip(ip: IpAddr) -> Result<(), &'static str> {
|
||||
if is_metadata_endpoint(ip) {
|
||||
return Err("metadata endpoint");
|
||||
@@ -488,7 +549,7 @@ fn embedded_ipv4(v6: Ipv6Addr) -> Option<Ipv4Addr> {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{OutboundPolicy, OutboundUrlError, validate_outbound_url};
|
||||
use super::{OutboundPolicy, OutboundUrlError, find_outbound_dns_policy_rejection, validate_outbound_url};
|
||||
use std::collections::HashMap;
|
||||
use std::net::SocketAddr;
|
||||
use url::Url;
|
||||
@@ -835,12 +896,38 @@ mod tests {
|
||||
.map(|addr| addr.ip())
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(addrs, vec!["8.8.8.8".parse::<std::net::IpAddr>().expect("public IP")]);
|
||||
assert!(
|
||||
reqwest::dns::Resolve::resolve(&resolver, "rebound.test".parse().expect("resolver hostname"))
|
||||
.await
|
||||
.is_err(),
|
||||
"a rebound answer containing only restricted addresses must fail closed"
|
||||
);
|
||||
let error = match reqwest::dns::Resolve::resolve(&resolver, "rebound.test".parse().expect("resolver hostname")).await {
|
||||
Ok(_) => panic!("a rebound answer containing only restricted addresses must fail closed"),
|
||||
Err(error) => error,
|
||||
};
|
||||
let io_error = error
|
||||
.downcast_ref::<std::io::Error>()
|
||||
.expect("resolver must preserve its PermissionDenied root error");
|
||||
assert_eq!(io_error.kind(), std::io::ErrorKind::PermissionDenied);
|
||||
let rejection =
|
||||
find_outbound_dns_policy_rejection(&*error).expect("typed policy rejection must remain in the error chain");
|
||||
assert_eq!(rejection.host, "rebound.test");
|
||||
assert!(rejection.allow_origin_can_recover());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn outbound_dns_resolver_does_not_classify_empty_answers_as_policy_rejections() {
|
||||
let endpoint = Url::parse("https://empty.test/hook").expect("endpoint should parse");
|
||||
let resolver = OutboundPolicy::default()
|
||||
.resolver_for(&endpoint)
|
||||
.expect("public hostname should be accepted")
|
||||
.with_overrides(HashMap::from([("empty.test".to_string(), Vec::new())]));
|
||||
|
||||
let error = match reqwest::dns::Resolve::resolve(&resolver, "empty.test".parse().expect("resolver hostname")).await {
|
||||
Ok(_) => panic!("an empty DNS answer must fail"),
|
||||
Err(error) => error,
|
||||
};
|
||||
let io_error = error
|
||||
.downcast_ref::<std::io::Error>()
|
||||
.expect("resolver errors must remain I/O errors");
|
||||
|
||||
assert_eq!(io_error.kind(), std::io::ErrorKind::NotFound);
|
||||
assert!(find_outbound_dns_policy_rejection(&*error).is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -976,4 +1063,73 @@ mod tests {
|
||||
"request must fail in the DNS policy layer: {error_chain:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reqwest_preserves_recoverable_private_dns_policy_rejection() {
|
||||
let endpoint = Url::parse("http://keycloak.internal:8080/realms/rustfs").expect("endpoint should parse");
|
||||
let resolver = OutboundPolicy::default()
|
||||
.resolver_for(&endpoint)
|
||||
.expect("hostname should pass the URL-shape check")
|
||||
.with_overrides(HashMap::from([(
|
||||
"keycloak.internal".to_string(),
|
||||
vec![
|
||||
"10.96.0.20".parse().expect("private IP"),
|
||||
"169.254.169.254".parse().expect("metadata IP"),
|
||||
],
|
||||
)]));
|
||||
let client = reqwest::Client::builder()
|
||||
.no_proxy()
|
||||
.dns_resolver(resolver)
|
||||
.timeout(std::time::Duration::from_secs(2))
|
||||
.build()
|
||||
.expect("test client should build");
|
||||
|
||||
let error = client
|
||||
.get(endpoint)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("private DNS answer should be rejected before connecting");
|
||||
let rejection = find_outbound_dns_policy_rejection(&error).expect("typed DNS policy rejection should be preserved");
|
||||
let mut current: Option<&(dyn std::error::Error + 'static)> = Some(&error);
|
||||
let mut io_kind = None;
|
||||
while let Some(source) = current {
|
||||
if let Some(io_error) = source.downcast_ref::<std::io::Error>() {
|
||||
io_kind = Some(io_error.kind());
|
||||
break;
|
||||
}
|
||||
current = source.source();
|
||||
}
|
||||
|
||||
assert_eq!(rejection.host, "keycloak.internal");
|
||||
assert!(rejection.allow_origin_can_recover());
|
||||
assert_eq!(io_kind, Some(std::io::ErrorKind::PermissionDenied));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reqwest_preserves_nonrecoverable_metadata_dns_policy_rejection() {
|
||||
let endpoint = Url::parse("http://metadata.internal/latest").expect("endpoint should parse");
|
||||
let resolver = OutboundPolicy::default()
|
||||
.resolver_for(&endpoint)
|
||||
.expect("hostname should pass the URL-shape check")
|
||||
.with_overrides(HashMap::from([(
|
||||
"metadata.internal".to_string(),
|
||||
vec!["169.254.169.254".parse().expect("metadata IP")],
|
||||
)]));
|
||||
let client = reqwest::Client::builder()
|
||||
.no_proxy()
|
||||
.dns_resolver(resolver)
|
||||
.timeout(std::time::Duration::from_secs(2))
|
||||
.build()
|
||||
.expect("test client should build");
|
||||
|
||||
let error = client
|
||||
.get(endpoint)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("metadata DNS answer should be rejected before connecting");
|
||||
let rejection = find_outbound_dns_policy_rejection(&error).expect("typed DNS policy rejection should be preserved");
|
||||
|
||||
assert_eq!(rejection.host, "metadata.internal");
|
||||
assert!(!rejection.allow_origin_can_recover());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -154,9 +154,11 @@ If RustFS reaches Keycloak through an internal URL while tokens use a public iss
|
||||
```bash
|
||||
export RUSTFS_IDENTITY_OPENID_CONFIG_URL="http://keycloak.keycloak.svc.cluster.local:8080/realms/rustfs/.well-known/openid-configuration"
|
||||
export RUSTFS_IDENTITY_OPENID_ISSUER="https://keycloak.example.com/realms/rustfs"
|
||||
export RUSTFS_OUTBOUND_ALLOW_ORIGINS="http://keycloak.keycloak.svc.cluster.local:8080"
|
||||
```
|
||||
|
||||
Discovery and issuer-relative JWKS requests use the internal `CONFIG_URL` base. ID token issuer validation still uses `ISSUER`.
|
||||
The outbound allowlist entry is the exact internal origin only; do not include the realm or discovery path. RustFS reads this process setting at startup, so restart every RustFS node after changing it.
|
||||
Use HTTPS with a trusted CA for the internal URL whenever possible. Discovery and JWKS define the token-signing trust root; use HTTP only on a network where DNS and traffic cannot be tampered with, because a compromised response can authorize forged tokens.
|
||||
|
||||
For short-lived connectivity testing only, you may temporarily add:
|
||||
@@ -287,6 +289,7 @@ Expected flow:
|
||||
| Groups appear as `/consoleAdmin` | Keycloak `Full group path` is enabled | Disable `Full group path`. |
|
||||
| Console redirects to an internal host | Missing `RUSTFS_BROWSER_REDIRECT_URL` or incorrect proxy headers | Set `RUSTFS_BROWSER_REDIRECT_URL` to the public browser origin. |
|
||||
| Invalid or expired OIDC state | Callback reached a different RustFS node | Configure load-balancer session affinity for authorize and callback requests. |
|
||||
| OIDC provider or login button is missing after upgrading to beta.12+ | The internal Keycloak origin is blocked by the outbound policy | Add the exact `scheme://host:port` origin to `RUSTFS_OUTBOUND_ALLOW_ORIGINS` and restart every RustFS node. |
|
||||
|
||||
## 7. Production Checklist
|
||||
|
||||
@@ -299,3 +302,4 @@ Expected flow:
|
||||
- [ ] `role_policy=consoleAdmin` is not used as a permanent production shortcut.
|
||||
- [ ] The load balancer preserves query strings.
|
||||
- [ ] OIDC authorize and callback requests have session affinity to the same RustFS node.
|
||||
- [ ] Internal Keycloak origins are listed exactly in `RUSTFS_OUTBOUND_ALLOW_ORIGINS` on every RustFS node.
|
||||
|
||||
@@ -4,10 +4,11 @@ This document describes the outbound connection policy that RustFS applies to
|
||||
server-initiated HTTP(S) requests, and the `RUSTFS_OUTBOUND_ALLOW_ORIGINS`
|
||||
allowlist operators can use to reach endpoints on private or container networks.
|
||||
|
||||
It is written for operators who upgraded to `1.0.0-beta.11` (or later) and found
|
||||
that event-notification webhooks, audit webhooks, or other outbound integrations
|
||||
stopped reaching endpoints that worked before — typically Docker Compose service
|
||||
names, `host.docker.internal`, or RFC 1918 addresses.
|
||||
It is written for operators whose outbound integrations stopped reaching
|
||||
endpoints after an upgrade — typically Docker Compose service names,
|
||||
`host.docker.internal`, or RFC 1918 addresses. Webhook and audit clients adopted
|
||||
this policy in `1.0.0-beta.11`; OIDC provider requests adopted it in
|
||||
`1.0.0-beta.12`.
|
||||
|
||||
## Background: what the policy protects
|
||||
|
||||
@@ -21,14 +22,16 @@ The policy governs the outbound clients used by:
|
||||
|
||||
- event-notification webhooks (`RUSTFS_NOTIFY_WEBHOOK_*`);
|
||||
- audit webhooks (`RUSTFS_AUDIT_WEBHOOK_*`);
|
||||
- OIDC identity-provider requests;
|
||||
- OIDC identity-provider discovery, JWKS, and token requests (since `1.0.0-beta.12`);
|
||||
- S3 tiering (warm-backend) endpoints;
|
||||
- Keystone auth URLs.
|
||||
|
||||
The webhook and audit outbound clients also **disable proxies and do not follow
|
||||
redirects**, so the destination must be reachable directly at the configured URL.
|
||||
|
||||
## What changed in beta.11
|
||||
## What changed in beta.11 (and for OIDC in beta.12)
|
||||
|
||||
For webhook and audit clients:
|
||||
|
||||
| | beta.10 | beta.11+ |
|
||||
|---|---|---|
|
||||
@@ -43,6 +46,11 @@ exact origin is on the allowlist. This is why a Compose setup that delivered
|
||||
events on beta.10 can go silent after the upgrade even though the configuration
|
||||
is unchanged.
|
||||
|
||||
OIDC joined the same policy in beta.12. An internal identity provider that
|
||||
worked in beta.11 can therefore fail discovery after upgrading to beta.12 unless
|
||||
its exact origin is allowlisted. The policy remains active for discovery, JWKS,
|
||||
and token requests.
|
||||
|
||||
## Symptoms
|
||||
|
||||
- Bucket event rules and webhook configuration look correct.
|
||||
@@ -52,6 +60,9 @@ is unchanged.
|
||||
a loopback, private, shared, or reserved address.
|
||||
- Startup or target validation reports `webhook endpoint is not allowed: ...`
|
||||
with a reason such as `private address` or `loopback host`.
|
||||
- An OIDC provider or login button is missing, and startup reports
|
||||
`OIDC provider discovery blocked by outbound policy` with the exact origin to
|
||||
allowlist.
|
||||
|
||||
## `RUSTFS_OUTBOUND_ALLOW_ORIGINS`
|
||||
|
||||
@@ -127,7 +138,7 @@ services:
|
||||
The endpoint keeps its full path (`/events`); the allowlist entry is the origin
|
||||
(`http://logstash:8080`) only.
|
||||
|
||||
## Upgrade checklist (beta.10 → beta.11+)
|
||||
## Upgrade checklist (beta.10 → beta.11+, or OIDC beta.11 → beta.12+)
|
||||
|
||||
1. List every outbound endpoint whose hostname resolves to a loopback, private,
|
||||
shared, or reserved address: notification webhooks, audit webhooks, OIDC
|
||||
|
||||
@@ -39,7 +39,7 @@ use rustfs_utils::{
|
||||
http::{AMZ_REQUEST_ID, REQUEST_ID_HEADER},
|
||||
};
|
||||
use s3s::{
|
||||
Body, S3Request, S3Response, S3Result,
|
||||
Body, S3Error, S3ErrorCode, S3Request, S3Response, S3Result,
|
||||
header::{CONTENT_LENGTH, CONTENT_TYPE},
|
||||
s3_error,
|
||||
};
|
||||
@@ -102,6 +102,10 @@ fn rebalance_start_rollback_error(start_err: &str, rollback_result: &Result<(),
|
||||
}
|
||||
}
|
||||
|
||||
fn rebalance_internal_error(message: impl Into<String>) -> S3Error {
|
||||
S3Error::with_message(S3ErrorCode::InternalError, message.into())
|
||||
}
|
||||
|
||||
fn rebalance_rollback_stop_failure_message(rebalance_id: &str, failures: &[String]) -> String {
|
||||
format!("cluster stop_rebalance rollback for {rebalance_id} partial: {}", failures.join("; "))
|
||||
}
|
||||
@@ -128,6 +132,28 @@ fn rebalance_rollback_failure_message(
|
||||
failures.join("; ")
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
enum RebalanceStartStep {
|
||||
PropagateFence,
|
||||
StartLocal,
|
||||
PropagateWorkers,
|
||||
}
|
||||
|
||||
const DISTRIBUTED_REBALANCE_START_STEPS: [RebalanceStartStep; 3] = [
|
||||
RebalanceStartStep::PropagateFence,
|
||||
RebalanceStartStep::StartLocal,
|
||||
RebalanceStartStep::PropagateWorkers,
|
||||
];
|
||||
const LOCAL_REBALANCE_START_STEPS: [RebalanceStartStep; 1] = [RebalanceStartStep::StartLocal];
|
||||
|
||||
fn rebalance_start_steps(has_notification_sys: bool) -> &'static [RebalanceStartStep] {
|
||||
if has_notification_sys {
|
||||
&DISTRIBUTED_REBALANCE_START_STEPS
|
||||
} else {
|
||||
&LOCAL_REBALANCE_START_STEPS
|
||||
}
|
||||
}
|
||||
|
||||
async fn rollback_cluster_rebalance_start(
|
||||
store: &Arc<ECStore>,
|
||||
notification_sys: Option<&NotificationSys>,
|
||||
@@ -177,6 +203,50 @@ async fn rollback_cluster_rebalance_start(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn rollback_rebalance_start_for_admin(
|
||||
store: &Arc<ECStore>,
|
||||
notification_sys: Option<&NotificationSys>,
|
||||
rebalance_id: &str,
|
||||
start_err: &str,
|
||||
request_id: &str,
|
||||
actor: &str,
|
||||
remote_addr: &str,
|
||||
) -> S3Result<()> {
|
||||
let rollback_result = rollback_cluster_rebalance_start(store, notification_sys, rebalance_id).await;
|
||||
let rollback_label = rollback_result_label(&rollback_result);
|
||||
match &rollback_result {
|
||||
Ok(_) => info!(
|
||||
event = EVENT_ADMIN_REBALANCE_STATE,
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
action = "start",
|
||||
result = rollback_label,
|
||||
request_id = %request_id,
|
||||
actor = %actor,
|
||||
remote_addr = %remote_addr,
|
||||
rebalance_id = %rebalance_id,
|
||||
propagation_error = %start_err,
|
||||
"admin rebalance state"
|
||||
),
|
||||
Err(rollback_err) => error!(
|
||||
event = EVENT_ADMIN_REBALANCE_STATE,
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
action = "start",
|
||||
result = rollback_label,
|
||||
request_id = %request_id,
|
||||
actor = %actor,
|
||||
remote_addr = %remote_addr,
|
||||
rebalance_id = %rebalance_id,
|
||||
propagation_error = %start_err,
|
||||
rollback_error = %rollback_err,
|
||||
"admin rebalance state"
|
||||
),
|
||||
}
|
||||
|
||||
Err(rebalance_internal_error(rebalance_start_rollback_error(start_err, &rollback_result)))
|
||||
}
|
||||
|
||||
pub fn register_rebalance_route(r: &mut S3Router<AdminOperation>) -> std::io::Result<()> {
|
||||
r.insert(
|
||||
Method::POST,
|
||||
@@ -473,7 +543,7 @@ impl Operation for RebalanceStart {
|
||||
|
||||
let buckets: Vec<String> = bucket_infos.into_iter().map(|bucket| bucket.name).collect();
|
||||
|
||||
let id = match store.init_and_start_rebalance(buckets).await {
|
||||
let id = match store.init_rebalance_start(buckets).await {
|
||||
Ok(id) => id,
|
||||
Err(StorageError::DecommissionAlreadyRunning) => {
|
||||
log_rebalance_request_rejected("start", "decommission_in_progress", &request_id, &actor, &remote_addr);
|
||||
@@ -484,7 +554,7 @@ impl Operation for RebalanceStart {
|
||||
return Err(s3_error!(OperationAborted, "rebalance is already in progress"));
|
||||
}
|
||||
Err(e) => {
|
||||
return Err(s3_error!(InternalError, "failed to start rebalance: {}", e));
|
||||
return Err(s3_error!(InternalError, "failed to initialize rebalance: {}", e));
|
||||
}
|
||||
};
|
||||
|
||||
@@ -493,80 +563,186 @@ impl Operation for RebalanceStart {
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
action = "start",
|
||||
state = "started",
|
||||
state = "metadata_initialized",
|
||||
request_id = %request_id,
|
||||
actor = %actor,
|
||||
remote_addr = %remote_addr,
|
||||
rebalance_id = %id,
|
||||
"admin rebalance state"
|
||||
);
|
||||
if let Some(notification_sys) = current_notification_system() {
|
||||
info!(
|
||||
event = EVENT_ADMIN_REBALANCE_STATE,
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
action = "start",
|
||||
state = "propagation_started",
|
||||
request_id = %request_id,
|
||||
actor = %actor,
|
||||
remote_addr = %remote_addr,
|
||||
rebalance_id = %id,
|
||||
"admin rebalance state"
|
||||
);
|
||||
if let Err(err) = notification_sys.load_rebalance_meta(true).await {
|
||||
error!(
|
||||
event = EVENT_ADMIN_REBALANCE_STATE,
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
action = "start",
|
||||
result = "propagation_failed",
|
||||
request_id = %request_id,
|
||||
actor = %actor,
|
||||
remote_addr = %remote_addr,
|
||||
rebalance_id = %id,
|
||||
error = %err,
|
||||
"admin rebalance state"
|
||||
);
|
||||
let notification_sys = current_notification_system();
|
||||
for step in rebalance_start_steps(notification_sys.is_some()) {
|
||||
match step {
|
||||
RebalanceStartStep::PropagateFence => {
|
||||
if let Some(notification_sys) = notification_sys.as_ref() {
|
||||
info!(
|
||||
event = EVENT_ADMIN_REBALANCE_STATE,
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
action = "start",
|
||||
state = "fence_propagation_started",
|
||||
request_id = %request_id,
|
||||
actor = %actor,
|
||||
remote_addr = %remote_addr,
|
||||
rebalance_id = %id,
|
||||
"admin rebalance state"
|
||||
);
|
||||
if let Err(err) = notification_sys.load_rebalance_meta(false).await {
|
||||
error!(
|
||||
event = EVENT_ADMIN_REBALANCE_STATE,
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
action = "start",
|
||||
result = "fence_propagation_failed",
|
||||
request_id = %request_id,
|
||||
actor = %actor,
|
||||
remote_addr = %remote_addr,
|
||||
rebalance_id = %id,
|
||||
error = %err,
|
||||
"admin rebalance state"
|
||||
);
|
||||
|
||||
let start_err = err.to_string();
|
||||
let rollback_result = rollback_cluster_rebalance_start(&store, Some(¬ification_sys), &id).await;
|
||||
let rollback_label = rollback_result_label(&rollback_result);
|
||||
match &rollback_result {
|
||||
Ok(_) => info!(
|
||||
event = EVENT_ADMIN_REBALANCE_STATE,
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
action = "start",
|
||||
result = rollback_label,
|
||||
request_id = %request_id,
|
||||
actor = %actor,
|
||||
remote_addr = %remote_addr,
|
||||
rebalance_id = %id,
|
||||
propagation_error = %start_err,
|
||||
"admin rebalance state"
|
||||
),
|
||||
Err(rollback_err) => error!(
|
||||
event = EVENT_ADMIN_REBALANCE_STATE,
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
action = "start",
|
||||
result = rollback_label,
|
||||
request_id = %request_id,
|
||||
actor = %actor,
|
||||
remote_addr = %remote_addr,
|
||||
rebalance_id = %id,
|
||||
propagation_error = %start_err,
|
||||
rollback_error = %rollback_err,
|
||||
"admin rebalance state"
|
||||
),
|
||||
let start_err = err.to_string();
|
||||
rollback_rebalance_start_for_admin(
|
||||
&store,
|
||||
Some(notification_sys),
|
||||
&id,
|
||||
&start_err,
|
||||
&request_id,
|
||||
&actor,
|
||||
&remote_addr,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
}
|
||||
}
|
||||
RebalanceStartStep::StartLocal => {
|
||||
if let Err(err) = store.start_rebalance_for_id(&id).await {
|
||||
error!(
|
||||
event = EVENT_ADMIN_REBALANCE_STATE,
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
action = "start",
|
||||
result = "local_start_failed",
|
||||
request_id = %request_id,
|
||||
actor = %actor,
|
||||
remote_addr = %remote_addr,
|
||||
rebalance_id = %id,
|
||||
error = %err,
|
||||
"admin rebalance state"
|
||||
);
|
||||
|
||||
return Err(s3_error!(
|
||||
InternalError,
|
||||
"{}",
|
||||
rebalance_start_rollback_error(&start_err, &rollback_result)
|
||||
));
|
||||
let start_err = err.to_string();
|
||||
if let Err(rollback_err) = store.rollback_rebalance_start_for_id(Some(&id), start_err.clone()).await {
|
||||
error!(
|
||||
event = EVENT_ADMIN_REBALANCE_STATE,
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
action = "start",
|
||||
result = "local_start_rollback_failed",
|
||||
request_id = %request_id,
|
||||
actor = %actor,
|
||||
remote_addr = %remote_addr,
|
||||
rebalance_id = %id,
|
||||
start_error = %start_err,
|
||||
rollback_error = %rollback_err,
|
||||
"admin rebalance state"
|
||||
);
|
||||
return Err(rebalance_internal_error(format!(
|
||||
"failed to start rebalance after metadata initialized for {id}; rollback failed: {rollback_err}"
|
||||
)));
|
||||
}
|
||||
|
||||
info!(
|
||||
event = EVENT_ADMIN_REBALANCE_STATE,
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
action = "start",
|
||||
result = "local_start_rollback_success",
|
||||
request_id = %request_id,
|
||||
actor = %actor,
|
||||
remote_addr = %remote_addr,
|
||||
rebalance_id = %id,
|
||||
start_error = %start_err,
|
||||
"admin rebalance state"
|
||||
);
|
||||
if let Some(notification_sys) = notification_sys.as_ref() {
|
||||
let terminal_reload_attempt_at = OffsetDateTime::now_utc();
|
||||
let terminal_reload_failures = match notification_sys.load_rebalance_meta_failures(false).await {
|
||||
Ok(failures) => failures,
|
||||
Err(err) => vec![format!("terminal rebalance reload rollback for {id} failed: {err}")],
|
||||
};
|
||||
if !terminal_reload_failures.is_empty() {
|
||||
let record = RebalanceStopPropagationRecord {
|
||||
stop_attempt_at: None,
|
||||
stop_failures: Vec::new(),
|
||||
terminal_reload_attempt_at: Some(terminal_reload_attempt_at),
|
||||
terminal_reload_failures: terminal_reload_failures.clone(),
|
||||
};
|
||||
store.record_rebalance_stop_propagation(record).await.map_err(|err| {
|
||||
rebalance_internal_error(format!(
|
||||
"failed to persist rebalance local-start rollback propagation metadata: {err}"
|
||||
))
|
||||
})?;
|
||||
return Err(rebalance_internal_error(format!(
|
||||
"failed to start rebalance after metadata initialized for {}; local metadata was finalized as failed, but terminal peer reload was incomplete: {}",
|
||||
id,
|
||||
rebalance_rollback_terminal_reload_failure_message(&id, &terminal_reload_failures)
|
||||
)));
|
||||
}
|
||||
}
|
||||
return Err(rebalance_internal_error(format!(
|
||||
"failed to start rebalance after metadata initialized for {id}; local metadata was finalized as failed: {start_err}"
|
||||
)));
|
||||
}
|
||||
}
|
||||
RebalanceStartStep::PropagateWorkers => {
|
||||
if let Some(notification_sys) = notification_sys.as_ref() {
|
||||
info!(
|
||||
event = EVENT_ADMIN_REBALANCE_STATE,
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
action = "start",
|
||||
state = "worker_propagation_started",
|
||||
request_id = %request_id,
|
||||
actor = %actor,
|
||||
remote_addr = %remote_addr,
|
||||
rebalance_id = %id,
|
||||
"admin rebalance state"
|
||||
);
|
||||
if let Err(err) = notification_sys.load_rebalance_meta(true).await {
|
||||
error!(
|
||||
event = EVENT_ADMIN_REBALANCE_STATE,
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
action = "start",
|
||||
result = "worker_propagation_failed",
|
||||
request_id = %request_id,
|
||||
actor = %actor,
|
||||
remote_addr = %remote_addr,
|
||||
rebalance_id = %id,
|
||||
error = %err,
|
||||
"admin rebalance state"
|
||||
);
|
||||
|
||||
let start_err = err.to_string();
|
||||
rollback_rebalance_start_for_admin(
|
||||
&store,
|
||||
Some(notification_sys),
|
||||
&id,
|
||||
&start_err,
|
||||
&request_id,
|
||||
&actor,
|
||||
&remote_addr,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if notification_sys.is_some() {
|
||||
info!(
|
||||
event = EVENT_ADMIN_REBALANCE_STATE,
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
@@ -902,10 +1078,11 @@ mod rebalance_handler_tests {
|
||||
use super::build_rebalance_pool_progress;
|
||||
use super::calculate_rebalance_progress;
|
||||
use super::{
|
||||
RebalPoolProgress, RebalanceAdminStatus, RebalancePoolStatus, RebalanceStopPropagationStatus,
|
||||
RebalPoolProgress, RebalanceAdminStatus, RebalancePoolStatus, RebalanceStartStep, RebalanceStopPropagationStatus,
|
||||
build_rebalance_admin_status, build_rebalance_pool_statuses, build_rebalance_stop_propagation_status,
|
||||
rebalance_pool_used, rebalance_query_present, rebalance_remaining_buckets, rebalance_rollback_failure_message,
|
||||
rebalance_rollback_stop_failure_message, rebalance_start_rollback_error, rebalance_used_pct, rollback_result_label,
|
||||
rebalance_rollback_stop_failure_message, rebalance_start_rollback_error, rebalance_start_steps, rebalance_used_pct,
|
||||
rollback_result_label,
|
||||
};
|
||||
use crate::admin::storage_api::rebalance::{
|
||||
DiskStat, RebalStatus, RebalanceCleanupWarningEntry, RebalanceCleanupWarnings, RebalanceInfo, RebalanceMeta,
|
||||
@@ -994,6 +1171,23 @@ mod rebalance_handler_tests {
|
||||
assert_eq!(rollback_result_label(&rollback_result), "rollback_failed");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_distributed_rebalance_start_fences_peers_before_workers() {
|
||||
assert_eq!(
|
||||
rebalance_start_steps(true),
|
||||
&[
|
||||
RebalanceStartStep::PropagateFence,
|
||||
RebalanceStartStep::StartLocal,
|
||||
RebalanceStartStep::PropagateWorkers
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_local_rebalance_start_has_no_peer_propagation_steps() {
|
||||
assert_eq!(rebalance_start_steps(false), &[RebalanceStartStep::StartLocal]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_calculate_rebalance_progress_stopped_by_end_time() {
|
||||
let start = OffsetDateTime::from_unix_timestamp(1_000).unwrap();
|
||||
|
||||
@@ -2687,6 +2687,12 @@ fn canonicalize_admin_path(path: &str) -> std::borrow::Cow<'_, str> {
|
||||
std::borrow::Cow::Borrowed(path)
|
||||
}
|
||||
|
||||
fn is_admin_v4_fallback_path(path: &str) -> bool {
|
||||
path.strip_prefix(ADMIN_PREFIX)
|
||||
.or_else(|| path.strip_prefix(MINIO_ADMIN_PREFIX))
|
||||
.is_some_and(|suffix| suffix == "/v4" || suffix.starts_with("/v4/"))
|
||||
}
|
||||
|
||||
impl<T: Operation> S3Router<T> {
|
||||
pub fn new(console_enabled: bool) -> Self {
|
||||
let router = Router::new();
|
||||
@@ -2886,6 +2892,12 @@ where
|
||||
return Ok(response);
|
||||
}
|
||||
|
||||
if is_admin_v4_fallback_path(req.uri.path()) {
|
||||
let mut resp = S3Response::new(Body::empty());
|
||||
resp.status = Some(StatusCode::UPGRADE_REQUIRED);
|
||||
return Ok(resp);
|
||||
}
|
||||
|
||||
Err(s3_error!(NotImplemented))
|
||||
}
|
||||
}
|
||||
@@ -2978,6 +2990,29 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
struct StatusOperation(StatusCode);
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl Operation for StatusOperation {
|
||||
async fn call(&self, _req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
|
||||
Ok(S3Response::new((self.0, Body::empty())))
|
||||
}
|
||||
}
|
||||
|
||||
fn router_request(method: Method, uri: &'static str) -> S3Request<Body> {
|
||||
S3Request {
|
||||
input: Body::empty(),
|
||||
method,
|
||||
uri: uri.parse().expect("uri should parse"),
|
||||
headers: HeaderMap::new(),
|
||||
extensions: http::Extensions::new(),
|
||||
credentials: None,
|
||||
region: None,
|
||||
service: None,
|
||||
trailing_headers: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn canonicalize_admin_path_maps_compat_prefix_to_rustfs_prefix() {
|
||||
assert_eq!(canonicalize_admin_path("/minio/admin/v3/info").as_ref(), "/rustfs/admin/v3/info");
|
||||
@@ -2990,6 +3025,65 @@ mod tests {
|
||||
assert_eq!(canonicalize_admin_path("/minio/adminx/object").as_ref(), "/minio/adminx/object");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn admin_v4_fallback_path_matches_only_admin_v4_prefixes() {
|
||||
assert!(is_admin_v4_fallback_path("/rustfs/admin/v4/info-canned-policy"));
|
||||
assert!(is_admin_v4_fallback_path("/minio/admin/v4/add-canned-policy"));
|
||||
assert!(is_admin_v4_fallback_path("/rustfs/admin/v4"));
|
||||
assert!(!is_admin_v4_fallback_path("/rustfs/admin/v3/info-canned-policy"));
|
||||
assert!(!is_admin_v4_fallback_path("/minio/admin/v3/info-canned-policy"));
|
||||
assert!(!is_admin_v4_fallback_path("/rustfs/admin/v40/info-canned-policy"));
|
||||
assert!(!is_admin_v4_fallback_path("/minio/adminx/v4/info-canned-policy"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn unmatched_admin_v4_request_returns_upgrade_required_for_sdk_downgrade() {
|
||||
let router: S3Router<StatusOperation> = S3Router::new(false);
|
||||
|
||||
for (method, uri) in [
|
||||
(Method::GET, "/minio/admin/v4/info-canned-policy?name=readwrite"),
|
||||
(Method::PUT, "/rustfs/admin/v4/add-canned-policy?name=repro"),
|
||||
] {
|
||||
let resp = router
|
||||
.call(router_request(method, uri))
|
||||
.await
|
||||
.expect("unmatched v4 admin request should return downgrade signal");
|
||||
|
||||
assert_eq!(resp.status, Some(StatusCode::UPGRADE_REQUIRED), "{uri}");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn unmatched_non_v4_admin_request_keeps_not_implemented_error() {
|
||||
let router: S3Router<StatusOperation> = S3Router::new(false);
|
||||
|
||||
let err = router
|
||||
.call(router_request(Method::GET, "/rustfs/admin/v3/missing-route"))
|
||||
.await
|
||||
.expect_err("unknown v3 admin route must keep the existing error");
|
||||
|
||||
assert_eq!(err.code(), &S3ErrorCode::NotImplemented);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn registered_admin_v4_route_is_not_shadowed_by_fallback() {
|
||||
let mut router: S3Router<StatusOperation> = S3Router::new(false);
|
||||
router
|
||||
.insert(
|
||||
Method::GET,
|
||||
"/rustfs/admin/v4/runtime/capabilities",
|
||||
StatusOperation(StatusCode::IM_A_TEAPOT),
|
||||
)
|
||||
.expect("route should insert");
|
||||
|
||||
let resp = router
|
||||
.call(router_request(Method::GET, "/rustfs/admin/v4/runtime/capabilities"))
|
||||
.await
|
||||
.expect("registered v4 route must dispatch normally");
|
||||
|
||||
assert_eq!(resp.status, Some(StatusCode::IM_A_TEAPOT));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn is_admin_path_accepts_rustfs_and_compat_prefixes() {
|
||||
assert!(is_admin_path("/rustfs/admin/v3/info"));
|
||||
|
||||
@@ -1968,12 +1968,17 @@ impl GetObjectResumeContext {
|
||||
store: Arc<ECStore>,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
opts: ObjectOptions,
|
||||
mut opts: ObjectOptions,
|
||||
request_headers: &HeaderMap,
|
||||
info: &ObjectInfo,
|
||||
range_start: i64,
|
||||
range_end: i64,
|
||||
) -> Self {
|
||||
if opts.version_id.is_none()
|
||||
&& let Some(version_id) = info.version_id
|
||||
{
|
||||
opts.version_id = Some(version_id.to_string());
|
||||
}
|
||||
let mut ssec_headers = HeaderMap::new();
|
||||
for name in [SSEC_ALGORITHM_HEADER, SSEC_KEY_HEADER, SSEC_KEY_MD5_HEADER] {
|
||||
if let Some(value) = request_headers.get(name) {
|
||||
@@ -5104,7 +5109,7 @@ impl DefaultObjectUsecase {
|
||||
part_number: Option<usize>,
|
||||
has_range: bool,
|
||||
encryption_applied: bool,
|
||||
buffered_body: Option<Bytes>,
|
||||
mut buffered_body: Option<Bytes>,
|
||||
cache_hook_served: bool,
|
||||
cache_hook_probed: bool,
|
||||
cache_fill_allowed: bool,
|
||||
@@ -5120,7 +5125,7 @@ impl DefaultObjectUsecase {
|
||||
// ODC-16 (backlog#1121): when the ecstore hook or shared cold fill
|
||||
// already supplied this body, the request-level plan was built before
|
||||
// the authoritative lookup. Serve it without planning a second time.
|
||||
if cache_hook_served && let Some(bytes) = buffered_body.clone() {
|
||||
if cache_hook_served && let Some(bytes) = buffered_body.take() {
|
||||
return Ok(Self::build_memory_bytes_blob(
|
||||
bytes,
|
||||
response_content_length,
|
||||
@@ -13183,6 +13188,67 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn get_object_resume_context_pins_latest_read_to_resolved_version() {
|
||||
let (_disk_paths, store, _context) = real_get_resume_test_context().await;
|
||||
let resolved_version = Uuid::new_v4();
|
||||
let info = ObjectInfo {
|
||||
version_id: Some(resolved_version),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let ctx = GetObjectResumeContext::new(
|
||||
Arc::clone(&store),
|
||||
"bucket",
|
||||
"object.bin",
|
||||
ObjectOptions::default(),
|
||||
&HeaderMap::new(),
|
||||
&info,
|
||||
0,
|
||||
-1,
|
||||
);
|
||||
assert_eq!(
|
||||
ctx.opts.version_id,
|
||||
Some(resolved_version.to_string()),
|
||||
"latest GET resume must reopen the initially resolved version, not the moving latest"
|
||||
);
|
||||
|
||||
let explicit_version = Uuid::new_v4().to_string();
|
||||
let explicit_opts = ObjectOptions {
|
||||
version_id: Some(explicit_version.clone()),
|
||||
..Default::default()
|
||||
};
|
||||
let ctx = GetObjectResumeContext::new(
|
||||
Arc::clone(&store),
|
||||
"bucket",
|
||||
"object.bin",
|
||||
explicit_opts,
|
||||
&HeaderMap::new(),
|
||||
&info,
|
||||
0,
|
||||
-1,
|
||||
);
|
||||
assert_eq!(
|
||||
ctx.opts.version_id.as_deref(),
|
||||
Some(explicit_version.as_str()),
|
||||
"an explicit request version must stay authoritative"
|
||||
);
|
||||
|
||||
let unversioned_info = ObjectInfo::default();
|
||||
let ctx = GetObjectResumeContext::new(
|
||||
Arc::clone(&store),
|
||||
"bucket",
|
||||
"object.bin",
|
||||
ObjectOptions::default(),
|
||||
&HeaderMap::new(),
|
||||
&unversioned_info,
|
||||
0,
|
||||
-1,
|
||||
);
|
||||
assert_eq!(ctx.opts.version_id, None, "unversioned reads have no version to pin");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn get_object_resume_context_redacts_ssec_headers_and_flags_range_dependent_size() {
|
||||
|
||||
+9
-1
@@ -288,7 +288,11 @@ impl From<StorageError> for ApiError {
|
||||
StorageError::ObjectNameInvalid(_, _) => S3ErrorCode::InvalidArgument,
|
||||
StorageError::BucketExists(_) => S3ErrorCode::BucketAlreadyOwnedByYou,
|
||||
StorageError::StorageFull => S3ErrorCode::ServiceUnavailable,
|
||||
StorageError::SlowDown => S3ErrorCode::SlowDown,
|
||||
StorageError::SlowDown
|
||||
| StorageError::FaultyDisk
|
||||
| StorageError::FaultyRemoteDisk
|
||||
| StorageError::DiskNotFound
|
||||
| StorageError::TooManyOpenFiles => S3ErrorCode::SlowDown,
|
||||
StorageError::ErasureReadQuorum
|
||||
| StorageError::InsufficientReadQuorum(_, _)
|
||||
| StorageError::ErasureWriteQuorum
|
||||
@@ -598,6 +602,10 @@ mod tests {
|
||||
(StorageError::BucketExists("test".into()), S3ErrorCode::BucketAlreadyOwnedByYou),
|
||||
(StorageError::StorageFull, S3ErrorCode::ServiceUnavailable),
|
||||
(StorageError::SlowDown, S3ErrorCode::SlowDown),
|
||||
(StorageError::FaultyDisk, S3ErrorCode::SlowDown),
|
||||
(StorageError::FaultyRemoteDisk, S3ErrorCode::SlowDown),
|
||||
(StorageError::DiskNotFound, S3ErrorCode::SlowDown),
|
||||
(StorageError::TooManyOpenFiles, S3ErrorCode::SlowDown),
|
||||
(StorageError::ErasureReadQuorum, S3ErrorCode::SlowDown),
|
||||
(StorageError::InsufficientReadQuorum("test".into(), "test".into()), S3ErrorCode::SlowDown),
|
||||
(StorageError::ErasureWriteQuorum, S3ErrorCode::SlowDown),
|
||||
|
||||
@@ -16,8 +16,9 @@ use crate::server::RPC_PREFIX;
|
||||
use crate::storage::request_context::spawn_traced;
|
||||
use crate::storage::storage_api::DiskError;
|
||||
use crate::storage::storage_api::rpc_consumer::http_service::{
|
||||
DEFAULT_READ_BUFFER_SIZE, NS_SCANNER_PROTOCOL_VERSION, NsScannerCapabilityResponse, StorageDiskRpcExt as _,
|
||||
WALK_DIR_STREAM_COMPLETION_V1, WalkDirOptions, find_local_disk_by_ref, sign_ns_scanner_capability, verify_rpc_signature,
|
||||
DEFAULT_READ_BUFFER_SIZE, NS_SCANNER_PROTOCOL_VERSION, NsScannerCapabilityResponse, PUT_FILE_AUTH_TRAILER_LEN,
|
||||
PUT_FILE_AUTH_V1, StorageDiskRpcExt as _, WALK_DIR_STREAM_COMPLETION_V1, WalkDirOptions, check_and_record_signed_rpc_nonce,
|
||||
find_local_disk_by_ref, sign_ns_scanner_capability, verify_put_file_auth_trailer, verify_rpc_signature,
|
||||
};
|
||||
#[cfg(test)]
|
||||
use crate::storage::storage_api::rpc_consumer::http_service::{
|
||||
@@ -72,6 +73,12 @@ const NS_SCANNER_PATH: &str = "/rustfs/rpc/ns_scanner";
|
||||
const NS_SCANNER_REQUEST_BODY_TIMEOUT: Duration = Duration::from_secs(15);
|
||||
const NS_SCANNER_STREAM_BUFFER_SIZE: usize = 64 * 1024;
|
||||
static NS_SCANNER_SERVER_EPOCH: LazyLock<uuid::Uuid> = LazyLock::new(uuid::Uuid::new_v4);
|
||||
static PUT_FILE_AUTH_STRICT: LazyLock<bool> = LazyLock::new(|| {
|
||||
rustfs_utils::get_env_bool(
|
||||
rustfs_config::ENV_INTERNODE_RPC_BODY_DIGEST_STRICT,
|
||||
rustfs_config::DEFAULT_INTERNODE_RPC_BODY_DIGEST_STRICT,
|
||||
)
|
||||
});
|
||||
|
||||
macro_rules! log_internode_rpc_response_failure {
|
||||
($status:expr, $rpc_path:expr, $method:expr, $operation:expr, $reason:expr, $result:expr, Some(($context_key:expr, $context_value:expr)), Some($error_text:expr)) => {{
|
||||
@@ -311,13 +318,34 @@ fn validate_walk_dir_completion_request(query: &WalkDirQuery, body: &[u8]) -> Op
|
||||
Some(propagate_completion_errors)
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, serde::Deserialize)]
|
||||
#[derive(Clone, Debug, Default, serde::Deserialize)]
|
||||
struct PutFileQuery {
|
||||
disk: String,
|
||||
volume: String,
|
||||
path: String,
|
||||
append: bool,
|
||||
size: i64,
|
||||
put_file_auth: Option<String>,
|
||||
put_file_nonce: Option<uuid::Uuid>,
|
||||
}
|
||||
|
||||
fn put_file_auth_nonce(query: &PutFileQuery) -> io::Result<Option<uuid::Uuid>> {
|
||||
match query.put_file_auth.as_deref() {
|
||||
None => {
|
||||
if *PUT_FILE_AUTH_STRICT {
|
||||
return Err(io::Error::other("put_file auth required"));
|
||||
}
|
||||
Ok(None)
|
||||
}
|
||||
Some(PUT_FILE_AUTH_V1) => {
|
||||
let nonce = query
|
||||
.put_file_nonce
|
||||
.filter(|nonce| !nonce.is_nil())
|
||||
.ok_or_else(|| io::Error::other("Invalid RPC nonce"))?;
|
||||
Ok(Some(nonce))
|
||||
}
|
||||
Some(_) => Err(io::Error::other("Unsupported put_file auth version")),
|
||||
}
|
||||
}
|
||||
|
||||
impl<S> Service<Request<Incoming>> for InternodeRpcService<S>
|
||||
@@ -1117,10 +1145,48 @@ where
|
||||
async fn handle_put_file(req: Request<Incoming>) -> Response<Body> {
|
||||
let method = req.method().clone();
|
||||
let path = req.uri().path().to_string();
|
||||
let url = req.uri().to_string();
|
||||
let query = match parse_query::<PutFileQuery>(&req) {
|
||||
Ok(query) => query,
|
||||
Err(response) => return *response,
|
||||
};
|
||||
let auth_nonce = match put_file_auth_nonce(&query) {
|
||||
Ok(nonce) => nonce,
|
||||
Err(e) => {
|
||||
log_internode_rpc_response_failure!(
|
||||
StatusCode::FORBIDDEN,
|
||||
&path,
|
||||
&method,
|
||||
Some(INTERNODE_OPERATION_PUT_FILE_STREAM),
|
||||
"put_file_auth_invalid",
|
||||
"rejected",
|
||||
Some(("disk", query.disk.as_str())),
|
||||
Some(&e)
|
||||
);
|
||||
return response_with_status(StatusCode::FORBIDDEN, format!("invalid put_file auth: {e}"));
|
||||
}
|
||||
};
|
||||
if let Some(nonce) = auth_nonce
|
||||
&& let Err(e) = check_and_record_signed_rpc_nonce(
|
||||
req.headers(),
|
||||
nonce,
|
||||
PUT_FILE_STREAM_PATH,
|
||||
INTERNODE_OPERATION_PUT_FILE_STREAM,
|
||||
INTERNODE_TRANSPORT_BACKEND_TCP_HTTP,
|
||||
)
|
||||
{
|
||||
log_internode_rpc_response_failure!(
|
||||
StatusCode::FORBIDDEN,
|
||||
&path,
|
||||
&method,
|
||||
Some(INTERNODE_OPERATION_PUT_FILE_STREAM),
|
||||
"put_file_replay_rejected",
|
||||
"rejected",
|
||||
Some(("disk", query.disk.as_str())),
|
||||
Some(&e)
|
||||
);
|
||||
return response_with_status(StatusCode::FORBIDDEN, format!("invalid put_file auth: {e}"));
|
||||
}
|
||||
|
||||
let Some(disk) = find_local_disk_by_ref(&query.disk).await else {
|
||||
log_internode_rpc_response_failure!(
|
||||
@@ -1156,14 +1222,16 @@ async fn handle_put_file(req: Request<Incoming>) -> Response<Body> {
|
||||
}
|
||||
};
|
||||
|
||||
let copied = match write_body_chunks_to_writer(req.into_body().into_data_stream(), &mut file).await {
|
||||
Ok(copied) => copied,
|
||||
Err(e) => {
|
||||
let message = put_file_stage_error_message("write_body", &query, &e);
|
||||
log_internode_put_file_stage_failure!("write_body", query, e);
|
||||
return response_with_status(StatusCode::INTERNAL_SERVER_ERROR, message);
|
||||
}
|
||||
};
|
||||
let copied =
|
||||
match write_put_file_body_chunks_to_writer(req.into_body().into_data_stream(), &mut file, &query, auth_nonce, &url).await
|
||||
{
|
||||
Ok(copied) => copied,
|
||||
Err(e) => {
|
||||
let message = put_file_stage_error_message("write_body", &query, &e);
|
||||
log_internode_put_file_stage_failure!("write_body", query, e);
|
||||
return response_with_status(StatusCode::INTERNAL_SERVER_ERROR, message);
|
||||
}
|
||||
};
|
||||
|
||||
let metrics = runtime_sources::current_internode_metrics();
|
||||
metrics.record_incoming_request_for_operation_and_backend(
|
||||
@@ -1222,6 +1290,112 @@ where
|
||||
Ok(copied)
|
||||
}
|
||||
|
||||
async fn write_put_file_body_chunks_to_writer<S, E, W>(
|
||||
body: S,
|
||||
writer: &mut W,
|
||||
query: &PutFileQuery,
|
||||
auth_nonce: Option<uuid::Uuid>,
|
||||
url: &str,
|
||||
) -> io::Result<u64>
|
||||
where
|
||||
S: futures::TryStream<Ok = Bytes, Error = E> + Unpin,
|
||||
E: Into<BoxError>,
|
||||
W: tokio::io::AsyncWrite + Unpin,
|
||||
{
|
||||
let Some(nonce) = auth_nonce else {
|
||||
return write_body_chunks_to_writer(body, writer).await;
|
||||
};
|
||||
|
||||
let expected_size = (!query.append && query.size >= 0)
|
||||
.then(|| {
|
||||
u64::try_from(query.size)
|
||||
.map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "put_file auth size cannot be represented"))
|
||||
})
|
||||
.transpose()?;
|
||||
let mut body = body;
|
||||
let mut remaining = expected_size;
|
||||
let mut copied = 0_u64;
|
||||
let mut trailer = Vec::with_capacity(PUT_FILE_AUTH_TRAILER_LEN);
|
||||
let mut hasher = Sha256::new();
|
||||
|
||||
while let Some(bytes) = body.try_next().await.map_err(io::Error::other)? {
|
||||
if let Some(remaining) = remaining.as_mut() {
|
||||
let chunk_len = u64::try_from(bytes.len()).unwrap_or(u64::MAX);
|
||||
let data_len = usize::try_from((*remaining).min(chunk_len))
|
||||
.map_err(|_| io::Error::other("put_file body length cannot be represented"))?;
|
||||
if data_len > 0 {
|
||||
hasher.update(&bytes[..data_len]);
|
||||
copied = copied
|
||||
.checked_add(
|
||||
u64::try_from(data_len).map_err(|_| io::Error::other("put_file body length cannot be represented"))?,
|
||||
)
|
||||
.ok_or_else(|| io::Error::other("put_file body length overflow"))?;
|
||||
*remaining -=
|
||||
u64::try_from(data_len).map_err(|_| io::Error::other("put_file body length cannot be represented"))?;
|
||||
writer.write_all(&bytes[..data_len]).await?;
|
||||
}
|
||||
|
||||
if data_len < bytes.len() {
|
||||
trailer.extend_from_slice(&bytes[data_len..]);
|
||||
if trailer.len() > PUT_FILE_AUTH_TRAILER_LEN {
|
||||
return Err(io::Error::new(io::ErrorKind::InvalidData, "put_file auth trailer has trailing data"));
|
||||
}
|
||||
}
|
||||
} else {
|
||||
let write_len = trailer
|
||||
.len()
|
||||
.saturating_add(bytes.len())
|
||||
.saturating_sub(PUT_FILE_AUTH_TRAILER_LEN);
|
||||
if write_len > 0 {
|
||||
let buffered_write_len = write_len.min(trailer.len());
|
||||
if buffered_write_len > 0 {
|
||||
writer.write_all(&trailer[..buffered_write_len]).await?;
|
||||
hasher.update(&trailer[..buffered_write_len]);
|
||||
copied = copied
|
||||
.checked_add(
|
||||
u64::try_from(buffered_write_len)
|
||||
.map_err(|_| io::Error::other("put_file body length cannot be represented"))?,
|
||||
)
|
||||
.ok_or_else(|| io::Error::other("put_file body length overflow"))?;
|
||||
trailer = trailer.split_off(buffered_write_len);
|
||||
}
|
||||
|
||||
let chunk_write_len = write_len - buffered_write_len;
|
||||
if chunk_write_len > 0 {
|
||||
writer.write_all(&bytes[..chunk_write_len]).await?;
|
||||
hasher.update(&bytes[..chunk_write_len]);
|
||||
}
|
||||
copied = copied
|
||||
.checked_add(
|
||||
u64::try_from(chunk_write_len)
|
||||
.map_err(|_| io::Error::other("put_file body length cannot be represented"))?,
|
||||
)
|
||||
.ok_or_else(|| io::Error::other("put_file body length overflow"))?;
|
||||
trailer.extend_from_slice(&bytes[chunk_write_len..]);
|
||||
} else {
|
||||
trailer.extend_from_slice(&bytes);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if remaining.is_some_and(|remaining| remaining != 0) {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::UnexpectedEof,
|
||||
format!("body size mismatch: expected {} bytes, received {copied}", query.size),
|
||||
));
|
||||
}
|
||||
if trailer.len() != PUT_FILE_AUTH_TRAILER_LEN {
|
||||
return Err(io::Error::new(io::ErrorKind::UnexpectedEof, "put_file auth trailer is incomplete"));
|
||||
}
|
||||
|
||||
let expected = verify_put_file_auth_trailer(url, &Method::PUT, nonce, &trailer)?;
|
||||
let actual = hex_simd::encode_to_string(hasher.finalize(), hex_simd::AsciiCase::Lower);
|
||||
if actual != expected {
|
||||
return Err(io::Error::new(io::ErrorKind::InvalidData, "put_file body digest mismatch"));
|
||||
}
|
||||
Ok(copied)
|
||||
}
|
||||
|
||||
fn parse_query<T>(req: &Request<Incoming>) -> Result<T, RpcErrorResponse>
|
||||
where
|
||||
T: DeserializeOwned + Default,
|
||||
@@ -1308,11 +1482,12 @@ mod tests {
|
||||
NS_SCANNER_SESSION_ID_QUERY, NS_SCANNER_SESSION_SEQUENCE_QUERY, NsScannerQuery, PUT_FILE_STREAM_PATH, PutFileQuery,
|
||||
READ_FILE_STREAM_PATH, WALK_DIR_BODY_SHA256_QUERY, WALK_DIR_PATH, WalkDirQuery, append_walk_dir_completion,
|
||||
internode_http_operation, internode_rpc_subsystem, is_internode_rpc_path, ns_scanner_response_body,
|
||||
ns_scanner_server_epoch_matches, put_body_size_mismatch, put_file_stage_error_message, read_file_body_stream,
|
||||
remote_scanner_claim_rejection, response_with_disk_error, supports_walk_dir_stream_completion,
|
||||
ns_scanner_server_epoch_matches, put_body_size_mismatch, put_file_auth_nonce, put_file_stage_error_message,
|
||||
read_file_body_stream, remote_scanner_claim_rejection, response_with_disk_error, supports_walk_dir_stream_completion,
|
||||
validate_walk_dir_completion_request, verify_internode_rpc_signature, verify_ns_scanner_body_digest,
|
||||
verify_walk_dir_body_digest, walk_dir_response_body, write_body_chunks_to_writer,
|
||||
verify_walk_dir_body_digest, walk_dir_response_body, write_body_chunks_to_writer, write_put_file_body_chunks_to_writer,
|
||||
};
|
||||
use crate::storage::storage_api::ecstore_rpc::build_put_file_auth_trailer;
|
||||
use bytes::Bytes;
|
||||
use http::{HeaderMap, HeaderValue, Method, StatusCode, Uri};
|
||||
use http_body_util::BodyExt;
|
||||
@@ -1433,6 +1608,8 @@ mod tests {
|
||||
path: "tmp/object/part.1".to_string(),
|
||||
append: false,
|
||||
size: 1024,
|
||||
put_file_auth: None,
|
||||
put_file_nonce: None,
|
||||
};
|
||||
|
||||
let msg = put_file_stage_error_message("write_body", &query, &"connection reset");
|
||||
@@ -1452,6 +1629,8 @@ mod tests {
|
||||
path: "object/part.1".to_string(),
|
||||
append,
|
||||
size,
|
||||
put_file_auth: None,
|
||||
put_file_nonce: None,
|
||||
};
|
||||
|
||||
// Truncated (or over-long) body on the create path is rejected.
|
||||
@@ -1579,6 +1758,165 @@ mod tests {
|
||||
assert_eq!(out, b"hello world");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn put_file_auth_nonce_accepts_v1_requests_with_non_nil_nonce() {
|
||||
let nonce = uuid::Uuid::new_v4();
|
||||
let query = PutFileQuery {
|
||||
disk: "disk-a".to_string(),
|
||||
volume: "bucket".to_string(),
|
||||
path: "object/part.1".to_string(),
|
||||
append: false,
|
||||
size: 11,
|
||||
put_file_auth: Some("digest-trailer-v1".to_string()),
|
||||
put_file_nonce: Some(nonce),
|
||||
};
|
||||
|
||||
assert_eq!(put_file_auth_nonce(&query).expect("v1 auth should parse"), Some(nonce));
|
||||
|
||||
let mut append = query.clone();
|
||||
append.append = true;
|
||||
assert_eq!(put_file_auth_nonce(&append).expect("append auth should parse"), Some(nonce));
|
||||
|
||||
let mut unknown_size = query.clone();
|
||||
unknown_size.size = -1;
|
||||
assert_eq!(put_file_auth_nonce(&unknown_size).expect("unknown-size auth should parse"), Some(nonce));
|
||||
|
||||
let mut nil = query.clone();
|
||||
nil.put_file_nonce = Some(uuid::Uuid::nil());
|
||||
assert!(put_file_auth_nonce(&nil).is_err());
|
||||
|
||||
let mut unknown = query;
|
||||
unknown.put_file_auth = Some("digest-trailer-v2".to_string());
|
||||
assert!(put_file_auth_nonce(&unknown).is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn put_file_auth_body_writes_only_data_and_verifies_trailer() {
|
||||
let _ = rustfs_credentials::set_global_rpc_secret("put-file-auth-body-test-secret".to_string());
|
||||
let nonce = uuid::Uuid::parse_str("11111111-2222-4333-8444-555555555555").expect("nonce");
|
||||
let url = concat!(
|
||||
"/rustfs/rpc/put_file_stream?disk=disk-a&volume=bucket&path=object%2Fpart.1",
|
||||
"&append=false&size=11&put_file_auth=digest-trailer-v1&put_file_nonce=11111111-2222-4333-8444-555555555555"
|
||||
);
|
||||
let digest = hex_simd::encode_to_string(sha2::Sha256::digest(b"hello world"), hex_simd::AsciiCase::Lower);
|
||||
let trailer = build_put_file_auth_trailer(url, &Method::PUT, nonce, &digest).expect("trailer should build");
|
||||
let query = PutFileQuery {
|
||||
disk: "disk-a".to_string(),
|
||||
volume: "bucket".to_string(),
|
||||
path: "object/part.1".to_string(),
|
||||
append: false,
|
||||
size: 11,
|
||||
put_file_auth: Some("digest-trailer-v1".to_string()),
|
||||
put_file_nonce: Some(nonce),
|
||||
};
|
||||
let mut second = b"world".to_vec();
|
||||
second.extend_from_slice(&trailer[..7]);
|
||||
let body = iter(vec![
|
||||
Ok::<Bytes, io::Error>(Bytes::from_static(b"hello ")),
|
||||
Ok(Bytes::from(second)),
|
||||
Ok(Bytes::copy_from_slice(&trailer[7..])),
|
||||
]);
|
||||
let mut writer = Vec::new();
|
||||
|
||||
let copied = write_put_file_body_chunks_to_writer(body, &mut writer, &query, Some(nonce), url)
|
||||
.await
|
||||
.expect("authenticated body should verify");
|
||||
|
||||
assert_eq!(copied, 11);
|
||||
assert_eq!(writer, b"hello world");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn put_file_auth_body_rejects_tampered_data() {
|
||||
let _ = rustfs_credentials::set_global_rpc_secret("put-file-auth-body-test-secret".to_string());
|
||||
let nonce = uuid::Uuid::parse_str("22222222-3333-4444-8555-666666666666").expect("nonce");
|
||||
let url = concat!(
|
||||
"/rustfs/rpc/put_file_stream?disk=disk-a&volume=bucket&path=object%2Fpart.1",
|
||||
"&append=false&size=11&put_file_auth=digest-trailer-v1&put_file_nonce=22222222-3333-4444-8555-666666666666"
|
||||
);
|
||||
let signed_digest = hex_simd::encode_to_string(sha2::Sha256::digest(b"hello world"), hex_simd::AsciiCase::Lower);
|
||||
let trailer = build_put_file_auth_trailer(url, &Method::PUT, nonce, &signed_digest).expect("trailer should build");
|
||||
let query = PutFileQuery {
|
||||
disk: "disk-a".to_string(),
|
||||
volume: "bucket".to_string(),
|
||||
path: "object/part.1".to_string(),
|
||||
append: false,
|
||||
size: 11,
|
||||
put_file_auth: Some("digest-trailer-v1".to_string()),
|
||||
put_file_nonce: Some(nonce),
|
||||
};
|
||||
let mut payload = b"hello worle".to_vec();
|
||||
payload.extend_from_slice(&trailer);
|
||||
let body = iter(vec![Ok::<Bytes, io::Error>(Bytes::from(payload))]);
|
||||
let mut writer = Vec::new();
|
||||
|
||||
let err = write_put_file_body_chunks_to_writer(body, &mut writer, &query, Some(nonce), url)
|
||||
.await
|
||||
.expect_err("tampered body must fail digest verification");
|
||||
|
||||
assert_eq!(err.kind(), io::ErrorKind::InvalidData);
|
||||
assert_eq!(err.to_string(), "put_file body digest mismatch");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn put_file_auth_append_body_uses_trailing_auth_record() {
|
||||
let _ = rustfs_credentials::set_global_rpc_secret("put-file-auth-body-test-secret".to_string());
|
||||
let nonce = uuid::Uuid::parse_str("33333333-4444-4555-8666-777777777777").expect("nonce");
|
||||
let url = concat!(
|
||||
"/rustfs/rpc/put_file_stream?disk=disk-a&volume=bucket&path=object%2Fpart.1",
|
||||
"&append=true&size=0&put_file_auth=digest-trailer-v1&put_file_nonce=33333333-4444-4555-8666-777777777777"
|
||||
);
|
||||
let digest = hex_simd::encode_to_string(sha2::Sha256::digest(b"append-data"), hex_simd::AsciiCase::Lower);
|
||||
let trailer = build_put_file_auth_trailer(url, &Method::PUT, nonce, &digest).expect("trailer should build");
|
||||
let query = PutFileQuery {
|
||||
disk: "disk-a".to_string(),
|
||||
volume: "bucket".to_string(),
|
||||
path: "object/part.1".to_string(),
|
||||
append: true,
|
||||
size: 0,
|
||||
put_file_auth: Some("digest-trailer-v1".to_string()),
|
||||
put_file_nonce: Some(nonce),
|
||||
};
|
||||
let mut payload = b"append-data".to_vec();
|
||||
payload.extend_from_slice(&trailer);
|
||||
let body = iter(vec![Ok::<Bytes, io::Error>(Bytes::from(payload))]);
|
||||
let mut writer = Vec::new();
|
||||
|
||||
let copied = write_put_file_body_chunks_to_writer(body, &mut writer, &query, Some(nonce), url)
|
||||
.await
|
||||
.expect("append body should verify");
|
||||
|
||||
assert_eq!(copied, 11);
|
||||
assert_eq!(writer, b"append-data");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn put_file_auth_append_body_rejects_missing_trailer() {
|
||||
let nonce = uuid::Uuid::parse_str("44444444-5555-4666-8777-888888888888").expect("nonce");
|
||||
let url = concat!(
|
||||
"/rustfs/rpc/put_file_stream?disk=disk-a&volume=bucket&path=object%2Fpart.1",
|
||||
"&append=true&size=0&put_file_auth=digest-trailer-v1&put_file_nonce=44444444-5555-4666-8777-888888888888"
|
||||
);
|
||||
let query = PutFileQuery {
|
||||
disk: "disk-a".to_string(),
|
||||
volume: "bucket".to_string(),
|
||||
path: "object/part.1".to_string(),
|
||||
append: true,
|
||||
size: 0,
|
||||
put_file_auth: Some("digest-trailer-v1".to_string()),
|
||||
put_file_nonce: Some(nonce),
|
||||
};
|
||||
let body = iter(vec![Ok::<Bytes, io::Error>(Bytes::from_static(b"append-data"))]);
|
||||
let mut writer = Vec::new();
|
||||
|
||||
let err = write_put_file_body_chunks_to_writer(body, &mut writer, &query, Some(nonce), url)
|
||||
.await
|
||||
.expect_err("missing trailer must fail");
|
||||
|
||||
assert_eq!(err.kind(), io::ErrorKind::UnexpectedEof);
|
||||
assert_eq!(err.to_string(), "put_file auth trailer is incomplete");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn walk_dir_body_surfaces_background_failure_after_data() {
|
||||
let body = walk_dir_response_body(true, |mut writer| async move {
|
||||
|
||||
@@ -216,10 +216,12 @@ pub(crate) mod rpc_consumer {
|
||||
NS_SCANNER_SESSION_ID_QUERY, NS_SCANNER_SESSION_SEQUENCE_QUERY, WALK_DIR_BODY_SHA256_QUERY,
|
||||
};
|
||||
pub(crate) use super::super::storage_contracts::{
|
||||
NS_SCANNER_PROTOCOL_VERSION, NsScannerCapabilityResponse, WALK_DIR_STREAM_COMPLETION_V1,
|
||||
NS_SCANNER_PROTOCOL_VERSION, NsScannerCapabilityResponse, PUT_FILE_AUTH_TRAILER_LEN, PUT_FILE_AUTH_V1,
|
||||
WALK_DIR_STREAM_COMPLETION_V1,
|
||||
};
|
||||
pub(crate) use super::super::{
|
||||
StorageDiskRpcExt, WalkDirOptions, find_local_disk_by_ref, sign_ns_scanner_capability, verify_rpc_signature,
|
||||
StorageDiskRpcExt, WalkDirOptions, check_and_record_signed_rpc_nonce, find_local_disk_by_ref,
|
||||
sign_ns_scanner_capability, verify_put_file_auth_trailer, verify_rpc_signature,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -498,13 +500,15 @@ pub(crate) mod ecstore_rpc {
|
||||
pub(crate) use rustfs_ecstore::api::rpc::{
|
||||
KMS_SIGNAL_SUBSYSTEM, LocalPeerS3Client, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient,
|
||||
PeerS3Client, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, TONIC_RPC_PREFIX,
|
||||
normalize_tonic_rpc_audience, sign_ns_scanner_capability, sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge,
|
||||
tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason, verify_rpc_signature,
|
||||
verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest, verify_tonic_rpc_signature_with_bootstrap,
|
||||
check_and_record_signed_rpc_nonce, normalize_tonic_rpc_audience, sign_ns_scanner_capability,
|
||||
sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers,
|
||||
tonic_rpc_auth_failure_reason, verify_put_file_auth_trailer, verify_rpc_signature, verify_tonic_canonical_body_digest,
|
||||
verify_tonic_mutation_body_digest, verify_tonic_rpc_signature_with_bootstrap,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::api::rpc::{
|
||||
gen_signature_headers, gen_tonic_signature_headers, set_tonic_canonical_body_digest, verify_tonic_rpc_response_proof,
|
||||
build_put_file_auth_trailer, gen_signature_headers, gen_tonic_signature_headers, set_tonic_canonical_body_digest,
|
||||
verify_tonic_rpc_response_proof,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1655,6 +1659,25 @@ pub(crate) fn verify_rpc_signature(url: &str, method: &http::Method, headers: &h
|
||||
ecstore_rpc::verify_rpc_signature(url, method, headers)
|
||||
}
|
||||
|
||||
pub(crate) fn check_and_record_signed_rpc_nonce(
|
||||
headers: &http::HeaderMap,
|
||||
nonce: uuid::Uuid,
|
||||
rpc_path: &str,
|
||||
operation: &'static str,
|
||||
backend: &'static str,
|
||||
) -> std::io::Result<()> {
|
||||
ecstore_rpc::check_and_record_signed_rpc_nonce(headers, nonce, rpc_path, operation, backend)
|
||||
}
|
||||
|
||||
pub(crate) fn verify_put_file_auth_trailer(
|
||||
url: &str,
|
||||
method: &http::Method,
|
||||
nonce: uuid::Uuid,
|
||||
trailer: &[u8],
|
||||
) -> std::io::Result<String> {
|
||||
ecstore_rpc::verify_put_file_auth_trailer(url, method, nonce, trailer)
|
||||
}
|
||||
|
||||
pub(crate) fn sign_ns_scanner_capability(challenge: uuid::Uuid, server_epoch: uuid::Uuid) -> std::io::Result<Vec<u8>> {
|
||||
ecstore_rpc::sign_ns_scanner_capability(challenge, server_epoch)
|
||||
}
|
||||
|
||||
@@ -74,6 +74,8 @@ their issue closes.
|
||||
| `run_get_1mib_abba_stage_metrics.sh` | dev-tool | Exact-1MiB isolated-host GET ABBA/stage-metrics harness for backlog#1434 | `test_get_1mib_abba_stage_metrics.sh` |
|
||||
| `run_gt1g_get_http_matrix.sh` | dev-tool | >1 GiB GET HTTP matrix | `docs/testing/ecstore-validation-suite-design.md` |
|
||||
| `run_gt1g_multipart_put_matrix.sh` | dev-tool | >1 GiB multipart PUT matrix | `docs/testing/ecstore-validation-suite-design.md` |
|
||||
| `sample_remote_rustfs_rss.sh` | dev-tool | Remote RustFS PID CPU/RSS TSV sampler for hotpath profiling runs | `test_sample_remote_rustfs_rss.sh`; backlog#1647 |
|
||||
| `summarize_samply_profile_symbols.py` | dev-tool | Offline samply `profile.json.gz` + `.syms.json` function-level hotpath summarizer | `test_summarize_samply_profile_symbols.py`; backlog#1647 |
|
||||
| `run_scanner_benchmarks.sh` | dev-tool (disposition pending) | Scanner performance benchmark runner. Contains a hardcoded stale path; **disposition owned by backlog perf-10 — do not fix, move, or delete it here** | — |
|
||||
|
||||
## Local development & operations
|
||||
|
||||
Executable
+107
@@ -0,0 +1,107 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
usage() {
|
||||
cat <<'USAGE'
|
||||
Usage: scripts/capture_remote_journal_errors.sh --nodes <csv> --since <iso-time> --label <label> --out-dir <dir> [options]
|
||||
|
||||
Capture RustFS journal lines matching auth/error/failure patterns for a UTC
|
||||
validation window. The --since value is normalized to the journalctl-friendly
|
||||
"YYYY-MM-DD HH:MM:SS UTC" form before it is sent to remote nodes.
|
||||
|
||||
Options:
|
||||
--nodes <csv> Comma-separated node names, for example vm004,vm005.
|
||||
--since <iso-time> UTC ISO timestamp, for example 2026-08-08T09:05:45Z.
|
||||
--label <label> Prefix used for output files.
|
||||
--out-dir <dir> Local output directory.
|
||||
--unit <name> systemd unit name. Default: rustfs.
|
||||
--filter-regex <expr> grep -Ei pattern. Default captures auth/signature/error/warn/fail/panic.
|
||||
--ssh-bin <path> SSH binary or test double. Default: ssh.
|
||||
-h, --help Show this help.
|
||||
USAGE
|
||||
}
|
||||
|
||||
die() {
|
||||
echo "error: $*" >&2
|
||||
exit 2
|
||||
}
|
||||
|
||||
shell_quote() {
|
||||
local value=${1//\'/\'\\\'\'}
|
||||
printf "'%s'" "$value"
|
||||
}
|
||||
|
||||
validate_name() {
|
||||
local field="$1"
|
||||
local value="$2"
|
||||
[[ "$value" =~ ^[A-Za-z0-9._@-]+$ ]] || die "$field contains unsafe characters: $value"
|
||||
}
|
||||
|
||||
format_since_utc() {
|
||||
python3 - "$1" <<'PY'
|
||||
from datetime import datetime, timezone
|
||||
import sys
|
||||
|
||||
value = sys.argv[1].strip()
|
||||
if value.endswith("Z"):
|
||||
value = value[:-1] + "+00:00"
|
||||
try:
|
||||
parsed = datetime.fromisoformat(value)
|
||||
except ValueError as err:
|
||||
raise SystemExit(f"invalid ISO timestamp: {err}")
|
||||
if parsed.tzinfo is None:
|
||||
parsed = parsed.replace(tzinfo=timezone.utc)
|
||||
print(parsed.astimezone(timezone.utc).strftime("%Y-%m-%d %H:%M:%S UTC"))
|
||||
PY
|
||||
}
|
||||
|
||||
NODES_CSV=""
|
||||
SINCE_ISO=""
|
||||
LABEL=""
|
||||
OUT_DIR=""
|
||||
UNIT="rustfs"
|
||||
FILTER_REGEX="No valid auth token|auth|signature|error|panic|fail|warn"
|
||||
SSH_BIN="${SSH_BIN:-ssh}"
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--nodes) NODES_CSV="${2:-}"; shift 2 ;;
|
||||
--since) SINCE_ISO="${2:-}"; shift 2 ;;
|
||||
--label) LABEL="${2:-}"; shift 2 ;;
|
||||
--out-dir) OUT_DIR="${2:-}"; shift 2 ;;
|
||||
--unit) UNIT="${2:-}"; shift 2 ;;
|
||||
--filter-regex) FILTER_REGEX="${2:-}"; shift 2 ;;
|
||||
--ssh-bin) SSH_BIN="${2:-}"; shift 2 ;;
|
||||
-h|--help) usage; exit 0 ;;
|
||||
*) die "unknown argument: $1" ;;
|
||||
esac
|
||||
done
|
||||
|
||||
[[ -n "$NODES_CSV" ]] || die "--nodes is required"
|
||||
[[ -n "$SINCE_ISO" ]] || die "--since is required"
|
||||
[[ -n "$LABEL" ]] || die "--label is required"
|
||||
[[ -n "$OUT_DIR" ]] || die "--out-dir is required"
|
||||
[[ -n "$UNIT" ]] || die "--unit must not be empty"
|
||||
validate_name "--label" "$LABEL"
|
||||
validate_name "--unit" "$UNIT"
|
||||
|
||||
since_journal=$(format_since_utc "$SINCE_ISO")
|
||||
mkdir -p "$OUT_DIR"
|
||||
|
||||
IFS=',' read -r -a nodes <<<"$NODES_CSV"
|
||||
captured=0
|
||||
for node in "${nodes[@]}"; do
|
||||
node="${node//[[:space:]]/}"
|
||||
[[ -n "$node" ]] || continue
|
||||
validate_name "node" "$node"
|
||||
|
||||
output_file="$OUT_DIR/${LABEL}-${node}-journal-errors.txt"
|
||||
journal_cmd="journalctl -u $(shell_quote "$UNIT") --since $(shell_quote "$since_journal") --no-pager"
|
||||
remote_cmd="sudo su - root -c $(shell_quote "$journal_cmd")"
|
||||
"$SSH_BIN" "$node" "$remote_cmd" 2>&1 | grep -Ei "$FILTER_REGEX" >"$output_file" || true
|
||||
captured=$((captured + 1))
|
||||
done
|
||||
|
||||
[[ "$captured" -gt 0 ]] || die "--nodes did not contain any usable node names"
|
||||
echo "journal_since=$since_journal"
|
||||
echo "captured_nodes=$captured"
|
||||
@@ -24,7 +24,7 @@ cd "$(dirname "$0")/.."
|
||||
|
||||
# Baselines verified on 2026-08-06. Lower-only; see header.
|
||||
S3S_IMPORT_FILES_BASELINE=236
|
||||
S3_ERROR_LINES_BASELINE=1679
|
||||
S3_ERROR_LINES_BASELINE=1678
|
||||
S3S_PATH_PATTERN='(^|[^"[:alnum:]_])s3s::'
|
||||
|
||||
TMP_DIR="$(mktemp -d)"
|
||||
|
||||
Executable
+98
@@ -0,0 +1,98 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
usage() {
|
||||
cat <<'USAGE'
|
||||
Usage: scripts/collect_remote_samply_artifacts.sh --mapping <file> --out-dir <dir> [options]
|
||||
|
||||
Copy samply profiles and symbol artifacts from RustFS nodes without allowing
|
||||
scp to consume the mapping loop's stdin.
|
||||
|
||||
Mapping file format:
|
||||
<node> <remote_artifact_dir>
|
||||
|
||||
Example:
|
||||
vm004 /data/rustfs/hotpath/20260808-put-1m
|
||||
vm005 /data/rustfs/hotpath/20260808-put-1m
|
||||
|
||||
Options:
|
||||
--mapping <file> Node and remote artifact directory pairs.
|
||||
--out-dir <dir> Local directory where node subdirectories are created.
|
||||
--remote-root <dir> Required parent path for remote artifact directories.
|
||||
Default: /data/rustfs.
|
||||
--ssh-bin <path> SSH binary or test double. Default: ssh.
|
||||
--scp-bin <path> SCP binary or test double. Default: scp.
|
||||
-h, --help Show this help.
|
||||
USAGE
|
||||
}
|
||||
|
||||
die() {
|
||||
echo "error: $*" >&2
|
||||
exit 2
|
||||
}
|
||||
|
||||
shell_quote() {
|
||||
local value=${1//\'/\'\\\'\'}
|
||||
printf "'%s'" "$value"
|
||||
}
|
||||
|
||||
validate_node() {
|
||||
[[ "$1" =~ ^[A-Za-z0-9._-]+$ ]] || die "node contains unsafe characters: $1"
|
||||
}
|
||||
|
||||
validate_path() {
|
||||
[[ "$1" =~ ^/[A-Za-z0-9._/@+=-]+$ ]] || die "path contains unsafe characters: $1"
|
||||
}
|
||||
|
||||
MAPPING=""
|
||||
OUT_DIR=""
|
||||
REMOTE_ROOT="/data/rustfs"
|
||||
SSH_BIN="${SSH_BIN:-ssh}"
|
||||
SCP_BIN="${SCP_BIN:-scp}"
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--mapping) MAPPING="${2:-}"; shift 2 ;;
|
||||
--out-dir) OUT_DIR="${2:-}"; shift 2 ;;
|
||||
--remote-root) REMOTE_ROOT="${2:-}"; shift 2 ;;
|
||||
--ssh-bin) SSH_BIN="${2:-}"; shift 2 ;;
|
||||
--scp-bin) SCP_BIN="${2:-}"; shift 2 ;;
|
||||
-h|--help) usage; exit 0 ;;
|
||||
*) die "unknown argument: $1" ;;
|
||||
esac
|
||||
done
|
||||
|
||||
[[ -n "$MAPPING" ]] || die "--mapping is required"
|
||||
[[ -f "$MAPPING" ]] || die "mapping file not found: $MAPPING"
|
||||
[[ -n "$OUT_DIR" ]] || die "--out-dir is required"
|
||||
[[ -n "$REMOTE_ROOT" ]] || die "--remote-root must not be empty"
|
||||
[[ "$REMOTE_ROOT" == /* ]] || die "--remote-root must be an absolute path"
|
||||
validate_path "$REMOTE_ROOT"
|
||||
|
||||
REMOTE_ROOT="${REMOTE_ROOT%/}"
|
||||
mkdir -p "$OUT_DIR"
|
||||
|
||||
processed=0
|
||||
while IFS= read -r line || [[ -n "$line" ]]; do
|
||||
[[ -z "${line//[[:space:]]/}" ]] && continue
|
||||
[[ "$line" =~ ^[[:space:]]*# ]] && continue
|
||||
|
||||
read -r node remote_dir extra <<<"$line"
|
||||
[[ -n "${node:-}" && -n "${remote_dir:-}" && -z "${extra:-}" ]] || die "mapping lines must contain exactly two fields: $line"
|
||||
validate_node "$node"
|
||||
[[ "$remote_dir" == /* ]] || die "remote artifact dir must be absolute for $node: $remote_dir"
|
||||
validate_path "$remote_dir"
|
||||
[[ "$remote_dir" == "$REMOTE_ROOT"/* ]] || die "remote artifact dir must be under $REMOTE_ROOT for $node: $remote_dir"
|
||||
|
||||
node_out_dir="$OUT_DIR/$node"
|
||||
mkdir -p "$node_out_dir"
|
||||
|
||||
quoted_remote_dir=$(shell_quote "$remote_dir")
|
||||
remote_cmd="chmod -R a+rX $quoted_remote_dir; find $quoted_remote_dir -maxdepth 1 -type f -printf '%f %s bytes\n'"
|
||||
"$SSH_BIN" "$node" "sudo su - root -c $(shell_quote "$remote_cmd")" >"$OUT_DIR/${node}-files.txt" 2>&1 </dev/null
|
||||
"$SCP_BIN" -q -r "$node:${remote_dir%/}/"* "$node_out_dir/" </dev/null 2>"$OUT_DIR/${node}-scp.err"
|
||||
processed=$((processed + 1))
|
||||
done <"$MAPPING"
|
||||
|
||||
[[ "$processed" -gt 0 ]] || die "mapping file did not contain any nodes"
|
||||
echo "collected_nodes=$processed"
|
||||
@@ -12,7 +12,9 @@ usage() {
|
||||
Usage: scripts/run_samply_attach_window.sh --pid <pid> --duration-secs <n> --output <profile.json.gz> [options]
|
||||
|
||||
Attach samply to an already-running process for a bounded window and force a
|
||||
Ctrl+C-style shutdown so samply writes the profile artifact.
|
||||
Ctrl+C-style shutdown so samply writes the profile artifact. This script uses
|
||||
direct `samply record -p`; `cargo samply` launches a cargo target and is not
|
||||
suitable for attaching to an existing RustFS service PID.
|
||||
|
||||
Options:
|
||||
--pid <pid> Existing process id to profile.
|
||||
|
||||
Executable
+88
@@ -0,0 +1,88 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
usage() {
|
||||
cat <<'USAGE'
|
||||
Usage: scripts/sample_remote_rustfs_rss.sh --nodes <csv> --duration-secs <n> --out <file> [options]
|
||||
|
||||
Sample RustFS process CPU and RSS from remote nodes for a bounded window.
|
||||
The output is TSV and is intended to run beside warp/samply validation.
|
||||
|
||||
Options:
|
||||
--nodes <csv> Comma-separated node list, for example vm004,vm005.
|
||||
--duration-secs <n> Total sampling window in seconds.
|
||||
--out <file> TSV output path.
|
||||
--interval-secs <n> Sampling interval in seconds. Default: 5.
|
||||
--ssh-bin <path> SSH binary or test double. Default: ssh.
|
||||
-h, --help Show this help.
|
||||
USAGE
|
||||
}
|
||||
|
||||
die() {
|
||||
echo "error: $*" >&2
|
||||
exit 2
|
||||
}
|
||||
|
||||
shell_quote() {
|
||||
local value=${1//\'/\'\\\'\'}
|
||||
printf "'%s'" "$value"
|
||||
}
|
||||
|
||||
validate_node() {
|
||||
[[ "$1" =~ ^[A-Za-z0-9._-]+$ ]] || die "node contains unsafe characters: $1"
|
||||
}
|
||||
|
||||
NODES_CSV=""
|
||||
DURATION_SECS=""
|
||||
OUT=""
|
||||
INTERVAL_SECS="5"
|
||||
SSH_BIN="${SSH_BIN:-ssh}"
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--nodes) NODES_CSV="${2:-}"; shift 2 ;;
|
||||
--duration-secs) DURATION_SECS="${2:-}"; shift 2 ;;
|
||||
--out) OUT="${2:-}"; shift 2 ;;
|
||||
--interval-secs) INTERVAL_SECS="${2:-}"; shift 2 ;;
|
||||
--ssh-bin) SSH_BIN="${2:-}"; shift 2 ;;
|
||||
-h|--help) usage; exit 0 ;;
|
||||
*) die "unknown argument: $1" ;;
|
||||
esac
|
||||
done
|
||||
|
||||
[[ -n "$NODES_CSV" ]] || die "--nodes is required"
|
||||
[[ "$DURATION_SECS" =~ ^[0-9]+$ && "$DURATION_SECS" -gt 0 ]] || die "--duration-secs must be a positive integer"
|
||||
[[ "$INTERVAL_SECS" =~ ^[0-9]+$ && "$INTERVAL_SECS" -gt 0 ]] || die "--interval-secs must be a positive integer"
|
||||
[[ -n "$OUT" ]] || die "--out is required"
|
||||
|
||||
IFS=',' read -r -a nodes <<<"$NODES_CSV"
|
||||
[[ "${#nodes[@]}" -gt 0 ]] || die "--nodes did not contain any nodes"
|
||||
for node in "${nodes[@]}"; do
|
||||
[[ -n "$node" ]] || die "--nodes contains an empty entry"
|
||||
validate_node "$node"
|
||||
done
|
||||
|
||||
mkdir -p "$(dirname "$OUT")"
|
||||
printf 'ts_utc\tnode\tpid\tpcpu\trss_kib\tetime\n' >"$OUT"
|
||||
|
||||
# shellcheck disable=SC2016
|
||||
remote_inner='pid=$(pidof rustfs 2>/dev/null | awk "{print \$1}" || true); if [ -n "$pid" ]; then ps -o pid=,pcpu=,rss=,etime= -p "$pid"; fi'
|
||||
remote_cmd="sudo su - root -c $(shell_quote "$remote_inner")"
|
||||
|
||||
deadline=$((SECONDS + DURATION_SECS))
|
||||
while (( SECONDS < deadline )); do
|
||||
ts=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
|
||||
for node in "${nodes[@]}"; do
|
||||
sample=$("$SSH_BIN" "$node" "$remote_cmd" </dev/null || true)
|
||||
if [[ -n "${sample//[[:space:]]/}" ]]; then
|
||||
while read -r pid pcpu rss_kib etime extra; do
|
||||
[[ -n "${pid:-}" && -n "${pcpu:-}" && -n "${rss_kib:-}" && -n "${etime:-}" && -z "${extra:-}" ]] || continue
|
||||
printf '%s\t%s\t%s\t%s\t%s\t%s\n' "$ts" "$node" "$pid" "$pcpu" "$rss_kib" "$etime" >>"$OUT"
|
||||
done <<<"$sample"
|
||||
fi
|
||||
done
|
||||
(( SECONDS >= deadline )) && break
|
||||
sleep "$INTERVAL_SECS"
|
||||
done
|
||||
|
||||
echo "rss_samples=$(( $(wc -l <"$OUT") - 1 ))"
|
||||
Executable
+360
@@ -0,0 +1,360 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Summarize a samply/Firefox profile with a samply .syms.json sidecar."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import bisect
|
||||
import collections
|
||||
import gzip
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
ADDRESS_RE = re.compile(r"^0x[0-9a-fA-F]+$")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Symbol:
|
||||
rva: int
|
||||
size: int
|
||||
name: str
|
||||
library: str
|
||||
code_id: str | None
|
||||
|
||||
@property
|
||||
def end(self) -> int:
|
||||
return self.rva + max(self.size, 1)
|
||||
|
||||
|
||||
class IntervalIndex:
|
||||
def __init__(self, symbols: list[Symbol]) -> None:
|
||||
self._symbols = sorted(symbols, key=lambda symbol: symbol.rva)
|
||||
self._starts = [symbol.rva for symbol in self._symbols]
|
||||
|
||||
def lookup(self, address: int) -> Symbol | None:
|
||||
pos = bisect.bisect_right(self._starts, address)
|
||||
for symbol in reversed(self._symbols[max(0, pos - 8) : pos]):
|
||||
if symbol.rva <= address < symbol.end:
|
||||
return symbol
|
||||
return None
|
||||
|
||||
|
||||
class SymbolIndex:
|
||||
def __init__(self, symbols: list[Symbol]) -> None:
|
||||
self._all = IntervalIndex(symbols)
|
||||
by_code_id: dict[str, list[Symbol]] = collections.defaultdict(list)
|
||||
for symbol in symbols:
|
||||
if symbol.code_id:
|
||||
by_code_id[symbol.code_id.lower()].append(symbol)
|
||||
self._by_code_id = {code_id: IntervalIndex(items) for code_id, items in by_code_id.items()}
|
||||
|
||||
def lookup(self, address: int, code_id: str | None) -> Symbol | None:
|
||||
if code_id:
|
||||
index = self._by_code_id.get(code_id.lower())
|
||||
if index:
|
||||
symbol = index.lookup(address)
|
||||
if symbol:
|
||||
return symbol
|
||||
return self._all.lookup(address)
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Join a samply profile.json.gz and .syms.json sidecar into function-level hotpath counters.",
|
||||
)
|
||||
parser.add_argument("--profile", required=True, type=Path, help="samply Firefox profile JSON or JSON.GZ")
|
||||
parser.add_argument("--symbols", required=True, type=Path, help="samply .syms.json sidecar")
|
||||
parser.add_argument("--limit", type=int, default=20, help="number of functions to show per section")
|
||||
parser.add_argument("--thread", help="regular expression used to include matching thread names only")
|
||||
parser.add_argument(
|
||||
"--format",
|
||||
choices=("markdown", "json"),
|
||||
default="markdown",
|
||||
help="output format; markdown is issue-comment friendly",
|
||||
)
|
||||
parser.add_argument("--max-name-len", type=int, default=160, help="truncate long function names in markdown output")
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def load_json(path: Path) -> Any:
|
||||
if path.suffix == ".gz":
|
||||
with gzip.open(path, "rt", encoding="utf-8") as source:
|
||||
return json.load(source)
|
||||
with path.open("r", encoding="utf-8") as source:
|
||||
return json.load(source)
|
||||
|
||||
|
||||
def string_at(strings: list[Any], value: Any) -> str | None:
|
||||
if isinstance(value, int) and 0 <= value < len(strings):
|
||||
return str(strings[value])
|
||||
if isinstance(value, str):
|
||||
return value
|
||||
return None
|
||||
|
||||
|
||||
def parse_int(value: Any) -> int | None:
|
||||
if isinstance(value, int):
|
||||
return value
|
||||
if isinstance(value, str):
|
||||
try:
|
||||
return int(value, 0)
|
||||
except ValueError:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def iter_symbol_libraries(symbols_json: dict[str, Any]) -> list[tuple[str, dict[str, Any]]]:
|
||||
data = symbols_json.get("data", {})
|
||||
if isinstance(data, dict):
|
||||
return [(str(name), library) for name, library in data.items() if isinstance(library, dict)]
|
||||
if isinstance(data, list):
|
||||
result = []
|
||||
for index, library in enumerate(data):
|
||||
if isinstance(library, dict):
|
||||
name = library.get("debug_name") or library.get("name") or f"library-{index}"
|
||||
result.append((str(name), library))
|
||||
return result
|
||||
return []
|
||||
|
||||
|
||||
def symbol_name(entry: dict[str, Any], strings: list[Any]) -> str | None:
|
||||
symbol = string_at(strings, entry.get("symbol"))
|
||||
if symbol:
|
||||
return symbol
|
||||
|
||||
frames = entry.get("frames")
|
||||
if isinstance(frames, list):
|
||||
for frame in reversed(frames):
|
||||
if isinstance(frame, dict):
|
||||
name = string_at(strings, frame.get("function"))
|
||||
if name:
|
||||
return name
|
||||
return None
|
||||
|
||||
|
||||
def load_symbols(path: Path) -> SymbolIndex:
|
||||
symbols_json = load_json(path)
|
||||
if not isinstance(symbols_json, dict):
|
||||
raise ValueError("symbol sidecar does not look like samply .syms.json")
|
||||
strings = symbols_json.get("string_table", [])
|
||||
if not isinstance(strings, list):
|
||||
raise ValueError("symbol sidecar does not look like samply .syms.json")
|
||||
|
||||
symbols: list[Symbol] = []
|
||||
for library_name, library in iter_symbol_libraries(symbols_json):
|
||||
table = library.get("symbol_table", [])
|
||||
if not isinstance(table, list):
|
||||
continue
|
||||
code_id = library.get("code_id") or library.get("codeId")
|
||||
for entry in table:
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
rva = parse_int(entry.get("rva"))
|
||||
size = parse_int(entry.get("size")) or 1
|
||||
name = symbol_name(entry, strings)
|
||||
if rva is None or not name:
|
||||
continue
|
||||
symbols.append(Symbol(rva=rva, size=size, name=name, library=library_name, code_id=str(code_id) if code_id else None))
|
||||
|
||||
if not symbols:
|
||||
raise ValueError("symbol sidecar did not contain any usable symbols")
|
||||
return SymbolIndex(symbols)
|
||||
|
||||
|
||||
def table_get(table: dict[str, Any], column: str, index: int) -> Any:
|
||||
values = table.get(column, [])
|
||||
if isinstance(values, list) and 0 <= index < len(values):
|
||||
return values[index]
|
||||
return None
|
||||
|
||||
|
||||
def sample_stacks(samples: dict[str, Any]) -> list[Any]:
|
||||
if isinstance(samples.get("stack"), list):
|
||||
return samples["stack"]
|
||||
data = samples.get("data")
|
||||
schema = samples.get("schema", {})
|
||||
stack_column = schema.get("stack")
|
||||
if isinstance(data, list) and isinstance(stack_column, int):
|
||||
return [row[stack_column] if isinstance(row, list) and stack_column < len(row) else None for row in data]
|
||||
return []
|
||||
|
||||
|
||||
def frame_code_id(profile_libs: list[Any], thread: dict[str, Any], func_index: int | None) -> str | None:
|
||||
if func_index is None:
|
||||
return None
|
||||
resource_index = table_get(thread.get("funcTable", {}), "resource", func_index)
|
||||
if not isinstance(resource_index, int):
|
||||
return None
|
||||
lib_index = table_get(thread.get("resourceTable", {}), "lib", resource_index)
|
||||
if not isinstance(lib_index, int) or not (0 <= lib_index < len(profile_libs)):
|
||||
return None
|
||||
library = profile_libs[lib_index]
|
||||
if not isinstance(library, dict):
|
||||
return None
|
||||
code_id = library.get("codeId") or library.get("code_id")
|
||||
return str(code_id) if code_id else None
|
||||
|
||||
|
||||
def frame_name(profile_libs: list[Any], thread: dict[str, Any], frame_index: int, symbols: SymbolIndex) -> tuple[str, bool]:
|
||||
frame_table = thread.get("frameTable", {})
|
||||
func_table = thread.get("funcTable", {})
|
||||
strings = thread.get("stringArray", [])
|
||||
address = parse_int(table_get(frame_table, "address", frame_index))
|
||||
func_index = table_get(frame_table, "func", frame_index)
|
||||
typed_func_index = func_index if isinstance(func_index, int) else None
|
||||
if address is not None:
|
||||
symbol = symbols.lookup(address, frame_code_id(profile_libs, thread, typed_func_index))
|
||||
if symbol:
|
||||
return f"{symbol.name} [{symbol.library}]", True
|
||||
|
||||
if typed_func_index is not None:
|
||||
name_index = table_get(func_table, "name", typed_func_index)
|
||||
name = string_at(strings, name_index)
|
||||
if name and not ADDRESS_RE.match(name):
|
||||
return name, False
|
||||
|
||||
if address is not None:
|
||||
return f"0x{address:x}", False
|
||||
return "<unknown>", False
|
||||
|
||||
|
||||
def stack_frames(thread: dict[str, Any], stack_index: Any) -> list[int]:
|
||||
stack_table = thread.get("stackTable", {})
|
||||
if not isinstance(stack_index, int):
|
||||
return []
|
||||
|
||||
frames: list[int] = []
|
||||
seen: set[int] = set()
|
||||
current: int | None = stack_index
|
||||
while current is not None and current not in seen:
|
||||
seen.add(current)
|
||||
frame = table_get(stack_table, "frame", current)
|
||||
if isinstance(frame, int):
|
||||
frames.append(frame)
|
||||
prefix = table_get(stack_table, "prefix", current)
|
||||
current = prefix if isinstance(prefix, int) else None
|
||||
frames.reverse()
|
||||
return frames
|
||||
|
||||
|
||||
def summarize(profile: dict[str, Any], symbols: SymbolIndex, thread_filter: str | None) -> dict[str, Any]:
|
||||
thread_re = re.compile(thread_filter) if thread_filter else None
|
||||
profile_libs = profile.get("libs", [])
|
||||
if not isinstance(profile_libs, list):
|
||||
profile_libs = []
|
||||
leaf: collections.Counter[str] = collections.Counter()
|
||||
inclusive: collections.Counter[str] = collections.Counter()
|
||||
thread_counts: collections.Counter[str] = collections.Counter()
|
||||
resolved_samples = 0
|
||||
unresolved_samples = 0
|
||||
total_samples = 0
|
||||
|
||||
for thread in profile.get("threads", []):
|
||||
if not isinstance(thread, dict):
|
||||
continue
|
||||
thread_name = str(thread.get("name") or thread.get("processName") or "<unnamed>")
|
||||
if thread_re and not thread_re.search(thread_name):
|
||||
continue
|
||||
|
||||
for stack_index in sample_stacks(thread.get("samples", {})):
|
||||
frames = stack_frames(thread, stack_index)
|
||||
if not frames:
|
||||
continue
|
||||
names: list[str] = []
|
||||
any_resolved = False
|
||||
for frame_index in frames:
|
||||
name, resolved = frame_name(profile_libs, thread, frame_index, symbols)
|
||||
names.append(name)
|
||||
any_resolved = any_resolved or resolved
|
||||
leaf[names[-1]] += 1
|
||||
inclusive.update(set(names))
|
||||
thread_counts[thread_name] += 1
|
||||
total_samples += 1
|
||||
if any_resolved:
|
||||
resolved_samples += 1
|
||||
else:
|
||||
unresolved_samples += 1
|
||||
|
||||
return {
|
||||
"total_samples": total_samples,
|
||||
"resolved_samples": resolved_samples,
|
||||
"unresolved_samples": unresolved_samples,
|
||||
"threads": dict(thread_counts.most_common()),
|
||||
"leaf": leaf,
|
||||
"inclusive": inclusive,
|
||||
}
|
||||
|
||||
|
||||
def counter_rows(counter: collections.Counter[str], total: int, limit: int) -> list[dict[str, Any]]:
|
||||
rows = []
|
||||
for name, count in counter.most_common(limit):
|
||||
rows.append({"function": name, "samples": count, "percent": round((count * 100.0 / total), 2) if total else 0.0})
|
||||
return rows
|
||||
|
||||
|
||||
def truncate_name(name: str, max_len: int) -> str:
|
||||
if max_len <= 0 or len(name) <= max_len:
|
||||
return name
|
||||
if max_len <= 3:
|
||||
return name[:max_len]
|
||||
return f"{name[: max_len - 3]}..."
|
||||
|
||||
|
||||
def print_markdown(summary: dict[str, Any], limit: int, max_name_len: int) -> None:
|
||||
total = int(summary["total_samples"])
|
||||
resolved = int(summary["resolved_samples"])
|
||||
unresolved = int(summary["unresolved_samples"])
|
||||
print(f"- samples: {total}, resolved stacks: {resolved}, unresolved stacks: {unresolved}")
|
||||
if summary["threads"]:
|
||||
print("- threads:")
|
||||
for thread, count in summary["threads"].items():
|
||||
pct = (count * 100.0 / total) if total else 0.0
|
||||
print(f" - `{thread}`: {count} ({pct:.2f}%)")
|
||||
|
||||
for title, key in (("Top leaf functions", "leaf"), ("Top inclusive functions", "inclusive")):
|
||||
print()
|
||||
print(f"### {title}")
|
||||
print("| rank | samples | pct | function |")
|
||||
print("|---:|---:|---:|---|")
|
||||
for rank, row in enumerate(counter_rows(summary[key], total, limit), start=1):
|
||||
function = truncate_name(str(row["function"]), max_name_len).replace("|", "\\|")
|
||||
print(f"| {rank} | {row['samples']} | {row['percent']:.2f}% | `{function}` |")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
if args.limit <= 0:
|
||||
print("error: --limit must be positive", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
profile = load_json(args.profile)
|
||||
symbols = load_symbols(args.symbols)
|
||||
summary = summarize(profile, symbols, args.thread)
|
||||
if args.format == "json":
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"total_samples": summary["total_samples"],
|
||||
"resolved_samples": summary["resolved_samples"],
|
||||
"unresolved_samples": summary["unresolved_samples"],
|
||||
"threads": summary["threads"],
|
||||
"leaf": counter_rows(summary["leaf"], summary["total_samples"], args.limit),
|
||||
"inclusive": counter_rows(summary["inclusive"], summary["total_samples"], args.limit),
|
||||
},
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
sort_keys=True,
|
||||
)
|
||||
)
|
||||
else:
|
||||
print_markdown(summary, args.limit, args.max_name_len)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
+69
@@ -0,0 +1,69 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
|
||||
tmp_dir=$(mktemp -d)
|
||||
trap 'rm -rf "$tmp_dir"' EXIT
|
||||
|
||||
mock_bin="$tmp_dir/bin"
|
||||
mkdir -p "$mock_bin"
|
||||
|
||||
cat >"$mock_bin/ssh" <<'EOF'
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
printf 'ssh:%s\n' "$*" >>"$CALL_LOG"
|
||||
if [[ "${MOCK_JOURNAL_MODE:-}" == "clean" ]]; then
|
||||
echo "rustfs request completed"
|
||||
exit 0
|
||||
fi
|
||||
echo "rustfs request completed"
|
||||
echo "No valid auth token"
|
||||
echo "WARN replay cache overflow"
|
||||
EOF
|
||||
|
||||
chmod +x "$mock_bin/ssh"
|
||||
|
||||
export CALL_LOG="$tmp_dir/calls.log"
|
||||
output=$("$repo_root/scripts/capture_remote_journal_errors.sh" \
|
||||
--nodes "vm004,vm005" \
|
||||
--since "2026-08-08T09:05:45Z" \
|
||||
--label "get-1m" \
|
||||
--out-dir "$tmp_dir/out" \
|
||||
--ssh-bin "$mock_bin/ssh")
|
||||
|
||||
grep -q 'journal_since=2026-08-08 09:05:45 UTC' <<<"$output"
|
||||
grep -q 'captured_nodes=2' <<<"$output"
|
||||
grep -q '2026-08-08 09:05:45 UTC' "$CALL_LOG"
|
||||
grep -q 'sudo su - root -c' "$CALL_LOG"
|
||||
if grep -q '2026-08-08T09:05:45Z' "$CALL_LOG"; then
|
||||
echo "raw ISO timestamp was sent to journalctl" >&2
|
||||
exit 1
|
||||
fi
|
||||
grep -q 'No valid auth token' "$tmp_dir/out/get-1m-vm004-journal-errors.txt"
|
||||
grep -q 'WARN replay cache overflow' "$tmp_dir/out/get-1m-vm005-journal-errors.txt"
|
||||
if grep -q 'rustfs request completed' "$tmp_dir/out/get-1m-vm004-journal-errors.txt"; then
|
||||
echo "non-error journal line was not filtered out" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
export MOCK_JOURNAL_MODE=clean
|
||||
"$repo_root/scripts/capture_remote_journal_errors.sh" \
|
||||
--nodes "vm004" \
|
||||
--since "2026-08-08T09:05:45Z" \
|
||||
--label "clean" \
|
||||
--out-dir "$tmp_dir/clean-out" \
|
||||
--ssh-bin "$mock_bin/ssh" >"$tmp_dir/clean.stdout"
|
||||
[[ ! -s "$tmp_dir/clean-out/clean-vm004-journal-errors.txt" ]]
|
||||
|
||||
if "$repo_root/scripts/capture_remote_journal_errors.sh" \
|
||||
--nodes "vm004" \
|
||||
--since "2026-08-08T09:05:45Z" \
|
||||
--label "../escape" \
|
||||
--out-dir "$tmp_dir/unsafe-out" \
|
||||
--ssh-bin "$mock_bin/ssh" >"$tmp_dir/unsafe.stdout" 2>"$tmp_dir/unsafe.stderr"; then
|
||||
echo "unsafe label was accepted" >&2
|
||||
exit 1
|
||||
fi
|
||||
grep -q -- '--label contains unsafe characters' "$tmp_dir/unsafe.stderr"
|
||||
|
||||
echo "test_capture_remote_journal_errors: ok"
|
||||
+89
@@ -0,0 +1,89 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
|
||||
tmp_dir=$(mktemp -d)
|
||||
trap 'rm -rf "$tmp_dir"' EXIT
|
||||
|
||||
mock_bin="$tmp_dir/bin"
|
||||
mkdir -p "$mock_bin"
|
||||
|
||||
cat >"$mock_bin/ssh" <<'EOF'
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
if read -r consumed; then
|
||||
printf 'ssh-consumed-stdin:%s\n' "$consumed" >>"$CALL_LOG"
|
||||
exit 24
|
||||
fi
|
||||
printf 'ssh:%s\n' "$*" >>"$CALL_LOG"
|
||||
echo "profile.json.gz 128 bytes"
|
||||
echo "profile.syms.json 64 bytes"
|
||||
EOF
|
||||
|
||||
cat >"$mock_bin/scp" <<'EOF'
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
if read -r consumed; then
|
||||
printf 'scp-consumed-stdin:%s\n' "$consumed" >>"$CALL_LOG"
|
||||
exit 23
|
||||
fi
|
||||
printf 'scp:%s\n' "$*" >>"$CALL_LOG"
|
||||
dest="${@: -1}"
|
||||
mkdir -p "$dest"
|
||||
touch "$dest/copied-profile.json.gz"
|
||||
EOF
|
||||
|
||||
chmod +x "$mock_bin/ssh" "$mock_bin/scp"
|
||||
|
||||
mapping="$tmp_dir/nodes.txt"
|
||||
cat >"$mapping" <<'EOF'
|
||||
vm004 /data/rustfs/hotpath/put-1m
|
||||
vm005 /data/rustfs/hotpath/get-1m
|
||||
EOF
|
||||
|
||||
export CALL_LOG="$tmp_dir/calls.log"
|
||||
output=$("$repo_root/scripts/collect_remote_samply_artifacts.sh" \
|
||||
--mapping "$mapping" \
|
||||
--out-dir "$tmp_dir/out" \
|
||||
--ssh-bin "$mock_bin/ssh" \
|
||||
--scp-bin "$mock_bin/scp")
|
||||
|
||||
[[ "$output" == "collected_nodes=2" ]]
|
||||
[[ -f "$tmp_dir/out/vm004/copied-profile.json.gz" ]]
|
||||
[[ -f "$tmp_dir/out/vm005/copied-profile.json.gz" ]]
|
||||
[[ "$(grep -c '^ssh:' "$CALL_LOG")" -eq 2 ]]
|
||||
[[ "$(grep -c '^scp:' "$CALL_LOG")" -eq 2 ]]
|
||||
if grep -q '^ssh-consumed-stdin:' "$CALL_LOG"; then
|
||||
echo "ssh consumed the mapping loop stdin" >&2
|
||||
exit 1
|
||||
fi
|
||||
if grep -q '^scp-consumed-stdin:' "$CALL_LOG"; then
|
||||
echo "scp consumed the mapping loop stdin" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
bad_mapping="$tmp_dir/bad-nodes.txt"
|
||||
printf 'vm004 /\n' >"$bad_mapping"
|
||||
if "$repo_root/scripts/collect_remote_samply_artifacts.sh" \
|
||||
--mapping "$bad_mapping" \
|
||||
--out-dir "$tmp_dir/bad-out" \
|
||||
--ssh-bin "$mock_bin/ssh" \
|
||||
--scp-bin "$mock_bin/scp" >"$tmp_dir/bad.stdout" 2>"$tmp_dir/bad.stderr"; then
|
||||
echo "unsafe remote directory was accepted" >&2
|
||||
exit 1
|
||||
fi
|
||||
grep -q 'path contains unsafe characters' "$tmp_dir/bad.stderr"
|
||||
|
||||
unsafe_mapping="$tmp_dir/unsafe-nodes.txt"
|
||||
printf 'vm004 /data/rustfs/hotpath/put;rm\n' >"$unsafe_mapping"
|
||||
if "$repo_root/scripts/collect_remote_samply_artifacts.sh" \
|
||||
--mapping "$unsafe_mapping" \
|
||||
--out-dir "$tmp_dir/unsafe-out" \
|
||||
--ssh-bin "$mock_bin/ssh" \
|
||||
--scp-bin "$mock_bin/scp" >"$tmp_dir/unsafe.stdout" 2>"$tmp_dir/unsafe.stderr"; then
|
||||
echo "unsafe remote path was accepted" >&2
|
||||
exit 1
|
||||
fi
|
||||
grep -q 'path contains unsafe characters' "$tmp_dir/unsafe.stderr"
|
||||
|
||||
echo "test_collect_remote_samply_artifacts: ok"
|
||||
Executable
+51
@@ -0,0 +1,51 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
|
||||
tmp_dir=$(mktemp -d)
|
||||
trap 'rm -rf "$tmp_dir"' EXIT
|
||||
|
||||
mock_ssh="$tmp_dir/ssh"
|
||||
cat >"$mock_ssh" <<'EOF'
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
if read -r consumed; then
|
||||
printf 'ssh-consumed-stdin:%s\n' "$consumed" >>"$CALL_LOG"
|
||||
exit 24
|
||||
fi
|
||||
printf 'ssh:%s\n' "$*" >>"$CALL_LOG"
|
||||
echo "123 12.5 456789 00:01:02"
|
||||
EOF
|
||||
chmod +x "$mock_ssh"
|
||||
|
||||
export CALL_LOG="$tmp_dir/calls.log"
|
||||
output=$("$repo_root/scripts/sample_remote_rustfs_rss.sh" \
|
||||
--nodes vm004,vm005 \
|
||||
--duration-secs 1 \
|
||||
--interval-secs 1 \
|
||||
--out "$tmp_dir/rss.tsv" \
|
||||
--ssh-bin "$mock_ssh")
|
||||
|
||||
[[ "$output" == "rss_samples=2" ]]
|
||||
[[ "$(wc -l <"$tmp_dir/rss.tsv")" -eq 3 ]]
|
||||
grep -q $'^ts_utc\tnode\tpid\tpcpu\trss_kib\tetime$' "$tmp_dir/rss.tsv"
|
||||
grep -q $'\tvm004\t123\t12.5\t456789\t00:01:02$' "$tmp_dir/rss.tsv"
|
||||
grep -q $'\tvm005\t123\t12.5\t456789\t00:01:02$' "$tmp_dir/rss.tsv"
|
||||
[[ "$(grep -c '^ssh:' "$CALL_LOG")" -eq 2 ]]
|
||||
if grep -q '^ssh-consumed-stdin:' "$CALL_LOG"; then
|
||||
echo "ssh consumed the sampling loop stdin" >&2
|
||||
exit 1
|
||||
fi
|
||||
grep -q 'sudo su - root -c' "$CALL_LOG"
|
||||
|
||||
if "$repo_root/scripts/sample_remote_rustfs_rss.sh" \
|
||||
--nodes 'vm004;rm' \
|
||||
--duration-secs 1 \
|
||||
--out "$tmp_dir/bad.tsv" \
|
||||
--ssh-bin "$mock_ssh" >"$tmp_dir/bad.stdout" 2>"$tmp_dir/bad.stderr"; then
|
||||
echo "unsafe node name was accepted" >&2
|
||||
exit 1
|
||||
fi
|
||||
grep -q 'node contains unsafe characters' "$tmp_dir/bad.stderr"
|
||||
|
||||
echo "test_sample_remote_rustfs_rss: ok"
|
||||
+114
@@ -0,0 +1,114 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import gzip
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def write_json_gz(path: Path, value: object) -> None:
|
||||
with gzip.open(path, "wt", encoding="utf-8") as target:
|
||||
json.dump(value, target)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
tmp_path = Path(tmp)
|
||||
profile_path = tmp_path / "profile.json.gz"
|
||||
symbols_path = tmp_path / "profile.syms.json"
|
||||
|
||||
profile = {
|
||||
"libs": [
|
||||
{"debugName": "rustfs", "codeId": "abc"},
|
||||
{"debugName": "libc.so.6", "codeId": "def"},
|
||||
],
|
||||
"threads": [
|
||||
{
|
||||
"name": "rustfs-worker",
|
||||
"stringArray": ["0x1004", "0x2004"],
|
||||
"resourceTable": {"length": 2, "lib": [0, 1], "name": [0, 1], "host": [None, None], "type": [1, 1]},
|
||||
"funcTable": {"length": 2, "name": [0, 1], "resource": [0, 1]},
|
||||
"frameTable": {"length": 2, "address": [0x1004, 0x2004], "func": [0, 1]},
|
||||
"stackTable": {"length": 2, "prefix": [None, 0], "frame": [0, 1]},
|
||||
"samples": {"stack": [0, 1]},
|
||||
},
|
||||
{
|
||||
"name": "tokio-runtime-worker",
|
||||
"stringArray": ["0x1008"],
|
||||
"resourceTable": {"length": 1, "lib": [0], "name": [0], "host": [None], "type": [1]},
|
||||
"funcTable": {"length": 1, "name": [0], "resource": [0]},
|
||||
"frameTable": {"length": 1, "address": [0x1008], "func": [0]},
|
||||
"stackTable": {"length": 1, "prefix": [None], "frame": [0]},
|
||||
"samples": {"stack": [0]},
|
||||
},
|
||||
],
|
||||
}
|
||||
symbols = {
|
||||
"string_table": [
|
||||
"rustfs_ecstore::set_disk::read_all_data",
|
||||
"libc::writev",
|
||||
"rustfs_ecstore::set_disk::read_all_inline_data",
|
||||
],
|
||||
"data": {
|
||||
"rustfs": {
|
||||
"code_id": "abc",
|
||||
"symbol_table": [
|
||||
{"rva": 0x1000, "size": 0x10, "symbol": 0},
|
||||
],
|
||||
},
|
||||
"libc.so.6": {
|
||||
"code_id": "def",
|
||||
"symbol_table": [
|
||||
{"rva": 0x1000, "size": 0x10, "symbol": 2},
|
||||
{"rva": 0x2000, "size": 0x10, "symbol": 1},
|
||||
],
|
||||
},
|
||||
},
|
||||
}
|
||||
write_json_gz(profile_path, profile)
|
||||
symbols_path.write_text(json.dumps(symbols), encoding="utf-8")
|
||||
|
||||
result = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
str(REPO_ROOT / "scripts" / "summarize_samply_profile_symbols.py"),
|
||||
"--profile",
|
||||
str(profile_path),
|
||||
"--symbols",
|
||||
str(symbols_path),
|
||||
"--thread",
|
||||
"rustfs-worker",
|
||||
"--format",
|
||||
"json",
|
||||
"--limit",
|
||||
"5",
|
||||
],
|
||||
check=True,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
)
|
||||
summary = json.loads(result.stdout)
|
||||
|
||||
assert summary["total_samples"] == 2
|
||||
assert summary["resolved_samples"] == 2
|
||||
assert summary["unresolved_samples"] == 0
|
||||
assert summary["threads"] == {"rustfs-worker": 2}
|
||||
leaf_names = [row["function"] for row in summary["leaf"]]
|
||||
assert "rustfs_ecstore::set_disk::read_all_data [rustfs]" in leaf_names
|
||||
assert "libc::writev [libc.so.6]" in leaf_names
|
||||
inclusive_names = [row["function"] for row in summary["inclusive"]]
|
||||
assert "rustfs_ecstore::set_disk::read_all_data [rustfs]" in inclusive_names
|
||||
|
||||
print("test_summarize_samply_profile_symbols: ok")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user