gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-07-26 16:28:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor(tls): centralize runtime foundation (#3065)

Browse Source 
* refactor(targets): move notify net helpers from utils

* refactor(tls): centralize runtime foundation

* refactor(targets): move notify net helpers from utils

* refactor(tls): centralize runtime foundation

* feat(tls-runtime): add TLS debug state and admin handler

* refactor(tls-runtime): unify TLS debug consumer status view

* fix(tls): address PR3065 review feedback

* refactor(tls): align debug status payload types

* refactor(targets): harden TLS hot reload paths

* fix(targets): resolve review-4348251652 findings

* fix(targets): finalize tls runtime review follow-ups

* fix(targets): harden tls reload and review follow-ups

* fix(targets): align tls reload handling across targets

* fix(targets): finalize tls reload state and metrics updates

* chore(deps): trim unused TLS deps

* style(targets): normalize TLS reload formatting

* refactor(targets): introduce tls runtime adapter path

* chore: update workspace manifests for tls refactor

* fix(tls): stabilize material reload and audit workflow

* fix(targets): refresh tls fingerprint flow across sinks

* fix(tls): align runtime coordinator and http reader updates

* fix(sftp): simplify protocol error mapping

* fix(tls): harmonize material loading behavior

* fix(server): finalize tls material wiring in startup flow

* fix(protos): tighten tls generation cache and deps
This commit is contained in:
houseme
2026-05-24 14:41:15 +08:00
committed by GitHub
parent 8be787387c
commit d74e6eb042
67 changed files with 4978 additions and 1725 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/audit.yml
+2 -2
View File
@@ -75,7 +75,7 @@ jobs:
uses: actions/checkout@v6
- name: Dependency Review
uses: actions/dependency-review-action@v4
uses: actions/dependency-review-action@v5
with:
fail-on-severity: moderate
comment-summary-in-pr: true
comment-summary-in-pr: always
Cargo.lock
Generated
+88 -136
View File
@@ -761,9 +761,9 @@ dependencies = [
[[package]]
name = "async-rs"
version = "0.8.7"
version = "0.8.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "53bf71bee8a75907b6e3c81c5476efa7fcbb34df6e12d30b706888abded72091"
checksum = "d096d3e7d6c2cd3ff2bebba9273fef6e0c129ee04ec6c16a8023c19958b2cc90"
dependencies = [
"async-compat",
"async-global-executor",
@@ -840,9 +840,9 @@ dependencies = [
[[package]]
name = "autocfg"
version = "1.5.0"
version = "1.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8"
checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
[[package]]
name = "aws-config"
@@ -1645,9 +1645,9 @@ dependencies = [
[[package]]
name = "bumpalo"
version = "3.20.2"
version = "3.20.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb"
checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649"
[[package]]
name = "bytemuck"
@@ -3517,9 +3517,9 @@ dependencies = [
[[package]]
name = "dial9-tokio-telemetry"
version = "0.3.10"
version = "0.3.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "45521f70d67ff82b19ffddb55c7f039ee24c44a172aeb03ec2142f1808ae8562"
checksum = "7a8b3e96f10aa94341f08a672cf2bf4df3a0128a396e5d2ea2d41724205fd2c3"
dependencies = [
"arc-swap",
"bon",
@@ -3613,7 +3613,7 @@ dependencies = [
"libc",
"option-ext",
"redox_users",
"windows-sys 0.61.2",
"windows-sys 0.59.0",
]
[[package]]
@@ -3949,7 +3949,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
dependencies = [
"libc",
"windows-sys 0.61.2",
"windows-sys 0.59.0",
]
[[package]]
@@ -5426,7 +5426,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46"
dependencies = [
"hermit-abi",
"libc",
"windows-sys 0.61.2",
"windows-sys 0.59.0",
]
[[package]]
@@ -5503,7 +5503,7 @@ dependencies = [
"portable-atomic",
"portable-atomic-util",
"serde_core",
"windows-sys 0.61.2",
"windows-sys 0.59.0",
]
[[package]]
@@ -5593,9 +5593,9 @@ dependencies = [
[[package]]
name = "js-sys"
version = "0.3.98"
version = "0.3.99"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "67df7112613f8bfd9150013a0314e196f4800d3201ae742489d999db2f979f08"
checksum = "142bc4740e452c1e57ade0cbc129f139c9093e354346f0872ef985f4f5cf5f11"
dependencies = [
"cfg-if",
"futures-util",
@@ -5867,9 +5867,9 @@ checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981"
[[package]]
name = "libmimalloc-sys"
version = "0.1.48"
version = "0.1.49"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2892ae4ea6fa2cb7acb0e236a6880d39523239cd9089de71d220910ccc806790"
checksum = "6a45a52f43e1c16f667ccfe4dd8c85b7f7c204fd5e3bf46c5b0db9a5c3c0b8e9"
dependencies = [
"cc",
"cty",
@@ -6287,9 +6287,9 @@ dependencies = [
[[package]]
name = "mimalloc"
version = "0.1.51"
version = "0.1.52"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ebca48a43116bc25f18a61360f1be98412f50cc218f5e52c823086b999a4a21a"
checksum = "2d4139bb28d14ad1facf21d5eb8825051b326e172d216b39f6d31df53cc97862"
dependencies = [
"libmimalloc-sys",
]
@@ -6639,7 +6639,7 @@ version = "0.50.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5"
dependencies = [
"windows-sys 0.61.2",
"windows-sys 0.59.0",
]
[[package]]
@@ -8415,7 +8415,7 @@ dependencies = [
"once_cell",
"socket2",
"tracing",
"windows-sys 0.60.2",
"windows-sys 0.59.0",
]
[[package]]
@@ -9101,9 +9101,9 @@ dependencies = [
[[package]]
name = "russh-sftp"
version = "2.2.2"
version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c826d40e310bbcb377fd8f6c5873c12eaf54da30569d8e8da56914fdac9773cd"
checksum = "9ed8949eca4163c18a8f59ff96d32cf61e9c13b9735e21ef32b3907f4aafa1a9"
dependencies = [
"bitflags 2.11.1",
"bytes",
@@ -9288,10 +9288,12 @@ dependencies = [
"rustfs-scanner",
"rustfs-signer",
"rustfs-targets",
"rustfs-tls-runtime",
"rustfs-trusted-proxies",
"rustfs-utils",
"rustfs-zip",
"rustls",
"rustls-pki-types",
"s3s",
"serde",
"serde_json",
@@ -9510,9 +9512,11 @@ dependencies = [
"rustfs-rio",
"rustfs-s3-types",
"rustfs-signer",
"rustfs-tls-runtime",
"rustfs-utils",
"rustix 1.1.4",
"rustls",
"rustls-pki-types",
"s3s",
"serde",
"serde_json",
@@ -9937,7 +9941,6 @@ dependencies = [
"percent-encoding",
"proptest",
"quick-xml 0.40.1",
"rcgen",
"regex",
"russh",
"russh-sftp",
@@ -9948,6 +9951,7 @@ dependencies = [
"rustfs-keystone",
"rustfs-policy",
"rustfs-rio",
"rustfs-tls-runtime",
"rustfs-utils",
"rustls",
"s3s",
@@ -9980,7 +9984,9 @@ dependencies = [
"rustfs-common",
"rustfs-config",
"rustfs-io-metrics",
"rustfs-tls-runtime",
"rustfs-utils",
"tokio",
"tonic",
"tonic-prost",
"tonic-prost-build",
@@ -10007,7 +10013,9 @@ dependencies = [
"reqwest",
"rustfs-config",
"rustfs-io-metrics",
"rustfs-tls-runtime",
"rustfs-utils",
"rustls-pki-types",
"s3s",
"serde",
"serde_json",
@@ -10130,28 +10138,33 @@ dependencies = [
name = "rustfs-targets"
version = "1.0.0-beta.4"
dependencies = [
"arc-swap",
"async-nats",
"async-trait",
"chrono",
"criterion",
"deadpool-postgres",
"hashbrown 0.17.1",
"hyper",
"hyper-rustls",
"lapin",
"libc",
"metrics",
"mysql_async",
"parking_lot 0.12.5",
"pulsar",
"redis",
"regex",
"reqwest",
"rumqttc-next",
"rustfs-config",
"rustfs-ecstore",
"rustfs-kafka-async",
"rustfs-s3-types",
"rustfs-utils",
"rustfs-tls-runtime",
"rustls",
"rustls-native-certs",
"rustls-pki-types",
"s3s",
"serde",
"serde_json",
"snap",
@@ -10167,6 +10180,26 @@ dependencies = [
"uuid",
]
[[package]]
name = "rustfs-tls-runtime"
version = "1.0.0-beta.4"
dependencies = [
"arc-swap",
"metrics",
"rcgen",
"rustfs-common",
"rustfs-config",
"rustls",
"rustls-pki-types",
"serde",
"serde_json",
"sha2 0.11.0",
"tempfile",
"thiserror 2.0.18",
"tokio",
"tracing",
]
[[package]]
name = "rustfs-trusted-proxies"
version = "1.0.0-beta.4"
@@ -10203,23 +10236,17 @@ dependencies = [
"crc-fast",
"flate2",
"futures",
"hashbrown 0.17.1",
"hex-simd",
"highway",
"hmac 0.13.0",
"http 1.4.0",
"hyper",
"libc",
"local-ip-address",
"lz4",
"md-5 0.11.0",
"netif",
"rcgen",
"regex",
"rustix 1.1.4",
"rustls",
"rustls-pki-types",
"s3s",
"serde",
"sha1 0.11.0",
"sha2 0.11.0",
@@ -10227,7 +10254,6 @@ dependencies = [
"snap",
"temp-env",
"tempfile",
"thiserror 2.0.18",
"tokio",
"tracing",
"transform-stream",
@@ -10312,7 +10338,7 @@ dependencies = [
"errno",
"libc",
"linux-raw-sys 0.12.1",
"windows-sys 0.61.2",
"windows-sys 0.59.0",
]
[[package]]
@@ -10333,9 +10359,9 @@ dependencies = [
[[package]]
name = "rustls-connector"
version = "0.23.2"
version = "0.23.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "546c32c0a03187814b1e1239eec017778dc9b87d8241a2c5c1954c47ab8ac8fd"
checksum = "1869b2b1c5adf03c1025b23d8c23c36903a6bdd40a92332bfd8f309db5a460a1"
dependencies = [
"futures-io",
"futures-rustls",
@@ -10386,7 +10412,7 @@ dependencies = [
"security-framework",
"security-framework-sys",
"webpki-root-certs",
"windows-sys 0.61.2",
"windows-sys 0.59.0",
]
[[package]]
@@ -11493,9 +11519,9 @@ checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417"
[[package]]
name = "tcp-stream"
version = "0.34.10"
version = "0.34.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e8da40490cac3733b85c67b831f64e2132fdaa0929f42a6d4cf458d339777473"
checksum = "04a6c48788336a4223ddc6e4d4f2d623f4e5865058c0999d1502a8994816ec4f"
dependencies = [
"async-rs",
"cfg-if",
@@ -11524,7 +11550,7 @@ dependencies = [
"getrandom 0.4.2",
"once_cell",
"rustix 1.1.4",
"windows-sys 0.61.2",
"windows-sys 0.59.0",
]
[[package]]
@@ -12503,9 +12529,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen"
version = "0.2.121"
version = "0.2.122"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49ace1d07c165b0864824eee619580c4689389afa9dc9ed3a4c75040d82e6790"
checksum = "3ed04576f974d2b2fba0f38c51dbc5518011e38c36bf1143164be765528fd409"
dependencies = [
"cfg-if",
"once_cell",
@@ -12516,9 +12542,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-futures"
version = "0.4.71"
version = "0.4.72"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "96492d0d3ffba25305a7dc88720d250b1401d7edca02cc3bcd50633b424673b8"
checksum = "9473dbd2991ae90b6291c3c32c30c6187ac49aa32f9905d1cce280ec1e110b0f"
dependencies = [
"js-sys",
"wasm-bindgen",
@@ -12526,9 +12552,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-macro"
version = "0.2.121"
version = "0.2.122"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e68e6f4afd367a562002c05637acb8578ff2dea1943df76afb9e83d177c8578"
checksum = "916151b09da36bd82f6615cbf3a419e2f0ba23a03c6160e8e92eb6bd4aa1dec6"
dependencies = [
"quote",
"wasm-bindgen-macro-support",
@@ -12536,9 +12562,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-macro-support"
version = "0.2.121"
version = "0.2.122"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d95a9ec35c64b2a7cb35d3fead40c4238d0940c86d107136999567a4703259f2"
checksum = "299047362ccbfce148b67ab7e73349f77748e00c8296f9542adfad2ad82c5c5e"
dependencies = [
"bumpalo",
"proc-macro2",
@@ -12549,9 +12575,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-shared"
version = "0.2.121"
version = "0.2.122"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c4e0100b01e9f0d03189a92b96772a1fb998639d981193d7dbab487302513441"
checksum = "9a929b2c61f11ba3e9bc35b50c1f25cb38e0e892c0c231ae2b8cf78d5dad4437"
dependencies = [
"unicode-ident",
]
@@ -12605,9 +12631,9 @@ dependencies = [
[[package]]
name = "web-sys"
version = "0.3.98"
version = "0.3.99"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4b572dff8bcf38bad0fa19729c89bb5748b2b9b1d8be70cf90df697e3a8f32aa"
checksum = "6d621441cfc37b84979402712047321980c178f299193a3589d05b99e8763436"
dependencies = [
"js-sys",
"wasm-bindgen",
@@ -12712,7 +12738,7 @@ version = "0.1.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
dependencies = [
"windows-sys 0.61.2",
"windows-sys 0.59.0",
]
[[package]]
@@ -12839,7 +12865,7 @@ version = "0.52.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
dependencies = [
"windows-targets 0.52.6",
"windows-targets",
]
[[package]]
@@ -12848,16 +12874,7 @@ version = "0.59.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b"
dependencies = [
"windows-targets 0.52.6",
]
[[package]]
name = "windows-sys"
version = "0.60.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb"
dependencies = [
"windows-targets 0.53.5",
"windows-targets",
]
[[package]]
@@ -12875,31 +12892,14 @@ version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"
dependencies = [
"windows_aarch64_gnullvm 0.52.6",
"windows_aarch64_msvc 0.52.6",
"windows_i686_gnu 0.52.6",
"windows_i686_gnullvm 0.52.6",
"windows_i686_msvc 0.52.6",
"windows_x86_64_gnu 0.52.6",
"windows_x86_64_gnullvm 0.52.6",
"windows_x86_64_msvc 0.52.6",
]
[[package]]
name = "windows-targets"
version = "0.53.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3"
dependencies = [
"windows-link",
"windows_aarch64_gnullvm 0.53.1",
"windows_aarch64_msvc 0.53.1",
"windows_i686_gnu 0.53.1",
"windows_i686_gnullvm 0.53.1",
"windows_i686_msvc 0.53.1",
"windows_x86_64_gnu 0.53.1",
"windows_x86_64_gnullvm 0.53.1",
"windows_x86_64_msvc 0.53.1",
"windows_aarch64_gnullvm",
"windows_aarch64_msvc",
"windows_i686_gnu",
"windows_i686_gnullvm",
"windows_i686_msvc",
"windows_x86_64_gnu",
"windows_x86_64_gnullvm",
"windows_x86_64_msvc",
]
[[package]]
@@ -12917,96 +12917,48 @@ version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
[[package]]
name = "windows_aarch64_gnullvm"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53"
[[package]]
name = "windows_aarch64_msvc"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
[[package]]
name = "windows_aarch64_msvc"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006"
[[package]]
name = "windows_i686_gnu"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"
[[package]]
name = "windows_i686_gnu"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3"
[[package]]
name = "windows_i686_gnullvm"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
[[package]]
name = "windows_i686_gnullvm"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c"
[[package]]
name = "windows_i686_msvc"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
[[package]]
name = "windows_i686_msvc"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2"
[[package]]
name = "windows_x86_64_gnu"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
[[package]]
name = "windows_x86_64_gnu"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499"
[[package]]
name = "windows_x86_64_gnullvm"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
[[package]]
name = "windows_x86_64_gnullvm"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1"
[[package]]
name = "windows_x86_64_msvc"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
[[package]]
name = "windows_x86_64_msvc"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650"
[[package]]
name = "winnow"
version = "1.0.3"
Cargo.toml
+4 -2
View File
@@ -47,6 +47,7 @@ members = [
"crates/signer", # client signer
"crates/targets", # Target-specific configurations and utilities
"crates/trusted-proxies", # Trusted proxies management
"crates/tls-runtime", # Project-wide TLS runtime foundation
"crates/utils", # Utility functions and helpers
"crates/io-metrics", # Zero-copy metrics collection for performance analysis
"crates/io-core", # Zero-copy core reader and writer implementations
@@ -110,6 +111,7 @@ rustfs-scanner = { path = "crates/scanner", version = "1.0.0-beta.4" }
rustfs-signer = { path = "crates/signer", version = "1.0.0-beta.4" }
rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.0-beta.4" }
rustfs-targets = { path = "crates/targets", version = "1.0.0-beta.4" }
rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.0-beta.4" }
rustfs-utils = { path = "crates/utils", version = "1.0.0-beta.4" }
rustfs-zip = { path = "./crates/zip", version = "1.0.0-beta.4" }
@@ -141,7 +143,7 @@ tokio-rustls = { version = "0.26.4", default-features = false, features = ["logg
tokio-stream = { version = "0.1.18" }
tokio-test = "0.4.5"
tokio-util = { version = "0.7.18", features = ["io", "compat"] }
tonic = { version = "0.14.6", features = ["gzip"] }
tonic = { version = "0.14.6", features = ["gzip", "deflate"] }
tonic-prost = { version = "0.14.6" }
tonic-prost-build = { version = "0.14.6" }
tower = { version = "0.5.3", features = ["timeout"] }
@@ -307,7 +309,7 @@ unftp-core = "0.1.0"
suppaftp = { version = "8.0.3", features = ["tokio", "tokio-rustls-aws-lc-rs"] }
rcgen = "0.14.8"
russh = { version = "0.60.3", git = "https://github.com/Eugeny/russh", rev = "fc6e3ab4cd4338e94ae64e17aeed2acee9335e6b" }
russh-sftp = "2.2.2"
russh-sftp = "2.3.0"
# WebDAV
dav-server = "0.11.0"
crates/common/src/globals.rs
+12
View File
@@ -17,6 +17,7 @@
use chrono::{DateTime, Utc};
use std::collections::HashMap;
use std::sync::LazyLock;
use std::sync::atomic::{AtomicU64, Ordering};
use tokio::sync::RwLock;
use tonic::transport::Channel;
@@ -27,6 +28,7 @@ pub static GLOBAL_RUSTFS_ADDR: LazyLock<RwLock<String>> = LazyLock::new(|| RwLoc
pub static GLOBAL_CONN_MAP: LazyLock<RwLock<HashMap<String, Channel>>> = LazyLock::new(|| RwLock::new(HashMap::new()));
pub static GLOBAL_ROOT_CERT: LazyLock<RwLock<Option<Vec<u8>>>> = LazyLock::new(|| RwLock::new(None));
pub static GLOBAL_MTLS_IDENTITY: LazyLock<RwLock<Option<MtlsIdentityPem>>> = LazyLock::new(|| RwLock::new(None));
pub static GLOBAL_OUTBOUND_TLS_GENERATION: LazyLock<AtomicU64> = LazyLock::new(|| AtomicU64::new(0));
/// Global initialization time of the RustFS node.
pub static GLOBAL_INIT_TIME: LazyLock<RwLock<Option<DateTime<Utc>>>> = LazyLock::new(|| RwLock::new(None));
@@ -88,6 +90,16 @@ pub async fn set_global_mtls_identity(identity: Option<MtlsIdentityPem>) {
*GLOBAL_MTLS_IDENTITY.write().await = identity;
}
/// Set the global outbound TLS generation.
pub fn set_global_outbound_tls_generation(generation: u64) {
GLOBAL_OUTBOUND_TLS_GENERATION.store(generation, Ordering::Relaxed);
}
/// Get the global outbound TLS generation.
pub fn get_global_outbound_tls_generation() -> u64 {
GLOBAL_OUTBOUND_TLS_GENERATION.load(Ordering::Relaxed)
}
/// Evict a stale/dead connection from the global connection cache.
/// This is critical for cluster recovery when a node dies unexpectedly (e.g., power-off).
/// By removing the cached connection, subsequent requests will establish a fresh connection.
crates/ecstore/Cargo.toml
+2
View File
@@ -38,6 +38,7 @@ rustfs-filemeta.workspace = true
rustfs-utils = { workspace = true, features = ["full"] }
rustfs-rio.workspace = true
rustfs-signer.workspace = true
rustfs-tls-runtime.workspace = true
rustfs-checksums.workspace = true
rustfs-config = { workspace = true, features = ["constants", "notify", "audit"] }
rustfs-credentials = { workspace = true }
@@ -91,6 +92,7 @@ hyper.workspace = true
hyper-util.workspace = true
hyper-rustls.workspace = true
rustls.workspace = true
rustls-pki-types.workspace = true
tokio = { workspace = true, features = ["io-util", "sync", "signal","io-uring"] }
tonic.workspace = true
xxhash-rust = { workspace = true, features = ["xxh64", "xxh3"] }
crates/ecstore/src/client/transition_api.rs
+38 -46
View File
@@ -51,6 +51,7 @@ use md5::Md5;
use rand::{Rng, RngExt};
use rustfs_config::MAX_S3_CLIENT_RESPONSE_SIZE;
use rustfs_rio::HashReader;
use rustfs_tls_runtime::{load_global_outbound_tls_state, record_tls_generation};
use rustfs_utils::HashAlgorithm;
use rustfs_utils::{
net::get_endpoint_url,
@@ -58,6 +59,8 @@ use rustfs_utils::{
DEFAULT_RETRY_CAP, DEFAULT_RETRY_UNIT, MAX_JITTER, MAX_RETRY, RetryTimer, is_http_status_retryable, is_s3code_retryable,
},
};
use rustls_pki_types::PrivateKeyDer;
use rustls_pki_types::pem::PemObject;
use s3s::S3ErrorCode;
use s3s::dto::Owner;
use s3s::dto::ReplicationStatus;
@@ -152,24 +155,11 @@ pub enum BucketLookupType {
BucketLookupPath,
}
fn load_root_store_from_tls_path() -> Option<rustls::RootCertStore> {
// Load the root certificate bundle from the path specified by the
// RUSTFS_TLS_PATH environment variable.
let tp = rustfs_utils::get_env_str(rustfs_config::ENV_RUSTFS_TLS_PATH, rustfs_config::DEFAULT_RUSTFS_TLS_PATH);
// If no TLS path is configured, do not fall back to a CA bundle in the current directory.
if tp.is_empty() {
return None;
}
let ca = std::path::Path::new(&tp).join(rustfs_config::RUSTFS_CA_CERT);
if !ca.exists() {
return None;
}
let der_list = rustfs_utils::load_cert_bundle_der_bytes(ca.to_str().unwrap_or_default()).ok()?;
fn build_root_store_from_der_list(der_list: Vec<Vec<u8>>) -> Option<rustls::RootCertStore> {
let mut store = rustls::RootCertStore::empty();
for der in der_list {
if let Err(e) = store.add(der.into()) {
warn!("Warning: failed to add certificate from '{}' to root store: {e}", ca.display());
warn!("Warning: failed to add certificate to root store: {e}");
}
}
Some(store)
@@ -199,18 +189,38 @@ where
})
}
fn build_tls_config() -> Result<rustls::ClientConfig, std::io::Error> {
with_rustls_init_guard(|| {
let config = if let Some(store) = load_root_store_from_tls_path() {
rustls::ClientConfig::builder()
.with_root_certificates(store)
.with_no_client_auth()
} else {
rustls::ClientConfig::builder().with_native_roots()?.with_no_client_auth()
};
async fn build_tls_config() -> Result<rustls::ClientConfig, std::io::Error> {
with_rustls_init_guard(|| Ok(()))?;
Ok(config)
})
let outbound_tls = load_global_outbound_tls_state().await;
record_tls_generation("ecstore_transition_client", outbound_tls.generation.0);
let builder = if let Some(root_ca_pem) = outbound_tls.root_ca_pem.as_ref() {
let mut reader = std::io::BufReader::new(root_ca_pem.as_slice());
let certs_der = rustls_pki_types::CertificateDer::pem_reader_iter(&mut reader)
.collect::<Result<Vec<_>, _>>()
.map_err(|e| std::io::Error::other(format!("failed to parse published root CA PEM: {e}")))?;
let root_store = build_root_store_from_der_list(certs_der.into_iter().map(|cert| cert.to_vec()).collect::<Vec<_>>())
.ok_or_else(|| std::io::Error::other("published outbound root CA material could not build root store"))?;
rustls::ClientConfig::builder().with_root_certificates(root_store)
} else {
rustls::ClientConfig::builder().with_native_roots()?
};
let config = if let Some(identity) = outbound_tls.mtls_identity.as_ref() {
let certs = rustls_pki_types::CertificateDer::pem_reader_iter(&mut std::io::BufReader::new(identity.cert_pem.as_slice()))
.collect::<Result<Vec<_>, _>>()
.map_err(|e| std::io::Error::other(format!("failed to parse published client cert PEM: {e}")))?;
let key = PrivateKeyDer::from_pem_reader(&mut std::io::BufReader::new(identity.key_pem.as_slice()))
.map_err(|e| std::io::Error::other(format!("failed to parse published client key PEM: {e}")))?;
builder
.with_client_auth_cert(certs, key)
.map_err(|e| std::io::Error::other(format!("failed to build client mTLS identity: {e}")))?
} else {
builder.with_no_client_auth()
};
Ok(config)
}
impl TransitionClient {
@@ -234,7 +244,7 @@ impl TransitionClient {
let endpoint_url = get_endpoint_url(endpoint, opts.secure)?;
let tls = build_tls_config()?;
let tls = build_tls_config().await?;
let https = hyper_rustls::HttpsConnectorBuilder::new()
.with_tls_config(tls)
@@ -1374,9 +1384,7 @@ pub struct CreateBucketConfiguration {
#[cfg(test)]
mod tests {
use super::{
build_tls_config, load_root_store_from_tls_path, signer_error_to_io_error, validate_header_values, with_rustls_init_guard,
};
use super::{build_tls_config, signer_error_to_io_error, validate_header_values, with_rustls_init_guard};
use http::{HeaderMap, HeaderValue};
#[test]
@@ -1395,22 +1403,6 @@ mod tests {
assert!(outcome.is_ok(), "TLS config creation should not panic");
}
/// When RUSTFS_TLS_PATH is not set, `load_root_store_from_tls_path` must return `None`
/// (i.e. it must not silently look for a CA bundle in the current working directory).
#[test]
fn tls_path_unset_returns_none() {
let result = temp_env::with_var_unset(rustfs_config::ENV_RUSTFS_TLS_PATH, || load_root_store_from_tls_path());
assert!(result.is_none(), "expected None when RUSTFS_TLS_PATH is unset, but got a root store");
}
/// When RUSTFS_TLS_PATH is set to an empty string, `load_root_store_from_tls_path` must
/// return `None` to avoid accidentally trusting a CA bundle in the current directory.
#[test]
fn tls_path_empty_returns_none() {
let result = temp_env::with_var(rustfs_config::ENV_RUSTFS_TLS_PATH, Some(""), || load_root_store_from_tls_path());
assert!(result.is_none(), "expected None when RUSTFS_TLS_PATH is empty, but got a root store");
}
/// Installing the rustls crypto provider when one is already set must not panic or return
/// an error that surfaces to callers (the race-safe `get_default` check guards the install).
#[test]
crates/protocols/Cargo.toml
+3 -3
View File
@@ -27,7 +27,7 @@ categories = ["network-programming", "filesystem"]
[features]
default = []
ftps = ["dep:libunftp", "dep:unftp-core", "dep:rustls"]
ftps = ["dep:libunftp", "dep:unftp-core", "dep:rustls", "dep:rustfs-tls-runtime"]
swift = [
"dep:rustfs-keystone",
"dep:rustfs-ecstore",
@@ -53,7 +53,7 @@ swift = [
"dep:base64",
"dep:async-compression",
]
webdav = ["dep:dav-server", "dep:hyper", "dep:hyper-util", "dep:http-body-util", "dep:tokio-rustls", "dep:base64", "dep:rustls", "dep:percent-encoding"]
webdav = ["dep:dav-server", "dep:hyper", "dep:hyper-util", "dep:http-body-util", "dep:tokio-rustls", "dep:base64", "dep:rustls", "dep:percent-encoding", "dep:rustfs-tls-runtime"]
sftp = ["dep:russh", "dep:russh-sftp", "dep:uuid", "dep:subtle", "dep:tokio-util", "dep:socket2"]
[dependencies]
@@ -63,6 +63,7 @@ rustfs-credentials = { workspace = true }
rustfs-policy = { workspace = true }
rustfs-utils = { workspace = true }
rustfs-config = { workspace = true }
rustfs-tls-runtime = { workspace = true, optional = true }
# Async dependencies
tokio = { workspace = true, features = ["fs", "io-util", "sync", "time","io-uring"] }
@@ -128,7 +129,6 @@ socket2 = { workspace = true, optional = true }
[dev-dependencies]
tempfile = { workspace = true }
proptest = "1"
rcgen = { workspace = true }
tracing-subscriber = { workspace = true }
[package.metadata.docs.rs]
crates/protocols/src/ftps/server.rs
+18 -3
View File
@@ -17,12 +17,14 @@ use super::driver::FtpsDriver;
use crate::common::client::s3::StorageBackend;
use crate::common::session::{Protocol, ProtocolPrincipal, SessionContext};
use crate::constants::{network::DEFAULT_SOURCE_IP, paths::ROOT_PATH};
use crate::tls_hot_reload::{ReloadableCertResolver, spawn_cert_reload_loop};
use libunftp::options::FtpsRequired;
use rustfs_config::{DEFAULT_TLS_RELOAD_ENABLE, DEFAULT_TLS_RELOAD_INTERVAL, ENV_TLS_RELOAD_ENABLE, ENV_TLS_RELOAD_INTERVAL};
use rustfs_tls_runtime::{ReloadableServerCertResolver, TlsReloadOptions, spawn_server_cert_reload_loop};
use std::fmt::{Debug, Display, Formatter};
use std::net::IpAddr;
use std::path::Path;
use std::sync::Arc;
use std::time::Duration;
use tokio::sync::broadcast;
use tokio::sync::watch;
use tracing::{debug, error, info, warn};
@@ -68,6 +70,14 @@ impl<S> FtpsServer<S>
where
S: StorageBackend + Clone + Send + Sync + 'static + Debug,
{
fn tls_reload_options() -> TlsReloadOptions {
TlsReloadOptions {
enabled: rustfs_utils::get_env_bool(ENV_TLS_RELOAD_ENABLE, DEFAULT_TLS_RELOAD_ENABLE),
interval: Duration::from_secs(rustfs_utils::get_env_u64(ENV_TLS_RELOAD_INTERVAL, DEFAULT_TLS_RELOAD_INTERVAL).max(5)),
..TlsReloadOptions::default()
}
}
/// Create a new FTPS server
pub async fn new(config: FtpsConfig, storage: S) -> Result<Self, FtpsInitError> {
config.validate().await?;
@@ -114,9 +124,14 @@ where
if let Some(cert_dir) = &self.config.cert_dir {
debug!("Enabling FTPS with multi-certificate support from directory: {}", cert_dir);
let resolver = ReloadableCertResolver::load_from_directory(cert_dir)
let resolver = ReloadableServerCertResolver::load_from_directory(cert_dir)
.map_err(|e| FtpsInitError::InvalidConfig(format!("Failed to create certificate resolver: {}", e)))?;
let _reload_task = spawn_cert_reload_loop("ftps", cert_dir.clone(), resolver.clone(), reload_shutdown_rx.clone());
let _reload_task = spawn_server_cert_reload_loop(
"ftps",
resolver.clone(),
Self::tls_reload_options(),
reload_shutdown_rx.clone(),
);
// Build ServerConfig with SNI support
let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();
crates/protocols/src/lib.rs
-3
View File
@@ -17,9 +17,6 @@
pub mod common;
pub mod constants;
#[cfg(any(feature = "ftps", feature = "webdav"))]
mod tls_hot_reload;
#[cfg(feature = "ftps")]
pub mod ftps;
crates/protocols/src/sftp/errors.rs
+7
View File
@@ -18,6 +18,7 @@
use super::constants::{http_error_codes, s3_error_codes};
use russh_sftp::protocol::{Status, StatusCode};
use russh_sftp::server::StatusReply;
use s3s::{S3Error, S3ErrorCode};
use std::{any::Any, fmt::Display};
@@ -31,6 +32,12 @@ impl From<SftpError> for StatusCode {
}
}
impl From<SftpError> for StatusReply {
fn from(err: SftpError) -> Self {
StatusReply::new(err.0)
}
}
impl SftpError {
pub(super) fn code(code: StatusCode) -> Self {
Self(code)
crates/protocols/src/webdav/server.rs
+18 -4
View File
@@ -16,7 +16,6 @@ use super::config::{WebDavConfig, WebDavInitError};
use super::driver::WebDavDriver;
use crate::common::client::s3::StorageBackend;
use crate::common::session::{Protocol, ProtocolPrincipal, SessionContext};
use crate::tls_hot_reload::{ReloadableCertResolver, spawn_cert_reload_loop};
use bytes::Bytes;
use dav_server::DavHandler;
use dav_server::fakels::FakeLs;
@@ -25,10 +24,13 @@ use hyper::server::conn::http1;
use hyper::service::service_fn;
use hyper::{Request, Response, StatusCode};
use hyper_util::rt::TokioIo;
use rustfs_config::{DEFAULT_TLS_RELOAD_ENABLE, DEFAULT_TLS_RELOAD_INTERVAL, ENV_TLS_RELOAD_ENABLE, ENV_TLS_RELOAD_INTERVAL};
use rustfs_tls_runtime::{ReloadableServerCertResolver, TlsReloadOptions, spawn_server_cert_reload_loop};
use rustls::ServerConfig;
use std::convert::Infallible;
use std::net::IpAddr;
use std::sync::Arc;
use std::time::Duration;
use tokio::net::TcpListener;
use tokio::sync::{broadcast, watch};
use tokio_rustls::TlsAcceptor;
@@ -49,6 +51,14 @@ impl<S> WebDavServer<S>
where
S: StorageBackend + Clone + Send + Sync + 'static + std::fmt::Debug,
{
fn tls_reload_options() -> TlsReloadOptions {
TlsReloadOptions {
enabled: rustfs_utils::get_env_bool(ENV_TLS_RELOAD_ENABLE, DEFAULT_TLS_RELOAD_ENABLE),
interval: Duration::from_secs(rustfs_utils::get_env_u64(ENV_TLS_RELOAD_INTERVAL, DEFAULT_TLS_RELOAD_INTERVAL).max(5)),
..TlsReloadOptions::default()
}
}
/// Create a new WebDAV server
pub async fn new(config: WebDavConfig, storage: S) -> Result<Self, WebDavInitError> {
config.validate().await?;
@@ -68,10 +78,14 @@ where
if let Some(cert_dir) = &self.config.cert_dir {
debug!("Enabling WebDAV TLS with certificates from: {}", cert_dir);
let resolver = ReloadableCertResolver::load_from_directory(cert_dir)
let resolver = ReloadableServerCertResolver::load_from_directory(cert_dir)
.map_err(|e| WebDavInitError::Tls(format!("Failed to create certificate resolver: {}", e)))?;
let _reload_task =
spawn_cert_reload_loop("webdav", cert_dir.clone(), resolver.clone(), reload_shutdown_rx.clone());
let _reload_task = spawn_server_cert_reload_loop(
"webdav",
resolver.clone(),
Self::tls_reload_options(),
reload_shutdown_rx.clone(),
);
let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();
crates/protos/Cargo.toml
+3 -2
View File
@@ -36,14 +36,15 @@ path = "src/main.rs"
rustfs-common.workspace = true
rustfs-io-metrics.workspace = true
rustfs-config.workspace = true
rustfs-tls-runtime.workspace = true
rustfs-utils.workspace = true
flatbuffers = { workspace = true }
prost = { workspace = true }
tonic = { workspace = true, features = ["transport"] }
tonic = { workspace = true, features = ["transport", "tls-native-roots", "tls-aws-lc"] }
tonic-prost = { workspace = true }
tonic-prost-build = { workspace = true }
tokio = { workspace = true, features = ["sync"] }
tracing = { workspace = true }
[lib]
test = false
doctest = false
crates/protos/src/lib.rs
+62 -13
View File
@@ -18,12 +18,16 @@
mod generated;
use proto_gen::node_service::node_service_client::NodeServiceClient;
use rustfs_common::{GLOBAL_CONN_MAP, GLOBAL_MTLS_IDENTITY, GLOBAL_ROOT_CERT, evict_connection};
use rustfs_common::{GLOBAL_CONN_MAP, evict_connection};
use rustfs_io_metrics::internode_metrics::global_internode_metrics;
use rustfs_tls_runtime::{load_global_outbound_tls_state, record_tls_consumer_stale_generation};
use std::{
collections::HashMap,
error::Error,
sync::LazyLock,
time::{Duration, Instant},
};
use tokio::sync::Mutex;
use tonic::{
Request, Status,
service::interceptor::InterceptedService,
@@ -46,6 +50,21 @@ pub const DEFAULT_GRPC_SERVER_MESSAGE_LEN: usize = 100 * 1024 * 1024;
/// It is used to identify HTTPS URLs.
/// Default value: https://
const RUSTFS_HTTPS_PREFIX: &str = "https://";
const TLS_GENERATION_CACHE_MAX_SIZE: usize = 512;
static TLS_GENERATION_CACHE: LazyLock<Mutex<HashMap<String, u64>>> = LazyLock::new(|| Mutex::new(HashMap::new()));
fn enforce_tls_generation_cache_bound(generation_cache: &mut HashMap<String, u64>, generation: u64, addr: &str) {
if generation_cache.len() < TLS_GENERATION_CACHE_MAX_SIZE || generation_cache.contains_key(addr) {
return;
}
generation_cache.retain(|_, g| *g == generation);
if generation_cache.len() >= TLS_GENERATION_CACHE_MAX_SIZE
&& let Some(victim) = generation_cache.keys().next().cloned()
{
generation_cache.remove(&victim);
}
}
fn internode_connect_timeout() -> Duration {
Duration::from_secs(rustfs_utils::get_env_u64(
@@ -114,14 +133,19 @@ pub async fn create_new_channel(addr: &str) -> Result<Channel, Box<dyn Error>> {
// Overall timeout for any RPC - fail fast on unresponsive peers
.timeout(rpc_timeout);
let root_cert = GLOBAL_ROOT_CERT.read().await;
if addr.starts_with(RUSTFS_HTTPS_PREFIX) {
if root_cert.is_none() {
debug!("No custom root certificate configured; using system roots for TLS: {}", addr);
// If no custom root cert is configured, try to use system roots.
connector = connector.tls_config(ClientTlsConfig::new())?;
let outbound_tls = load_global_outbound_tls_state().await;
let generation = outbound_tls.generation.0;
let mut stale_generation = false;
{
let generation_cache = TLS_GENERATION_CACHE.lock().await;
if let Some(cached_generation) = generation_cache.get(addr)
&& *cached_generation != generation
{
stale_generation = true;
}
if let Some(cert_pem) = root_cert.as_ref() {
}
if addr.starts_with(RUSTFS_HTTPS_PREFIX) {
if let Some(cert_pem) = outbound_tls.root_ca_pem.as_ref() {
let ca = Certificate::from_pem(cert_pem);
// Derive the hostname from the HTTPS URL for TLS hostname verification.
let domain = addr
@@ -134,8 +158,7 @@ pub async fn create_new_channel(addr: &str) -> Result<Channel, Box<dyn Error>> {
.unwrap_or("");
let tls = if !domain.is_empty() {
let mut cfg = ClientTlsConfig::new().ca_certificate(ca).domain_name(domain);
let mtls_identity = GLOBAL_MTLS_IDENTITY.read().await;
if let Some(id) = mtls_identity.as_ref() {
if let Some(id) = outbound_tls.mtls_identity.as_ref() {
let identity = tonic::transport::Identity::from_pem(id.cert_pem.clone(), id.key_pem.clone());
cfg = cfg.identity(identity);
}
@@ -147,9 +170,10 @@ pub async fn create_new_channel(addr: &str) -> Result<Channel, Box<dyn Error>> {
connector = connector.tls_config(tls)?;
debug!("Configured TLS with custom root certificate for: {}", addr);
} else {
return Err(std::io::Error::other(
"HTTPS requested but no trusted roots are configured. Provide tls/ca.crt (or enable system roots via RUSTFS_TRUST_SYSTEM_CA=true)."
).into());
// No custom root CA published — fall back to system roots.
// This is the expected path when no TLS path is configured.
debug!("No custom root certificate configured; using system roots for TLS: {}", addr);
connector = connector.tls_config(ClientTlsConfig::new())?;
}
}
@@ -168,6 +192,14 @@ pub async fn create_new_channel(addr: &str) -> Result<Channel, Box<dyn Error>> {
{
GLOBAL_CONN_MAP.write().await.insert(addr.to_string(), channel.clone());
}
{
let mut generation_cache = TLS_GENERATION_CACHE.lock().await;
enforce_tls_generation_cache_bound(&mut generation_cache, generation, addr);
generation_cache.insert(addr.to_string(), generation);
}
if stale_generation {
record_tls_consumer_stale_generation("protos_grpc_channel");
}
debug!("Successfully created and cached gRPC channel to: {}", addr);
Ok(channel)
@@ -178,4 +210,21 @@ pub async fn create_new_channel(addr: &str) -> Result<Channel, Box<dyn Error>> {
pub async fn evict_failed_connection(addr: &str) {
warn!("Evicting failed gRPC connection: {}", addr);
evict_connection(addr).await;
TLS_GENERATION_CACHE.lock().await.remove(addr);
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn enforce_tls_generation_cache_bound_evicts_when_retained_entries_still_full() {
let mut cache = HashMap::new();
for i in 0..TLS_GENERATION_CACHE_MAX_SIZE {
cache.insert(format!("node-{i}"), 42);
}
enforce_tls_generation_cache_bound(&mut cache, 42, "new-node");
assert_eq!(cache.len(), TLS_GENERATION_CACHE_MAX_SIZE - 1);
}
}
crates/rio/Cargo.toml
+3 -1
View File
@@ -38,12 +38,14 @@ pin-project-lite.workspace = true
serde = { workspace = true }
bytes.workspace = true
reqwest.workspace = true
rustls-pki-types.workspace = true
tokio-util.workspace = true
faster-hex.workspace = true
futures.workspace = true
rustfs-config = { workspace = true, features = ["constants"] }
rustfs-io-metrics.workspace = true
rustfs-utils = { workspace = true, features = ["io", "hash", "compress", "tls"] }
rustfs-tls-runtime.workspace = true
rustfs-utils = { workspace = true, features = ["io", "hash", "compress"] }
serde_json.workspace = true
md-5 = { workspace = true }
tracing.workspace = true
crates/rio/src/http_reader.rs
+106 -63
View File
@@ -22,7 +22,12 @@ use rustfs_io_metrics::internode_metrics::{
INTERNODE_OPERATION_PUT_FILE_STREAM, INTERNODE_OPERATION_READ_FILE_STREAM, INTERNODE_OPERATION_WALK_DIR,
INTERNODE_TRANSPORT_BACKEND_TCP_HTTP, global_internode_metrics,
};
use rustfs_tls_runtime::{
load_cert_bundle_der_bytes, load_global_outbound_tls_generation, load_global_outbound_tls_state,
record_tls_consumer_stale_generation,
};
use rustfs_utils::get_env_opt_str;
use rustls_pki_types::pem::PemObject;
use std::io::IoSlice;
use std::io::{self, Error};
use std::net::IpAddr;
@@ -32,76 +37,36 @@ use std::sync::LazyLock;
use std::task::{Context, Poll};
use std::time::Duration;
use tokio::io::{AsyncRead, AsyncWrite, ReadBuf};
use tokio::sync::mpsc;
use tokio::sync::{Mutex, mpsc};
use tokio::time::{self, Sleep};
use tokio_util::io::StreamReader;
use tokio_util::sync::PollSender;
use tracing::error;
use tracing::{error, warn};
const READ_FILE_STREAM_PATH: &str = "/rustfs/rpc/read_file_stream";
const PUT_FILE_STREAM_PATH: &str = "/rustfs/rpc/put_file_stream";
const WALK_DIR_PATH: &str = "/rustfs/rpc/walk_dir";
/// Get the TLS path from the RUSTFS_TLS_PATH environment variable.
/// If the variable is not set, return None.
fn tls_path() -> Option<&'static std::path::PathBuf> {
static TLS_PATH: LazyLock<Option<std::path::PathBuf>> =
LazyLock::new(|| get_env_opt_str("RUSTFS_TLS_PATH").and_then(|s| if s.is_empty() { None } else { Some(s.into()) }));
TLS_PATH.as_ref()
}
/// Load CA root certificates from the RUSTFS_TLS_PATH directory.
/// The CA certificates should be in PEM format and stored in the file
/// specified by the RUSTFS_CA_CERT constant.
/// If the file does not exist or cannot be read, return the builder unchanged.
fn load_ca_roots_from_tls_path(builder: reqwest::ClientBuilder) -> reqwest::ClientBuilder {
let Some(tp) = tls_path() else {
return builder;
};
let ca_path = tp.join(rustfs_config::RUSTFS_CA_CERT);
if !ca_path.exists() {
return builder;
}
let Ok(certs_der) = rustfs_utils::load_cert_bundle_der_bytes(ca_path.to_str().unwrap_or_default()) else {
return builder;
};
fn add_root_certificates_from_der(builder: reqwest::ClientBuilder, certs_der: &[Vec<u8>]) -> reqwest::ClientBuilder {
let mut b = builder;
for der in certs_der {
if let Ok(cert) = Certificate::from_der(&der) {
if let Ok(cert) = Certificate::from_der(der) {
b = b.add_root_certificate(cert);
}
}
b
}
/// Load optional mTLS identity from the RUSTFS_TLS_PATH directory.
/// The client certificate and private key should be in PEM format and stored in the files
/// specified by RUSTFS_CLIENT_CERT_FILENAME and RUSTFS_CLIENT_KEY_FILENAME constants.
/// If the files do not exist or cannot be read, return None.
fn load_optional_mtls_identity_from_tls_path() -> Option<Identity> {
let tp = tls_path()?;
let cert = std::fs::read(tp.join(rustfs_config::RUSTFS_CLIENT_CERT_FILENAME)).ok()?;
let key = std::fs::read(tp.join(rustfs_config::RUSTFS_CLIENT_KEY_FILENAME)).ok()?;
let mut pem = Vec::with_capacity(cert.len() + key.len() + 1);
pem.extend_from_slice(&cert);
if !pem.ends_with(b"\n") {
pem.push(b'\n');
}
pem.extend_from_slice(&key);
match Identity::from_pem(&pem) {
Ok(id) => Some(id),
Err(e) => {
error!("Failed to load mTLS identity from PEM: {e}");
None
}
}
#[derive(Clone)]
struct CachedClients {
generation: u64,
client: Client,
local_client: Client,
}
fn build_http_client(disable_proxy: bool) -> Client {
static CLIENT_CACHE: LazyLock<Mutex<Option<CachedClients>>> = LazyLock::new(|| Mutex::new(None));
async fn build_http_client(disable_proxy: bool, outbound_tls: &rustfs_tls_runtime::GlobalPublishedOutboundTlsState) -> Client {
let mut builder = Client::builder()
.connect_timeout(std::time::Duration::from_secs(5))
.tcp_keepalive(std::time::Duration::from_secs(10))
@@ -113,9 +78,51 @@ fn build_http_client(disable_proxy: bool) -> Client {
builder = builder.no_proxy();
}
builder = load_ca_roots_from_tls_path(builder);
if let Some(id) = load_optional_mtls_identity_from_tls_path() {
builder = builder.identity(id);
if let Some(root_ca_pem) = outbound_tls.root_ca_pem.as_ref() {
let mut reader = std::io::BufReader::new(root_ca_pem.as_slice());
match rustls_pki_types::CertificateDer::pem_reader_iter(&mut reader).collect::<Result<Vec<_>, _>>() {
Ok(certs_der) => {
let certs_der = certs_der.into_iter().map(|cert| cert.to_vec()).collect::<Vec<_>>();
builder = add_root_certificates_from_der(builder, &certs_der);
}
Err(err) => {
warn!("Failed to parse published outbound root CA PEM; falling back to default trust roots: {err}");
}
}
} else if let Some(tp) = get_env_opt_str(rustfs_config::ENV_RUSTFS_TLS_PATH).and_then(|s| {
if s.is_empty() {
None
} else {
Some(std::path::PathBuf::from(s))
}
}) {
let ca_path = tp.join(rustfs_config::RUSTFS_CA_CERT);
if ca_path.exists()
&& let Some(ca_path_str) = ca_path.to_str()
{
match load_cert_bundle_der_bytes(ca_path_str) {
Ok(certs_der) => {
builder = add_root_certificates_from_der(builder, &certs_der);
}
Err(err) => {
warn!("Failed to parse fallback root CA bundle '{}': {}", ca_path.display(), err);
}
}
}
}
if let Some(identity) = outbound_tls.mtls_identity.as_ref() {
let mut pem = Vec::with_capacity(identity.cert_pem.len() + identity.key_pem.len() + 1);
pem.extend_from_slice(&identity.cert_pem);
if !pem.ends_with(b"\n") {
pem.push(b'\n');
}
pem.extend_from_slice(&identity.key_pem);
match Identity::from_pem(&pem) {
Ok(id) => builder = builder.identity(id),
Err(e) => error!("Failed to load mTLS identity from PEM: {e}"),
}
}
builder.build().expect("Failed to create global HTTP client")
@@ -133,17 +140,53 @@ fn should_bypass_proxy_for_url(url: &str) -> bool {
host.eq_ignore_ascii_case("localhost") || host.parse::<IpAddr>().is_ok_and(|addr| addr.is_loopback())
}
fn get_http_client(url: &str) -> Client {
async fn get_http_client(url: &str) -> Client {
// Reuse HTTP connection pools while keeping loopback traffic away from
// system proxies so local RPC/tests do not leak to proxy listeners.
static CLIENT: LazyLock<Client> = LazyLock::new(|| build_http_client(false));
static LOCAL_CLIENT: LazyLock<Client> = LazyLock::new(|| build_http_client(true));
let disable_proxy = should_bypass_proxy_for_url(url);
if should_bypass_proxy_for_url(url) {
return LOCAL_CLIENT.clone();
// Fast path: check generation first (cheap atomic read) to avoid cloning
// the full PEM + identity bytes when the TLS state hasn't changed.
let generation = load_global_outbound_tls_generation().0;
let guard = CLIENT_CACHE.lock().await;
if let Some(cached) = guard.as_ref() {
if cached.generation == generation {
return if disable_proxy {
cached.local_client.clone()
} else {
cached.client.clone()
};
}
record_tls_consumer_stale_generation("rio_http_reader");
}
drop(guard);
CLIENT.clone()
// Cache miss or stale generation — load full outbound TLS state.
let outbound_tls = load_global_outbound_tls_state().await;
let client = build_http_client(false, &outbound_tls).await;
let local_client = build_http_client(true, &outbound_tls).await;
let cached = CachedClients {
generation,
client,
local_client,
};
let return_client = if disable_proxy {
cached.local_client.clone()
} else {
cached.client.clone()
};
let mut guard = CLIENT_CACHE.lock().await;
// Guard against races: only overwrite the cache if it is empty or
// contains an older generation, so a slower task cannot regress the
// TLS state after a faster task already cached a newer generation.
if guard.as_ref().is_none_or(|c| c.generation <= generation) {
*guard = Some(cached);
}
return_client
}
pin_project! {
@@ -197,7 +240,7 @@ impl HttpReader {
) -> io::Result<Self> {
let track_internode_metrics = is_internode_rpc_url(&url);
let internode_operation = internode_rpc_operation(&url);
let client = get_http_client(&url);
let client = get_http_client(&url).await;
let mut request: RequestBuilder = client.request(method.clone(), url.clone()).headers(headers.clone());
if let Some(body) = body {
request = request.body(body);
@@ -379,7 +422,7 @@ impl HttpWriter {
// "[HttpWriter::spawn] sending HTTP request: url={url_clone}, method={method_clone:?}, headers={headers_clone:?}"
// );
let client = get_http_client(&url_clone);
let client = get_http_client(&url_clone).await;
let request = client
.request(method_clone, url_clone.clone())
.headers(headers_clone.clone())
crates/targets/Cargo.toml
+7 -2
View File
@@ -14,20 +14,23 @@ documentation = "https://docs.rs/rustfs-targets/latest/rustfs_targets/"
[dependencies]
rustfs-config = { workspace = true, features = ["notify", "constants", "audit"] }
rustfs-ecstore = { workspace = true }
rustfs-utils = { workspace = true, features = ["notify", "tls"] }
rustfs-tls-runtime = { workspace = true }
rustfs-s3-types = { workspace = true }
async-trait = { workspace = true }
async-nats = { workspace = true }
deadpool-postgres = { workspace = true }
hyper = { workspace = true }
hyper-rustls = { workspace = true }
lapin = { workspace = true }
libc = { workspace = true }
pulsar = { workspace = true }
regex = { workspace = true }
reqwest = { workspace = true }
rumqttc = { workspace = true }
redis = { workspace = true }
rustls = { workspace = true }
rustls-native-certs = { workspace = true }
rustls-pki-types = { workspace = true }
s3s = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
snap = { workspace = true }
@@ -45,6 +48,8 @@ mysql_async = { workspace = true }
chrono = { workspace = true }
parking_lot = { workspace = true }
hashbrown = { workspace = true }
arc-swap = { workspace = true }
metrics = { workspace = true }
[dev-dependencies]
criterion = { workspace = true }
crates/targets/src/check.rs
+2 -2
View File
@@ -64,8 +64,8 @@ pub async fn check_mqtt_broker_available_with_tls(
use crate::target::mqtt::build_mqtt_options;
use rumqttc::{AsyncClient, QoS};
let url = rustfs_utils::parse_url(broker_url)
.map_err(|e| crate::TargetError::Configuration(format!("Broker URL parsing failed: {e}")))?;
let url =
crate::parse_url(broker_url).map_err(|e| crate::TargetError::Configuration(format!("Broker URL parsing failed: {e}")))?;
let url = url.url();
// build_mqtt_options returns TargetError directly; Configuration variants propagate as-is.
crates/targets/src/lib.rs
+2
View File
@@ -20,6 +20,7 @@ pub mod control_plane;
pub mod domain;
pub mod error;
pub mod manifest;
mod net;
pub mod plugin;
pub mod runtime;
pub mod store;
@@ -48,6 +49,7 @@ pub use manifest::{
TargetPluginExternalRuntimeContract, TargetPluginManifest, TargetPluginMarketplaceManifest, TargetPluginPackaging,
TargetPluginRuntimeTransport, builtin_target_marketplace_manifest, installable_target_marketplace_manifest,
};
pub use net::*;
pub use plugin::{
BuiltinTargetAdminDescriptor, BuiltinTargetDescriptor, TargetAdminMetadata, TargetPluginDescriptor, TargetPluginRegistry,
TargetRequestValidator, boxed_target,
crates/utils/src/notify/net.rs → crates/targets/src/net.rs
+179 -201
View File
@@ -1,18 +1,21 @@
// Copyright 2024 RustFS Team
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use hashbrown::HashMap;
use hyper::HeaderMap;
use regex::Regex;
use s3s::{S3Request, S3Response};
use serde::{Deserialize, Serialize};
use std::net::IpAddr;
use std::path::Path;
@@ -20,7 +23,6 @@ use std::sync::LazyLock;
use thiserror::Error;
use url::Url;
// Lazy static for the host label regex.
static HOST_LABEL_REGEX: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"^[a-zA-Z0-9]([a-zA-Z0-9-]*[a-zA-Z0-9])?$").unwrap());
/// NetError represents errors that can occur in network operations.
@@ -41,21 +43,17 @@ pub enum NetError {
}
/// Host represents a network host with IP/name and port.
/// Similar to Go's net.Host structure.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct Host {
pub name: String,
pub port: Option<u16>, // Using Option<u16> to represent if port is set, similar to IsPortSet.
pub port: Option<u16>,
}
// Implementation of Host methods.
impl Host {
// is_empty returns true if the host name is empty.
pub fn is_empty(&self) -> bool {
self.name.is_empty()
}
// equal checks if two hosts are equal by comparing their string representations.
pub fn equal(&self, other: &Host) -> bool {
self.to_string() == other.to_string()
}
@@ -70,13 +68,122 @@ impl std::fmt::Display for Host {
}
}
// parse_host parses a string into a Host, with validation similar to Go's ParseHost.
/// Extract request parameters from S3Request, mainly header information.
pub fn extract_req_params<T>(req: &S3Request<T>) -> HashMap<String, String> {
extract_params_header(&req.headers)
}
/// Extract request parameters from hyper::HeaderMap, mainly header information.
/// This function is useful when you have a raw HTTP request and need to extract parameters.
#[deprecated(since = "0.1.0", note = "Use extract_params_header instead")]
pub fn extract_req_params_header(head: &HeaderMap) -> HashMap<String, String> {
extract_params_header(head)
}
/// Extract parameters from hyper::HeaderMap, mainly header information.
pub fn extract_params_header(head: &HeaderMap) -> HashMap<String, String> {
let mut params = HashMap::new();
for (key, value) in head.iter() {
if let Ok(val_str) = value.to_str() {
params.insert(key.as_str().to_string(), val_str.to_string());
}
}
params
}
/// Extract response elements from S3Response, mainly header information.
pub fn extract_resp_elements<T>(resp: &S3Response<T>) -> HashMap<String, String> {
extract_params_header(&resp.headers)
}
/// Get host from header information.
pub fn get_request_host(headers: &HeaderMap) -> String {
headers
.get("host")
.and_then(|v| v.to_str().ok())
.unwrap_or_default()
.to_string()
}
/// Get Port from header information.
/// Priority:
/// 1. x-forwarded-port
/// 2. host header (parse port)
/// 3. x-forwarded-proto inferred default (http=80, https=443) when host has no explicit port
/// 4. port header
pub fn get_request_port(headers: &HeaderMap) -> u16 {
if let Some(port) = headers
.get("x-forwarded-port")
.and_then(|v| v.to_str().ok())
.and_then(|s| s.parse::<u16>().ok())
{
return port;
}
if let Some(host) = headers.get("host").and_then(|v| v.to_str().ok()) {
if let Some(idx) = host.rfind(':') {
let valid_colon = match host.rfind(']') {
Some(close_bracket_idx) => idx > close_bracket_idx,
None => true,
};
if valid_colon
&& let Ok(port) = host[idx + 1..].parse::<u16>()
&& port > 0
{
return port;
}
}
if let Some(proto) = headers.get("x-forwarded-proto").and_then(|v| v.to_str().ok()) {
match proto {
"http" => return 80,
"https" => return 443,
_ => {}
}
}
}
headers
.get("port")
.and_then(|v| v.to_str().ok())
.and_then(|s| s.parse::<u16>().ok())
.unwrap_or(0)
}
/// Get content-length from header information.
pub fn get_request_content_length(headers: &HeaderMap) -> u64 {
headers
.get("content-length")
.and_then(|v| v.to_str().ok())
.and_then(|s| s.parse::<u64>().ok())
.unwrap_or(0)
}
/// Get referer from header information.
pub fn get_request_referer(headers: &HeaderMap) -> String {
headers
.get("referer")
.and_then(|v| v.to_str().ok())
.unwrap_or_default()
.to_string()
}
/// Get user-agent from header information.
pub fn get_request_user_agent(headers: &HeaderMap) -> String {
headers
.get("user-agent")
.and_then(|v| v.to_str().ok())
.unwrap_or_default()
.to_string()
}
/// parse_host parses a string into a Host, with validation similar to Go's ParseHost.
pub fn parse_host(s: &str) -> Result<Host, NetError> {
if s.is_empty() {
return Err(NetError::InvalidArgument);
}
// is_valid_host validates the host string, checking for IP or hostname validity.
let is_valid_host = |host: &str| -> bool {
if host.is_empty() {
return true;
@@ -122,9 +229,6 @@ pub fn parse_host(s: &str) -> Result<Host, NetError> {
return Err(NetError::MissingBracket);
}
// A host with multiple colons is an IPv6 literal, optionally with a
// zone identifier. Unbracketed IPv6 with port is ambiguous, so callers
// must use the standard bracketed form when they need a port.
let (host_str, port_str) = if s.matches(':').count() > 1 {
(s, "")
} else {
@@ -139,7 +243,6 @@ pub fn parse_host(s: &str) -> Result<Host, NetError> {
(trim_ipv6(host_str)?, port)
};
// Handle IPv6 zone identifier.
let trimmed_host = host.split('%').next().unwrap_or(&host);
if !is_valid_host(trimmed_host) {
@@ -149,7 +252,6 @@ pub fn parse_host(s: &str) -> Result<Host, NetError> {
Ok(Host { name: host, port })
}
// trim_ipv6 removes square brackets from IPv6 addresses, similar to Go's trimIPv6.
fn trim_ipv6(host: &str) -> Result<String, NetError> {
if host.ends_with(']') {
if !host.starts_with('[') {
@@ -162,37 +264,18 @@ fn trim_ipv6(host: &str) -> Result<String, NetError> {
}
/// URL is a wrapper around url::Url for custom handling.
/// Provides methods similar to Go's URL struct.
#[derive(Debug, Clone)]
pub struct ParsedURL(pub Url);
impl ParsedURL {
/// is_empty returns true if the URL is empty or "about:blank".
///
/// # Arguments
/// * `&self` - Reference to the ParsedURL instance.
///
/// # Returns
/// * `bool` - True if the URL is empty or "about:blank", false otherwise.
///
pub fn is_empty(&self) -> bool {
self.0.as_str() == "" || (self.0.scheme() == "about" && self.0.path() == "blank")
}
/// hostname returns the hostname of the URL.
///
/// # Returns
/// * `String` - The hostname of the URL, or an empty string if not set.
///
pub fn hostname(&self) -> String {
self.0.host_str().unwrap_or("").to_string()
}
/// port returns the port of the URL as a string, defaulting to "80" for http and "443" for https if not set.
///
/// # Returns
/// * `String` - The port of the URL as a string.
///
pub fn port(&self) -> String {
match self.0.port() {
Some(p) => p.to_string(),
@@ -204,20 +287,10 @@ impl ParsedURL {
}
}
/// scheme returns the scheme of the URL.
///
/// # Returns
/// * `&str` - The scheme of the URL.
///
pub fn scheme(&self) -> &str {
self.0.scheme()
}
/// url returns a reference to the underlying Url.
///
/// # Returns
/// * `&Url` - Reference to the underlying Url.
///
pub fn url(&self) -> &Url {
&self.0
}
@@ -235,7 +308,6 @@ impl std::fmt::Display for ParsedURL {
}
let mut s = url.to_string();
// If the URL ends with a slash and the path is just "/", remove the trailing slash.
if s.ends_with('/') && url.path() == "/" {
s.pop();
}
@@ -268,17 +340,6 @@ impl<'de> serde::Deserialize<'de> for ParsedURL {
}
/// parse_url parses a string into a ParsedURL, with host validation and path cleaning.
///
/// # Arguments
/// * `s` - The URL string to parse.
///
/// # Returns
/// * `Ok(ParsedURL)` - If parsing is successful.
/// * `Err(NetError)` - If parsing fails or host is invalid.
///
/// # Errors
/// Returns NetError if parsing fails or host is invalid.
///
pub fn parse_url(s: &str) -> Result<ParsedURL, NetError> {
if let Some(scheme_end) = s.find("://")
&& s[scheme_end + 3..].starts_with('/')
@@ -303,13 +364,11 @@ pub fn parse_url(s: &str) -> Result<ParsedURL, NetError> {
if !port_str.is_empty() {
let host_port = format!("{}:{}", uu.host_str().unwrap(), port_str);
parse_host(&host_port)?; // Validate host.
parse_host(&host_port)?;
}
}
// Clean path: Use Url's path_segments to normalize.
if !uu.path().is_empty() {
// Url automatically cleans paths, but we ensure trailing slash if original had it.
let mut cleaned_path = String::new();
for comp in Path::new(uu.path()).components() {
use std::path::Component;
@@ -337,15 +396,6 @@ pub fn parse_url(s: &str) -> Result<ParsedURL, NetError> {
}
#[allow(dead_code)]
/// parse_http_url parses a string into a ParsedURL, ensuring the scheme is http or https.
///
/// # Arguments
/// * `s` - The URL string to parse.
///
/// # Returns
/// * `Ok(ParsedURL)` - If parsing is successful and scheme is http/https.
/// * `Err(NetError)` - If parsing fails or scheme is not http/https.
///
pub fn parse_http_url(s: &str) -> Result<ParsedURL, NetError> {
let u = parse_url(s)?;
match u.0.scheme() {
@@ -355,20 +405,10 @@ pub fn parse_http_url(s: &str) -> Result<ParsedURL, NetError> {
}
#[allow(dead_code)]
/// is_network_or_host_down checks if an error indicates network or host down, considering timeouts.
///
/// # Arguments
/// * `err` - The std::io::Error to check.
/// * `expect_timeouts` - Whether timeouts are expected.
///
/// # Returns
/// * `bool` - True if the error indicates network or host down, false otherwise.
///
pub fn is_network_or_host_down(err: &std::io::Error, expect_timeouts: bool) -> bool {
if err.kind() == std::io::ErrorKind::TimedOut {
return !expect_timeouts;
}
// Simplified checks based on Go logic; adapt for Rust as needed
let err_str = err.to_string().to_lowercase();
err_str.contains("connection reset by peer")
|| err_str.contains("connection timed out")
@@ -377,27 +417,11 @@ pub fn is_network_or_host_down(err: &std::io::Error, expect_timeouts: bool) -> b
}
#[allow(dead_code)]
/// is_conn_reset_err checks if an error indicates a connection reset by peer.
///
/// # Arguments
/// * `err` - The std::io::Error to check.
///
/// # Returns
/// * `bool` - True if the error indicates connection reset, false otherwise.
///
pub fn is_conn_reset_err(err: &std::io::Error) -> bool {
err.to_string().contains("connection reset by peer") || matches!(err.raw_os_error(), Some(libc::ECONNRESET))
}
#[allow(dead_code)]
/// is_conn_refused_err checks if an error indicates a connection refused.
///
/// # Arguments
/// * `err` - The std::io::Error to check.
///
/// # Returns
/// * `bool` - True if the error indicates connection refused, false otherwise.
///
pub fn is_conn_refused_err(err: &std::io::Error) -> bool {
err.to_string().contains("connection refused") || matches!(err.raw_os_error(), Some(libc::ECONNREFUSED))
}
@@ -405,6 +429,51 @@ pub fn is_conn_refused_err(err: &std::io::Error) -> bool {
#[cfg(test)]
mod tests {
use super::*;
use hyper::header::HeaderValue;
#[test]
fn test_get_request_port() {
let mut headers = HeaderMap::new();
assert_eq!(get_request_port(&headers), 0);
headers.insert("port", HeaderValue::from_static("8080"));
assert_eq!(get_request_port(&headers), 8080);
headers.remove("port");
headers.insert("host", HeaderValue::from_static("example.com:9000"));
assert_eq!(get_request_port(&headers), 9000);
headers.insert("host", HeaderValue::from_static("example.com"));
assert_eq!(get_request_port(&headers), 0);
headers.insert("host", HeaderValue::from_static("[::1]:9001"));
assert_eq!(get_request_port(&headers), 9001);
headers.insert("host", HeaderValue::from_static("[::1]"));
assert_eq!(get_request_port(&headers), 0);
headers.insert("x-forwarded-port", HeaderValue::from_static("7000"));
assert_eq!(get_request_port(&headers), 7000);
headers.remove("x-forwarded-port");
headers.insert("host", HeaderValue::from_static("example.com"));
headers.insert("x-forwarded-proto", HeaderValue::from_static("http"));
assert_eq!(get_request_port(&headers), 80);
headers.insert("x-forwarded-proto", HeaderValue::from_static("https"));
assert_eq!(get_request_port(&headers), 443);
headers.insert("x-forwarded-proto", HeaderValue::from_static("ftp"));
assert_eq!(get_request_port(&headers), 0);
headers.insert("host", HeaderValue::from_static("example.com:0"));
headers.insert("x-forwarded-proto", HeaderValue::from_static("https"));
assert_eq!(get_request_port(&headers), 443);
headers.remove("x-forwarded-proto");
assert_eq!(get_request_port(&headers), 0);
}
#[test]
fn parse_host_with_empty_string_returns_error() {
@@ -532,32 +601,6 @@ mod tests {
assert_eq!(host.to_string(), "example.com");
}
#[test]
fn host_equal_when_same() {
let host1 = Host {
name: "example.com".to_string(),
port: Some(80),
};
let host2 = Host {
name: "example.com".to_string(),
port: Some(80),
};
assert!(host1.equal(&host2));
}
#[test]
fn host_not_equal_when_different() {
let host1 = Host {
name: "example.com".to_string(),
port: Some(80),
};
let host2 = Host {
name: "example.com".to_string(),
port: Some(443),
};
assert!(!host1.equal(&host2));
}
#[test]
fn parse_url_with_valid_http_url() {
let result = parse_url("http://example.com/path");
@@ -565,100 +608,35 @@ mod tests {
let parsed = result.unwrap();
assert_eq!(parsed.hostname(), "example.com");
assert_eq!(parsed.port(), "80");
assert_eq!(parsed.scheme(), "http");
assert_eq!(parsed.to_string(), "http://example.com/path");
}
#[test]
fn parse_url_with_valid_https_url() {
fn parse_url_with_explicit_default_https_port() {
let result = parse_url("https://example.com:443/path");
assert!(result.is_ok());
let parsed = result.unwrap();
assert_eq!(parsed.hostname(), "example.com");
assert_eq!(parsed.port(), "443");
assert_eq!(parsed.to_string(), "https://example.com/path");
}
#[test]
fn parse_url_with_scheme_but_empty_host() {
fn parse_url_with_empty_host_returns_error() {
let result = parse_url("http:///path");
assert!(matches!(result, Err(NetError::SchemeWithEmptyHost)));
}
#[test]
fn parse_url_with_invalid_host() {
fn parse_url_with_invalid_host_returns_error() {
let result = parse_url("http://invalid..host/path");
assert!(matches!(result, Err(NetError::InvalidHost)));
}
#[test]
fn parse_url_with_path_cleaning() {
fn parse_url_normalizes_path() {
let result = parse_url("http://example.com//path/../path/");
assert!(result.is_ok());
let parsed = result.unwrap();
assert_eq!(parsed.0.path(), "/path/");
}
#[test]
fn parse_http_url_with_http_scheme() {
let result = parse_http_url("http://example.com");
assert!(result.is_ok());
}
#[test]
fn parse_http_url_with_https_scheme() {
let result = parse_http_url("https://example.com");
assert!(result.is_ok());
}
#[test]
fn parse_http_url_with_invalid_scheme() {
let result = parse_http_url("ftp://example.com");
assert!(matches!(result, Err(NetError::UnexpectedScheme(_))));
}
#[test]
fn parsed_url_is_empty_when_url_is_empty() {
let url = ParsedURL(Url::parse("about:blank").unwrap());
assert!(url.is_empty());
}
#[test]
fn parsed_url_hostname() {
let url = ParsedURL(Url::parse("http://example.com:8080").unwrap());
assert_eq!(url.hostname(), "example.com");
}
#[test]
fn parsed_url_port() {
let url = ParsedURL(Url::parse("http://example.com:8080").unwrap());
assert_eq!(url.port(), "8080");
}
#[test]
fn parsed_url_to_string_removes_default_ports() {
let url = ParsedURL(Url::parse("http://example.com:80").unwrap());
assert_eq!(url.to_string(), "http://example.com");
}
#[test]
fn is_network_or_host_down_with_timeout() {
let err = std::io::Error::new(std::io::ErrorKind::TimedOut, "timeout");
assert!(is_network_or_host_down(&err, false));
}
#[test]
fn is_network_or_host_down_with_expected_timeout() {
let err = std::io::Error::new(std::io::ErrorKind::TimedOut, "timeout");
assert!(!is_network_or_host_down(&err, true));
}
#[test]
fn is_conn_reset_err_with_reset_message() {
let err = std::io::Error::other("connection reset by peer");
assert!(is_conn_reset_err(&err));
}
#[test]
fn is_conn_refused_err_with_refused_message() {
let err = std::io::Error::other("connection refused");
assert!(is_conn_refused_err(&err));
assert_eq!(parsed.to_string(), "http://example.com/path/");
}
}
crates/targets/src/runtime/mod.rs
+1
View File
@@ -15,6 +15,7 @@
pub mod adapter;
pub mod sidecar;
pub mod sidecar_protocol;
pub mod tls;
use crate::Target;
use crate::arn::TargetID;
crates/targets/src/runtime/tls/adapter.rs
+217
View File
@@ -0,0 +1,217 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! `TlsReloadAdapter<M>` — the single entry-point that connects a target to
//! the TLS reload coordinator. Each target holds an `Option<TlsReloadAdapter<M>>`
//! and calls `current_material()` on the hot path. When `None`, the target
//! falls back to its legacy inline fingerprint logic.
use super::config::TlsReloadOptions;
use super::coordinator::TargetTlsReloadCoordinator;
use super::state::{TargetTlsRuntimeState, TargetTlsStatusSnapshot};
use super::r#trait::ReloadableTargetTls;
use std::sync::Arc;
use tracing::warn;
/// Bridges a `ReloadableTargetTls` implementor and the reload coordinator.
///
/// Created via [`TlsReloadAdapter::try_register`]. Holds the coordinator-
/// managed runtime state and exposes a zero-cost `current_material()` accessor
/// for the send hot-path.
pub struct TlsReloadAdapter<M> {
runtime_state: Arc<TargetTlsRuntimeState<M>>,
options: TlsReloadOptions,
}
impl<M> std::fmt::Debug for TlsReloadAdapter<M> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("TlsReloadAdapter")
.field("target_label", &self.runtime_state.inputs.target_label)
.finish_non_exhaustive()
}
}
impl<M> Clone for TlsReloadAdapter<M> {
fn clone(&self) -> Self {
Self {
runtime_state: Arc::clone(&self.runtime_state),
options: self.options.clone(),
}
}
}
impl<M: Send + Sync + 'static> TlsReloadAdapter<M> {
/// Registers `target` with the coordinator and returns an adapter.
///
/// On success the coordinator has:
/// - built initial TLS material
/// - spawned a background poll loop
///
/// On failure returns `None` (the caller should keep its inline fallback
/// path intact — the target continues to work, just without coordinator
/// support).
pub async fn try_register<T: ReloadableTargetTls<Material = M>>(
target: Arc<T>,
options: TlsReloadOptions,
coordinator: &TargetTlsReloadCoordinator,
) -> Option<Self> {
let label = target.tls_input_set().target_label.clone();
match coordinator.register(target, options.clone()).await {
Ok(runtime_state) => {
tracing::info!(target = %label, "TLS reload adapter registered");
Some(Self { runtime_state, options })
}
Err(err) => {
warn!(target = %label, error = %err, "TLS reload adapter registration failed; target will use inline fallback");
None
}
}
}
/// Hot-path accessor: returns the current TLS material managed by the
/// coordinator. The returned `Arc<M>` is cheap to clone.
#[inline]
pub fn current_material(&self) -> Arc<M> {
Arc::clone(&self.runtime_state.current.load().material)
}
/// Returns the active generation counter.
#[inline]
pub fn generation(&self) -> u64 {
self.runtime_state.current.load().generation.0
}
/// Returns a read-only status snapshot for admin/observability.
pub fn status_snapshot(&self) -> TargetTlsStatusSnapshot {
TargetTlsReloadCoordinator::build_status_snapshot(&self.runtime_state, &self.options)
}
/// Returns the underlying runtime state (for `close()` cleanup etc.).
pub fn runtime_state(&self) -> &Arc<TargetTlsRuntimeState<M>> {
&self.runtime_state
}
/// Unregisters from the coordinator (stops the poll loop).
pub async fn unregister(&self, coordinator: &TargetTlsReloadCoordinator) {
let label = &self.runtime_state.inputs.target_label;
if let Err(err) = coordinator.unregister(label).await {
warn!(target = %label, error = %err, "Failed to unregister TLS reload adapter");
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::error::TargetError;
use async_trait::async_trait;
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
struct FakeTarget {
label: String,
build_calls: AtomicUsize,
should_fail: AtomicBool,
}
impl FakeTarget {
fn new(label: &str) -> Self {
Self {
label: label.to_string(),
build_calls: AtomicUsize::new(0),
should_fail: AtomicBool::new(false),
}
}
}
#[async_trait]
impl ReloadableTargetTls for FakeTarget {
type Material = String;
fn tls_input_set(&self) -> super::super::state::TargetTlsInputSet {
super::super::state::TargetTlsInputSet {
ca_path: String::new(),
client_cert_path: String::new(),
client_key_path: String::new(),
target_label: self.label.clone(),
}
}
async fn build_tls_material(&self) -> Result<Self::Material, TargetError> {
self.build_calls.fetch_add(1, Ordering::SeqCst);
if self.should_fail.load(Ordering::SeqCst) {
return Err(TargetError::Configuration("fail".to_string()));
}
Ok("material".to_string())
}
async fn apply_tls_material(
&self,
_generation: super::super::fingerprint::TargetTlsGeneration,
_material: Arc<Self::Material>,
_mode: super::super::config::ReloadApplyMode,
) -> Result<(), TargetError> {
Ok(())
}
}
#[tokio::test]
async fn try_register_returns_adapter_on_success() {
let coordinator = TargetTlsReloadCoordinator::new();
let target = Arc::new(FakeTarget::new("test:fake"));
let options = TlsReloadOptions::default();
let adapter = TlsReloadAdapter::try_register(target.clone(), options, &coordinator).await;
assert!(adapter.is_some());
assert_eq!(target.build_calls.load(Ordering::SeqCst), 1);
let a = adapter.unwrap();
assert_eq!(*a.current_material(), "material");
}
#[tokio::test]
async fn try_register_returns_none_on_failure() {
let coordinator = TargetTlsReloadCoordinator::new();
let target = Arc::new(FakeTarget::new("test:fail"));
target.should_fail.store(true, Ordering::SeqCst);
let options = TlsReloadOptions::default();
let adapter = TlsReloadAdapter::try_register(target, options, &coordinator).await;
assert!(adapter.is_none());
}
#[tokio::test]
async fn adapter_is_clone_and_shares_state() {
let coordinator = TargetTlsReloadCoordinator::new();
let target = Arc::new(FakeTarget::new("test:clone"));
let options = TlsReloadOptions::default();
let a = TlsReloadAdapter::try_register(target, options, &coordinator).await.unwrap();
let b = a.clone();
assert_eq!(*a.current_material(), *b.current_material());
assert_eq!(a.generation(), b.generation());
}
#[tokio::test]
async fn status_snapshot_contains_label() {
let coordinator = TargetTlsReloadCoordinator::new();
let target = Arc::new(FakeTarget::new("test:snap"));
let options = TlsReloadOptions::default();
let adapter = TlsReloadAdapter::try_register(target, options, &coordinator).await.unwrap();
let snap = adapter.status_snapshot();
assert_eq!(snap.target_label, "test:snap");
assert!(snap.reload_enabled);
}
}
crates/targets/src/runtime/tls/config.rs
+20
View File
@@ -0,0 +1,20 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Target-level TLS reload configuration.
//!
//! Re-exports the shared types from `rustfs_tls_runtime::config` so that
//! targets and their callers use a single source of truth.
pub use rustfs_tls_runtime::config::{ReloadApplyHint as ReloadApplyMode, ReloadDetectMode, TlsReloadOptions};
crates/targets/src/runtime/tls/coordinator.rs
+656
View File
@@ -0,0 +1,656 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Target TLS reload coordinator. Manages per-target background poll loops
//! that periodically check TLS material fingerprints and drive safe reload.
use super::config::{ReloadApplyMode, ReloadDetectMode, TlsReloadOptions};
#[cfg(test)]
use super::fingerprint::TargetTlsFingerprint;
use super::fingerprint::{TargetTlsGeneration, build_target_tls_fingerprint};
use super::metrics::{record_target_tls_publication_fail, record_target_tls_reload_result, record_target_tls_reload_skipped};
#[cfg(test)]
use super::state::TargetTlsInputSet;
use super::state::{TargetTlsPublishedState, TargetTlsRuntimeState, TargetTlsStatusSnapshot};
use super::r#trait::ReloadableTargetTls;
use super::validate::validate_tls_material;
use crate::error::TargetError;
use std::collections::HashMap;
use std::sync::Arc;
use std::time::{SystemTime, UNIX_EPOCH};
use tokio::sync::RwLock;
use tokio::task::JoinHandle;
use tracing::{debug, info, warn};
struct TargetReloadEntry {
#[expect(dead_code)]
target_label: String,
cancel_tx: tokio::sync::mpsc::Sender<()>,
poll_handle: JoinHandle<()>,
}
/// The top-level coordinator that manages TLS reload for all registered targets.
///
/// Typically one instance per process, held alongside `TargetRuntimeManager`.
/// Each registered target gets its own background poll loop that periodically
/// checks TLS fingerprints and drives the build/apply cycle.
pub struct TargetTlsReloadCoordinator {
entries: RwLock<HashMap<String, TargetReloadEntry>>,
}
impl Default for TargetTlsReloadCoordinator {
fn default() -> Self {
Self::new()
}
}
impl TargetTlsReloadCoordinator {
pub fn new() -> Self {
Self {
entries: RwLock::new(HashMap::new()),
}
}
/// Register a target for coordinated TLS reload. Spawns a background poll loop.
///
/// Returns the initial runtime state that the target should hold for
/// accessing the current TLS material via `ArcSwap`.
pub async fn register<T: ReloadableTargetTls>(
&self,
target: Arc<T>,
options: TlsReloadOptions,
) -> Result<Arc<TargetTlsRuntimeState<T::Material>>, TargetError> {
if !options.enabled {
return Err(TargetError::Configuration("TLS reload is disabled".to_string()));
}
let inputs = target.tls_input_set();
let target_label = inputs.target_label.clone();
// Build initial material
let initial_material = Arc::new(target.build_tls_material().await?);
let initial_fingerprint =
build_target_tls_fingerprint(&inputs.ca_path, &inputs.client_cert_path, &inputs.client_key_path).await?;
let initial_state = Arc::new(TargetTlsPublishedState {
generation: TargetTlsGeneration(1),
fingerprint: initial_fingerprint,
material: initial_material,
loaded_at_unix_ms: unix_time_ms(),
});
let runtime_state = Arc::new(TargetTlsRuntimeState::new(initial_state.clone(), inputs));
if options.detect_mode == ReloadDetectMode::Poll || options.detect_mode == ReloadDetectMode::Hybrid {
let (cancel_tx, cancel_rx) = tokio::sync::mpsc::channel(1);
let poll_handle = tokio::spawn(spawn_target_poll_loop(target, Arc::clone(&runtime_state), options, cancel_rx));
let mut entries = self.entries.write().await;
entries.insert(
target_label.clone(),
TargetReloadEntry {
target_label: target_label.clone(),
cancel_tx,
poll_handle,
},
);
info!(target = %target_label, "Registered target for TLS reload coordinator");
}
Ok(runtime_state)
}
/// Unregister a target and stop its poll loop.
pub async fn unregister(&self, target_label: &str) -> Result<(), TargetError> {
let mut entries = self.entries.write().await;
if let Some(entry) = entries.remove(target_label) {
let _ = entry.cancel_tx.send(()).await;
entry.poll_handle.abort();
info!(target = %target_label, "Unregistered target from TLS reload coordinator");
}
Ok(())
}
/// Force an immediate reload check for a specific target.
/// Used by admin endpoints and test harnesses.
pub async fn force_reload<T: ReloadableTargetTls>(
&self,
target: &T,
runtime_state: &TargetTlsRuntimeState<T::Material>,
options: &TlsReloadOptions,
) -> Result<TargetTlsGeneration, TargetError> {
reload_target_once(target, runtime_state, options).await
}
/// Stop all poll loops.
pub async fn shutdown(&self) {
let mut entries = self.entries.write().await;
for (label, entry) in entries.drain() {
let _ = entry.cancel_tx.send(()).await;
entry.poll_handle.abort();
debug!(target = %label, "Stopped TLS reload poll loop");
}
}
/// Collect status snapshots from all registered targets.
/// The caller must provide the runtime states separately since the
/// coordinator does not hold type-erased references to them.
pub fn build_status_snapshot<M>(
runtime_state: &TargetTlsRuntimeState<M>,
options: &TlsReloadOptions,
) -> TargetTlsStatusSnapshot {
let current = runtime_state.current.load();
let last_attempt = runtime_state.last_attempt_unix_ms();
let last_success = runtime_state.last_success_unix_ms();
let last_error = runtime_state.last_error.read().clone();
TargetTlsStatusSnapshot {
target_label: runtime_state.inputs.target_label.clone(),
generation: current.generation.0,
reload_enabled: options.enabled,
detect_mode: match options.detect_mode {
ReloadDetectMode::Poll => "poll",
ReloadDetectMode::Watch => "watch",
ReloadDetectMode::Hybrid => "hybrid",
},
apply_mode: match options.apply_hint {
ReloadApplyMode::Lazy => "lazy",
ReloadApplyMode::SoftReconnect => "soft_reconnect",
},
last_attempt_time: if last_attempt > 0 { Some(last_attempt) } else { None },
last_success_time: if last_success > 0 { Some(last_success) } else { None },
last_error,
ca_path: runtime_state.inputs.ca_path.clone(),
client_cert_path: runtime_state.inputs.client_cert_path.clone(),
client_key_path: runtime_state.inputs.client_key_path.clone(),
}
}
}
/// Background poll loop for a single target.
async fn spawn_target_poll_loop<T: ReloadableTargetTls>(
target: Arc<T>,
runtime_state: Arc<TargetTlsRuntimeState<T::Material>>,
options: TlsReloadOptions,
mut cancel_rx: tokio::sync::mpsc::Receiver<()>,
) {
let mut interval = tokio::time::interval(options.interval);
interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
interval.tick().await; // skip the immediate first tick
let label = &runtime_state.inputs.target_label;
let debounce = options.debounce;
debug!(target = %label, interval_secs = options.interval.as_secs(), "TLS reload poll loop started");
loop {
tokio::select! {
biased;
_ = cancel_rx.recv() => {
info!(target = %label, "TLS reload poll loop stopped");
return;
}
_ = interval.tick() => {
// Enforce minimum stable age: if the last attempt was too recent
// (e.g. a rapid succession of ticks), wait one debounce period
// before reading files again to avoid picking up half-written certs.
let last_attempt = runtime_state.last_attempt_unix_ms();
if last_attempt > 0 {
let elapsed_since_last = unix_time_ms().saturating_sub(last_attempt);
if elapsed_since_last < debounce.as_millis() as u64 {
continue;
}
}
if let Err(err) = reload_target_once(target.as_ref(), runtime_state.as_ref(), &options).await {
warn!(target = %label, error = %err, "TLS reload poll check failed (will retry)");
}
}
}
}
}
/// Single reload cycle: read → compare → validate → build → apply → publish.
///
/// Returns the new generation on success, or an error on failure.
/// On failure the current generation and material are untouched.
async fn reload_target_once<T: ReloadableTargetTls>(
target: &T,
runtime_state: &TargetTlsRuntimeState<T::Material>,
options: &TlsReloadOptions,
) -> Result<TargetTlsGeneration, TargetError> {
let now = unix_time_ms();
runtime_state.mark_attempt(now);
let started_at = std::time::Instant::now();
let label = &runtime_state.inputs.target_label;
// 1. Read TLS files and compute fingerprint
let inputs = &runtime_state.inputs;
let next_fingerprint =
build_target_tls_fingerprint(&inputs.ca_path, &inputs.client_cert_path, &inputs.client_key_path).await?;
// 2. Compare with current — skip if unchanged
let current = runtime_state.current.load();
if current.fingerprint == next_fingerprint {
record_target_tls_reload_skipped(label, "unchanged");
return Ok(current.generation);
}
// 3. Validate TLS files (cert/key pairing, CA parseable)
if let Err(err) = validate_tls_material(&inputs.ca_path, &inputs.client_cert_path, &inputs.client_key_path) {
*runtime_state.last_error.write() = Some(err.to_string());
record_target_tls_publication_fail(label);
return Err(err);
}
// Also call target-specific validation
if let Err(err) = target.validate_tls_files().await {
*runtime_state.last_error.write() = Some(err.to_string());
record_target_tls_publication_fail(label);
return Err(err);
}
// 4. Build new material (does not touch current state yet)
let new_material = match target.build_tls_material().await {
Ok(m) => Arc::new(m),
Err(err) => {
*runtime_state.last_error.write() = Some(err.to_string());
record_target_tls_publication_fail(label);
return Err(err);
}
};
// 5. Bump generation and apply
let new_generation = runtime_state.bump_generation();
if let Err(err) = target
.apply_tls_material(new_generation, Arc::clone(&new_material), options.apply_hint)
.await
{
*runtime_state.last_error.write() = Some(err.to_string());
record_target_tls_publication_fail(label);
return Err(err);
}
// 6. Publish new state
let published = Arc::new(TargetTlsPublishedState {
generation: new_generation,
fingerprint: next_fingerprint,
material: new_material,
loaded_at_unix_ms: now,
});
runtime_state.current.store(published.clone());
runtime_state.last_good.store(published);
runtime_state.mark_success(now);
*runtime_state.last_error.write() = None;
record_target_tls_reload_result(label, "ok", started_at.elapsed().as_secs_f64(), new_generation.0);
debug!(target = %label, generation = new_generation.0, "TLS reload successful");
Ok(new_generation)
}
fn unix_time_ms() -> u64 {
SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_millis() as u64
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
struct MockTarget {
inputs: TargetTlsInputSet,
build_calls: AtomicUsize,
apply_calls: AtomicUsize,
validate_calls: AtomicUsize,
should_fail_build: AtomicBool,
should_fail_apply: AtomicBool,
should_fail_validate: AtomicBool,
}
impl MockTarget {
fn new(label: &str) -> Self {
Self {
inputs: TargetTlsInputSet {
ca_path: String::new(),
client_cert_path: String::new(),
client_key_path: String::new(),
target_label: label.to_string(),
},
build_calls: AtomicUsize::new(0),
apply_calls: AtomicUsize::new(0),
validate_calls: AtomicUsize::new(0),
should_fail_build: AtomicBool::new(false),
should_fail_apply: AtomicBool::new(false),
should_fail_validate: AtomicBool::new(false),
}
}
}
#[async_trait::async_trait]
impl ReloadableTargetTls for MockTarget {
type Material = String;
fn tls_input_set(&self) -> TargetTlsInputSet {
self.inputs.clone()
}
async fn build_tls_material(&self) -> Result<Self::Material, TargetError> {
self.build_calls.fetch_add(1, Ordering::SeqCst);
if self.should_fail_build.load(Ordering::SeqCst) {
return Err(TargetError::Configuration("build failed".to_string()));
}
Ok("mock-material".to_string())
}
async fn apply_tls_material(
&self,
_generation: TargetTlsGeneration,
_material: Arc<Self::Material>,
_mode: ReloadApplyMode,
) -> Result<(), TargetError> {
self.apply_calls.fetch_add(1, Ordering::SeqCst);
if self.should_fail_apply.load(Ordering::SeqCst) {
return Err(TargetError::Configuration("apply failed".to_string()));
}
Ok(())
}
async fn validate_tls_files(&self) -> Result<(), TargetError> {
self.validate_calls.fetch_add(1, Ordering::SeqCst);
if self.should_fail_validate.load(Ordering::SeqCst) {
return Err(TargetError::Configuration("validate failed".to_string()));
}
Ok(())
}
}
fn default_options() -> TlsReloadOptions {
TlsReloadOptions {
enabled: true,
detect_mode: ReloadDetectMode::Poll,
interval: std::time::Duration::from_secs(1),
debounce: std::time::Duration::from_secs(1),
min_stable_age: std::time::Duration::from_millis(100),
apply_hint: ReloadApplyMode::Lazy,
}
}
#[tokio::test]
async fn register_builds_initial_material() {
let coordinator = TargetTlsReloadCoordinator::new();
let target = Arc::new(MockTarget::new("test:webhook"));
let options = TlsReloadOptions {
detect_mode: ReloadDetectMode::Watch, // no poll loop for this test
..default_options()
};
let state = coordinator.register(target.clone(), options).await.unwrap();
assert_eq!(state.current.load().generation, TargetTlsGeneration(1));
assert_eq!(target.build_calls.load(Ordering::SeqCst), 1);
}
#[tokio::test]
async fn register_disabled_returns_error() {
let coordinator = TargetTlsReloadCoordinator::new();
let target = Arc::new(MockTarget::new("test:webhook"));
let options = TlsReloadOptions {
enabled: false,
..default_options()
};
let result = coordinator.register(target, options).await;
assert!(result.is_err());
}
#[tokio::test]
async fn shutdown_stops_all_loops() {
let coordinator = TargetTlsReloadCoordinator::new();
let target = Arc::new(MockTarget::new("test:webhook"));
let _state = coordinator.register(target.clone(), default_options()).await.unwrap();
assert_eq!(coordinator.entries.read().await.len(), 1);
coordinator.shutdown().await;
assert!(coordinator.entries.read().await.is_empty());
}
#[tokio::test]
async fn force_reload_calls_build_and_apply() {
let target = MockTarget::new("test:webhook");
let initial_material = Arc::new("initial".to_string());
let initial_state = Arc::new(TargetTlsPublishedState {
generation: TargetTlsGeneration(1),
fingerprint: TargetTlsFingerprint::default(),
material: initial_material,
loaded_at_unix_ms: 0,
});
let inputs = target.tls_input_set();
let runtime_state = Arc::new(TargetTlsRuntimeState::new(initial_state, inputs));
let options = default_options();
// Force reload should succeed since MockTarget uses empty paths
// and the fingerprint won't change from default
let result = reload_target_once(&target, &runtime_state, &options).await.unwrap();
// Since fingerprint is unchanged (empty paths), generation stays at 1
assert_eq!(result, TargetTlsGeneration(1));
// Build should NOT be called because fingerprint unchanged
assert_eq!(target.build_calls.load(Ordering::SeqCst), 0);
}
#[tokio::test]
async fn build_failure_preserves_old_generation() {
let target = MockTarget::new("test:webhook");
target.should_fail_build.store(true, Ordering::SeqCst);
// Use a non-default fingerprint so the reload will detect a change
// (empty paths → default fingerprint ≠ initial fingerprint)
let initial_material = Arc::new("initial".to_string());
let initial_fingerprint = TargetTlsFingerprint {
ca_sha256: Some([1; 32]),
client_cert_sha256: None,
client_key_sha256: None,
};
let initial_state = Arc::new(TargetTlsPublishedState {
generation: TargetTlsGeneration(1),
fingerprint: initial_fingerprint,
material: initial_material,
loaded_at_unix_ms: 0,
});
let runtime_state = Arc::new(TargetTlsRuntimeState::new(initial_state, target.tls_input_set()));
let options = default_options();
// Empty paths produce default fingerprint which differs from initial →
// validate passes (empty paths), then build is called and fails.
let result = reload_target_once(&target, &runtime_state, &options).await;
assert!(result.is_err());
// Generation should remain at 1
assert_eq!(runtime_state.current_generation(), TargetTlsGeneration(1));
assert_eq!(target.build_calls.load(Ordering::SeqCst), 1);
}
#[tokio::test]
async fn status_snapshot_reflects_state() {
let target = MockTarget::new("test:webhook");
let initial_material = Arc::new("initial".to_string());
let initial_state = Arc::new(TargetTlsPublishedState {
generation: TargetTlsGeneration(1),
fingerprint: TargetTlsFingerprint::default(),
material: initial_material,
loaded_at_unix_ms: 0,
});
let runtime_state = Arc::new(TargetTlsRuntimeState::new(initial_state, target.tls_input_set()));
let options = default_options();
let snapshot = TargetTlsReloadCoordinator::build_status_snapshot(runtime_state.as_ref(), &options);
assert_eq!(snapshot.target_label, "test:webhook");
assert_eq!(snapshot.generation, 1);
assert!(snapshot.reload_enabled);
assert_eq!(snapshot.detect_mode, "poll");
assert_eq!(snapshot.apply_mode, "lazy");
assert!(snapshot.last_attempt_time.is_none());
assert!(snapshot.last_error.is_none());
}
#[tokio::test]
async fn apply_failure_preserves_old_generation() {
let target = MockTarget::new("test:webhook");
target.should_fail_apply.store(true, Ordering::SeqCst);
// Use a non-default fingerprint so reload detects a change
let initial_material = Arc::new("initial".to_string());
let initial_fingerprint = TargetTlsFingerprint {
ca_sha256: Some([42; 32]),
client_cert_sha256: None,
client_key_sha256: None,
};
let initial_state = Arc::new(TargetTlsPublishedState {
generation: TargetTlsGeneration(3),
fingerprint: initial_fingerprint,
material: initial_material,
loaded_at_unix_ms: 0,
});
let runtime_state = Arc::new(TargetTlsRuntimeState::new(initial_state, target.tls_input_set()));
let options = default_options();
let result = reload_target_once(&target, &runtime_state, &options).await;
assert!(result.is_err());
// Generation should remain at 3
assert_eq!(runtime_state.current_generation(), TargetTlsGeneration(3));
assert!(runtime_state.last_error.read().is_some());
}
#[tokio::test]
async fn validate_failure_prevents_build() {
let target = MockTarget::new("test:kafka");
target.should_fail_validate.store(true, Ordering::SeqCst);
// Non-default fingerprint to trigger reload
let initial_material = Arc::new("initial".to_string());
let initial_fingerprint = TargetTlsFingerprint {
ca_sha256: Some([99; 32]),
client_cert_sha256: None,
client_key_sha256: None,
};
let initial_state = Arc::new(TargetTlsPublishedState {
generation: TargetTlsGeneration(1),
fingerprint: initial_fingerprint,
material: initial_material,
loaded_at_unix_ms: 0,
});
let runtime_state = Arc::new(TargetTlsRuntimeState::new(initial_state, target.tls_input_set()));
let options = default_options();
let result = reload_target_once(&target, &runtime_state, &options).await;
assert!(result.is_err());
// Build should NOT have been called
assert_eq!(target.build_calls.load(Ordering::SeqCst), 0);
// Validate was called
assert!(target.validate_calls.load(Ordering::SeqCst) > 0);
assert_eq!(runtime_state.current_generation(), TargetTlsGeneration(1));
}
#[tokio::test]
async fn error_is_cleared_on_successful_reload() {
let target = MockTarget::new("test:nats");
let initial_material = Arc::new("initial".to_string());
let initial_fingerprint = TargetTlsFingerprint {
ca_sha256: Some([1; 32]),
client_cert_sha256: None,
client_key_sha256: None,
};
let initial_state = Arc::new(TargetTlsPublishedState {
generation: TargetTlsGeneration(1),
fingerprint: initial_fingerprint,
material: initial_material,
loaded_at_unix_ms: 0,
});
let runtime_state = Arc::new(TargetTlsRuntimeState::new(initial_state, target.tls_input_set()));
let options = default_options();
// First: fail the reload
target.should_fail_build.store(true, Ordering::SeqCst);
let _ = reload_target_once(&target, &runtime_state, &options).await;
assert!(runtime_state.last_error.read().is_some());
// Now succeed (fingerprint still different from default)
target.should_fail_build.store(false, Ordering::SeqCst);
let result = reload_target_once(&target, &runtime_state, &options).await;
assert!(result.is_ok());
assert!(runtime_state.last_error.read().is_none());
assert!(runtime_state.last_success_unix_ms() > 0);
}
#[tokio::test]
async fn last_good_is_never_overwritten_by_failed_reload() {
let target = MockTarget::new("test:amqp");
let initial_material = Arc::new("good".to_string());
let initial_fingerprint = TargetTlsFingerprint {
ca_sha256: Some([5; 32]),
client_cert_sha256: None,
client_key_sha256: None,
};
let initial_state = Arc::new(TargetTlsPublishedState {
generation: TargetTlsGeneration(2),
fingerprint: initial_fingerprint.clone(),
material: initial_material.clone(),
loaded_at_unix_ms: 100,
});
let runtime_state = Arc::new(TargetTlsRuntimeState::new(initial_state, target.tls_input_set()));
// Verify last_good matches initial
let good = runtime_state.last_good.load();
assert_eq!(good.generation, TargetTlsGeneration(2));
// Fail a reload
target.should_fail_build.store(true, Ordering::SeqCst);
let _ = reload_target_once(&target, &runtime_state, &default_options()).await;
// last_good should still be the initial state
let good_after = runtime_state.last_good.load();
assert_eq!(good_after.generation, TargetTlsGeneration(2));
assert_eq!(good_after.fingerprint, initial_fingerprint);
}
#[tokio::test]
async fn unregister_stops_target_poll_loop() {
let coordinator = TargetTlsReloadCoordinator::new();
let target = Arc::new(MockTarget::new("test:pulsar"));
let _state = coordinator.register(target.clone(), default_options()).await.unwrap();
assert_eq!(coordinator.entries.read().await.len(), 1);
coordinator.unregister("test:pulsar").await.unwrap();
assert!(coordinator.entries.read().await.is_empty());
}
#[tokio::test]
async fn bump_generation_saturates_at_max() {
let target = MockTarget::new("test:saturation");
let initial_material = Arc::new("initial".to_string());
let max_gen_state = Arc::new(TargetTlsPublishedState {
generation: TargetTlsGeneration(u64::MAX),
fingerprint: TargetTlsFingerprint::default(),
material: initial_material,
loaded_at_unix_ms: 0,
});
let runtime_state = Arc::new(TargetTlsRuntimeState::new(max_gen_state, target.tls_input_set()));
let bumped = runtime_state.bump_generation();
assert_eq!(bumped, TargetTlsGeneration(u64::MAX)); // saturating add
}
}
crates/targets/src/runtime/tls/fingerprint.rs
+160
View File
@@ -0,0 +1,160 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! TLS fingerprint types for per-target certificate hot-reload detection.
use crate::error::TargetError;
/// SHA256 digest per TLS file component used to detect certificate changes.
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct TargetTlsFingerprint {
pub ca_sha256: Option<[u8; 32]>,
pub client_cert_sha256: Option<[u8; 32]>,
pub client_key_sha256: Option<[u8; 32]>,
}
/// Monotonically increasing generation counter bumped on each successful reload.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct TargetTlsGeneration(pub u64);
/// Combined TLS state held per-target for tracking reload progress.
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct TargetTlsState {
pub generation: TargetTlsGeneration,
pub fingerprint: Option<TargetTlsFingerprint>,
}
impl TargetTlsState {
/// Compares `next_fingerprint` with the current one. If different, bumps
/// generation and stores the new fingerprint. Returns `true` when changed.
pub fn refresh(&mut self, next_fingerprint: TargetTlsFingerprint) -> bool {
if self.fingerprint.as_ref() == Some(&next_fingerprint) {
return false;
}
self.generation = TargetTlsGeneration(self.generation.0.saturating_add(1));
self.fingerprint = Some(next_fingerprint);
true
}
/// Checks whether `candidate` differs from the stored fingerprint without
/// mutating state. Use this to gate a rebuild, then call `refresh` only
/// after the rebuild succeeds.
pub fn needs_update(&self, candidate: &TargetTlsFingerprint) -> bool {
self.fingerprint.as_ref() != Some(candidate)
}
/// Resets state to default (generation 0, no fingerprint).
pub fn reset(&mut self) {
*self = Self::default();
}
}
/// Reads the three TLS material files from disk and returns a fingerprint
/// computed from their SHA256 digests. Empty paths produce `None` digests.
pub async fn build_target_tls_fingerprint(
ca_path: &str,
client_cert_path: &str,
client_key_path: &str,
) -> Result<TargetTlsFingerprint, TargetError> {
async fn load_optional_digest(path: &str) -> Result<Option<[u8; 32]>, TargetError> {
if path.is_empty() {
return Ok(None);
}
let bytes = tokio::fs::read(path)
.await
.map_err(|e| TargetError::Configuration(format!("Failed to read TLS material '{path}': {e}")))?;
let digest = rustfs_tls_runtime::TlsFingerprint::from_optional_bytes(Some(&bytes), None, None, None, None).server_sha256;
Ok(digest)
}
Ok(TargetTlsFingerprint {
ca_sha256: load_optional_digest(ca_path).await?,
client_cert_sha256: load_optional_digest(client_cert_path).await?,
client_key_sha256: load_optional_digest(client_key_path).await?,
})
}
#[cfg(test)]
mod tests {
use super::{TargetTlsFingerprint, TargetTlsGeneration, TargetTlsState};
#[test]
fn refresh_increments_generation_only_when_fingerprint_changes() {
let mut state = TargetTlsState::default();
let first = TargetTlsFingerprint {
ca_sha256: Some([1; 32]),
client_cert_sha256: None,
client_key_sha256: None,
};
let second = TargetTlsFingerprint {
ca_sha256: Some([2; 32]),
client_cert_sha256: None,
client_key_sha256: None,
};
assert!(state.refresh(first.clone()));
assert_eq!(state.generation, TargetTlsGeneration(1));
assert!(!state.refresh(first));
assert_eq!(state.generation, TargetTlsGeneration(1));
assert!(state.refresh(second));
assert_eq!(state.generation, TargetTlsGeneration(2));
}
#[test]
fn reset_clears_generation_and_fingerprint() {
let mut state = TargetTlsState {
generation: TargetTlsGeneration(5),
fingerprint: Some(TargetTlsFingerprint {
ca_sha256: Some([9; 32]),
client_cert_sha256: None,
client_key_sha256: None,
}),
};
state.reset();
assert_eq!(state, TargetTlsState::default());
}
#[test]
fn fingerprint_eq_when_all_fields_match() {
let a = TargetTlsFingerprint {
ca_sha256: Some([42; 32]),
client_cert_sha256: Some([1; 32]),
client_key_sha256: None,
};
let b = TargetTlsFingerprint {
ca_sha256: Some([42; 32]),
client_cert_sha256: Some([1; 32]),
client_key_sha256: None,
};
assert_eq!(a, b);
}
#[test]
fn fingerprint_ne_when_ca_differs() {
let a = TargetTlsFingerprint {
ca_sha256: Some([1; 32]),
client_cert_sha256: None,
client_key_sha256: None,
};
let b = TargetTlsFingerprint {
ca_sha256: Some([2; 32]),
client_cert_sha256: None,
client_key_sha256: None,
};
assert_ne!(a, b);
}
}
crates/targets/src/runtime/tls/metrics.rs
+63
View File
@@ -0,0 +1,63 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Target-level TLS reload metrics.
use ::metrics::{counter, describe_counter, describe_gauge, describe_histogram, gauge, histogram};
const TARGET_TLS_RELOAD_TOTAL: &str = "rustfs_target_tls_reload_total";
const TARGET_TLS_RELOAD_SKIPPED_TOTAL: &str = "rustfs_target_tls_reload_skipped_total";
const TARGET_TLS_GENERATION: &str = "rustfs_target_tls_generation";
const TARGET_TLS_RELOAD_DURATION_SECONDS: &str = "rustfs_target_tls_reload_duration_seconds";
const TARGET_TLS_PUBLICATION_FAIL_TOTAL: &str = "rustfs_target_tls_publication_fail_total";
const TARGET_TLS_ACTIVE_GENERATION_MISMATCH_TOTAL: &str = "rustfs_target_tls_active_generation_mismatch_total";
/// Describes all target TLS metrics. Call once during initialization.
pub fn init_target_tls_metrics() {
describe_counter!(TARGET_TLS_RELOAD_TOTAL, "Total number of TLS reload attempts per target");
describe_counter!(
TARGET_TLS_RELOAD_SKIPPED_TOTAL,
"Number of TLS reloads skipped per target (unchanged, etc.)"
);
describe_gauge!(TARGET_TLS_GENERATION, "Current TLS generation per target");
describe_histogram!(TARGET_TLS_RELOAD_DURATION_SECONDS, "Duration of TLS reload attempts per target");
describe_counter!(TARGET_TLS_PUBLICATION_FAIL_TOTAL, "Number of TLS reload publication failures per target");
describe_counter!(
TARGET_TLS_ACTIVE_GENERATION_MISMATCH_TOTAL,
"Number of times a target's active connection used a stale TLS generation"
);
}
/// Records a TLS reload result (success or failure).
pub fn record_target_tls_reload_result(target: &str, result: &str, duration_secs: f64, generation: u64) {
counter!(TARGET_TLS_RELOAD_TOTAL, "target_id" => target.to_string(), "result" => result.to_string()).increment(1);
histogram!(TARGET_TLS_RELOAD_DURATION_SECONDS, "target_id" => target.to_string(), "result" => result.to_string())
.record(duration_secs);
gauge!(TARGET_TLS_GENERATION, "target_id" => target.to_string()).set(generation as f64);
}
/// Records a skipped reload (typically because fingerprint was unchanged).
pub fn record_target_tls_reload_skipped(target: &str, reason: &str) {
counter!(TARGET_TLS_RELOAD_SKIPPED_TOTAL, "target_id" => target.to_string(), "reason" => reason.to_string()).increment(1);
}
/// Records a TLS reload publication failure.
pub fn record_target_tls_publication_fail(target: &str) {
counter!(TARGET_TLS_PUBLICATION_FAIL_TOTAL, "target_id" => target.to_string()).increment(1);
}
/// Records that a target used a stale TLS generation (active ≠ latest published).
pub fn record_target_tls_stale_generation(target: &str) {
counter!(TARGET_TLS_ACTIVE_GENERATION_MISMATCH_TOTAL, "target_id" => target.to_string()).increment(1);
}
crates/targets/src/runtime/tls/mod.rs
+41
View File
@@ -0,0 +1,41 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Unified TLS hot-reload infrastructure for notification targets.
//!
//! This module provides:
//! - Fingerprint-based change detection (`fingerprint`)
//! - Per-target reload configuration (`config`)
//! - Runtime state tracking with atomic timestamps (`state`)
//! - The `ReloadableTargetTls` trait protocol (`trait`)
//! - TLS material validation helpers (`validate`)
//! - The reload coordinator with background poll loops (`coordinator`)
//! - Target-level reload metrics (`metrics`)
pub mod adapter;
pub mod config;
pub mod coordinator;
pub mod fingerprint;
pub mod metrics;
pub mod state;
pub mod r#trait;
pub mod validate;
pub use adapter::TlsReloadAdapter;
pub use coordinator::TargetTlsReloadCoordinator;
pub use fingerprint::{TargetTlsFingerprint, TargetTlsGeneration, TargetTlsState, build_target_tls_fingerprint};
pub use metrics::init_target_tls_metrics;
pub use state::{TargetTlsInputSet, TargetTlsPublishedState, TargetTlsRuntimeState, TargetTlsStatusSnapshot};
pub use r#trait::ReloadableTargetTls;
pub use validate::validate_tls_material;
crates/targets/src/runtime/tls/state.rs
+127
View File
@@ -0,0 +1,127 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Per-target TLS reload runtime state with atomic timestamps and error tracking.
use super::fingerprint::{TargetTlsFingerprint, TargetTlsGeneration};
use ::arc_swap::ArcSwap;
use serde::Serialize;
use std::sync::Arc;
use std::sync::atomic::{AtomicU64, Ordering};
/// Describes which TLS files a target reads.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct TargetTlsInputSet {
pub ca_path: String,
pub client_cert_path: String,
pub client_key_path: String,
/// Human-readable label for logging and metrics (e.g. "webhook:primary").
pub target_label: String,
}
impl TargetTlsInputSet {
/// Returns `true` when no TLS paths are configured (no CA, cert, or key).
pub fn is_empty(&self) -> bool {
self.ca_path.is_empty() && self.client_cert_path.is_empty() && self.client_key_path.is_empty()
}
}
/// Immutable snapshot of a successfully published TLS material generation.
pub struct TargetTlsPublishedState<M> {
pub generation: TargetTlsGeneration,
pub fingerprint: TargetTlsFingerprint,
pub material: Arc<M>,
pub loaded_at_unix_ms: u64,
}
/// Per-target TLS reload runtime state. Owns the current published material
/// and tracks timestamps and the last error for observability.
pub struct TargetTlsRuntimeState<M> {
/// The currently active TLS material generation.
pub current: ArcSwap<TargetTlsPublishedState<M>>,
/// The last known-good generation (never overwritten by a failed reload).
pub last_good: ArcSwap<TargetTlsPublishedState<M>>,
/// Unix-millis timestamp of the last reload *attempt* (success or failure).
pub last_attempt_unix_ms: AtomicU64,
/// Unix-millis timestamp of the last *successful* reload.
pub last_success_unix_ms: AtomicU64,
/// Last reload error message, if any.
pub last_error: parking_lot::RwLock<Option<String>>,
/// The TLS file paths this state watches.
pub inputs: TargetTlsInputSet,
}
impl<M> TargetTlsRuntimeState<M> {
/// Creates a new runtime state with the given initial published state.
pub fn new(initial: Arc<TargetTlsPublishedState<M>>, inputs: TargetTlsInputSet) -> Self {
Self {
current: ArcSwap::from(initial.clone()),
last_good: ArcSwap::from(initial),
last_attempt_unix_ms: AtomicU64::new(0),
last_success_unix_ms: AtomicU64::new(0),
last_error: parking_lot::RwLock::new(None),
inputs,
}
}
/// Returns the generation of the currently active material.
pub fn current_generation(&self) -> TargetTlsGeneration {
self.current.load().generation
}
/// Atomically bumps and returns the next generation.
pub fn bump_generation(&self) -> TargetTlsGeneration {
// Load the current generation from the arc-swap, compute next,
// and return it. The caller is responsible for publishing the new state.
let current = self.current.load();
TargetTlsGeneration(current.generation.0.saturating_add(1))
}
/// Records the timestamp of a reload attempt.
pub fn mark_attempt(&self, unix_ms: u64) {
self.last_attempt_unix_ms.store(unix_ms, Ordering::Release);
}
/// Records the timestamp of a successful reload.
pub fn mark_success(&self, unix_ms: u64) {
self.last_success_unix_ms.store(unix_ms, Ordering::Release);
}
/// Returns the last attempt timestamp.
pub fn last_attempt_unix_ms(&self) -> u64 {
self.last_attempt_unix_ms.load(Ordering::Acquire)
}
/// Returns the last success timestamp.
pub fn last_success_unix_ms(&self) -> u64 {
self.last_success_unix_ms.load(Ordering::Acquire)
}
}
/// Read-only status snapshot for admin/debug visibility.
#[derive(Debug, Clone, Serialize)]
pub struct TargetTlsStatusSnapshot {
pub target_label: String,
pub generation: u64,
pub reload_enabled: bool,
pub detect_mode: &'static str,
pub apply_mode: &'static str,
pub last_attempt_time: Option<u64>,
pub last_success_time: Option<u64>,
pub last_error: Option<String>,
/// TLS file paths this target watches (for admin diagnostics).
pub ca_path: String,
pub client_cert_path: String,
pub client_key_path: String,
}
crates/targets/src/runtime/tls/trait.rs
+68
View File
@@ -0,0 +1,68 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! The `ReloadableTargetTls` trait — the public protocol each TLS-capable
//! target implements to participate in coordinated hot-reload.
use crate::error::TargetError;
use async_trait::async_trait;
use std::sync::Arc;
use super::config::ReloadApplyMode;
use super::fingerprint::TargetTlsGeneration;
use super::state::TargetTlsInputSet;
/// Protocol that each TLS-capable target implements so the reload coordinator
/// can drive certificate hot-reload without knowing the target's internals.
///
/// The target is responsible for:
/// - Declaring which TLS files it reads (`tls_input_set`)
/// - Building a new client/pool/connector from current files (`build_tls_material`)
/// - Atomically swapping the active connection state (`apply_tls_material`)
///
/// The coordinator is responsible for:
/// - Deciding *when* to check
/// - Detecting *whether* material changed
/// - Ensuring *safety* (validate, build-then-apply, fallback on failure)
#[async_trait]
pub trait ReloadableTargetTls: Send + Sync + 'static {
/// The rebuilt connection/client/pool object this target uses.
type Material: Send + Sync + 'static;
/// Returns the TLS file paths this target reads.
fn tls_input_set(&self) -> TargetTlsInputSet;
/// Build a fresh TLS material object from current files on disk.
///
/// Called by the coordinator on the reload path only — never on the send hot path.
async fn build_tls_material(&self) -> Result<Self::Material, TargetError>;
/// Atomically apply new TLS material, replacing the current active connection state.
///
/// On success, the target's internal state must point to the new material.
/// On failure, the target must keep its current state unchanged.
async fn apply_tls_material(
&self,
generation: TargetTlsGeneration,
material: Arc<Self::Material>,
mode: ReloadApplyMode,
) -> Result<(), TargetError>;
/// Optional pre-check: validate that TLS files on disk are self-consistent
/// (cert/key pair parseable, CA loadable) before attempting `build_tls_material`.
/// Default implementation returns `Ok(())`.
async fn validate_tls_files(&self) -> Result<(), TargetError> {
Ok(())
}
}
crates/targets/src/runtime/tls/validate.rs
+58
View File
@@ -0,0 +1,58 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! TLS material validation helpers used by the reload coordinator before
//! attempting to build new client/pool objects.
use crate::error::TargetError;
use rustfs_tls_runtime::{load_certs, load_private_key};
/// Validates that a client certificate and private key file can be loaded
/// and paired together. Returns `Ok(())` if both files parse successfully,
/// or `Ok(())` if both paths are empty (no mTLS configured).
pub fn validate_cert_key_pairing(cert_path: &str, key_path: &str) -> Result<(), TargetError> {
if cert_path.is_empty() && key_path.is_empty() {
return Ok(());
}
if cert_path.is_empty() || key_path.is_empty() {
return Err(TargetError::Configuration(
"Client certificate and key must both be specified or both be empty".to_string(),
));
}
load_certs(cert_path).map_err(|e| TargetError::Configuration(format!("Invalid client certificate '{cert_path}': {e}")))?;
load_private_key(key_path).map_err(|e| TargetError::Configuration(format!("Invalid client key '{key_path}': {e}")))?;
Ok(())
}
/// Validates that a CA certificate file can be loaded. Returns `Ok(())`
/// if the path is empty (no custom CA) or if the file parses successfully.
pub fn validate_ca_file(ca_path: &str) -> Result<(), TargetError> {
if ca_path.is_empty() {
return Ok(());
}
load_certs(ca_path).map_err(|e| TargetError::Configuration(format!("Invalid CA certificate '{ca_path}': {e}")))?;
Ok(())
}
/// Validates all three TLS material files in one call.
pub fn validate_tls_material(ca_path: &str, cert_path: &str, key_path: &str) -> Result<(), TargetError> {
validate_ca_file(ca_path)?;
validate_cert_key_pairing(cert_path, key_path)
}
crates/targets/src/target/amqp.rs
+104 -8
View File
@@ -22,11 +22,15 @@ use crate::{
StoreError, Target,
arn::TargetID,
error::TargetError,
runtime::tls::{
ReloadableTargetTls, TargetTlsGeneration, TargetTlsInputSet, TlsReloadAdapter, config::ReloadApplyMode,
validate_tls_material,
},
store::{Key, Store},
target::{
ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
TargetType, build_queued_payload_with_records, is_connectivity_error, open_target_queue_store,
persist_queued_payload_to_store,
TargetTlsState, TargetType, build_queued_payload_with_records, build_target_tls_fingerprint, is_connectivity_error,
open_target_queue_store, persist_queued_payload_to_store,
},
};
use async_trait::async_trait;
@@ -37,6 +41,7 @@ use lapin::{
};
use parking_lot::Mutex;
use rustfs_config::{AMQP_TLS_CA, AMQP_TLS_CLIENT_CERT, AMQP_TLS_CLIENT_KEY};
use rustfs_tls_runtime::load_cert_bundle_der_bytes;
use serde::Serialize;
use serde::de::DeserializeOwned;
use std::fmt;
@@ -196,16 +201,24 @@ async fn build_tls_config(args: &AMQPArgs) -> Result<OwnedTLSConfig, TargetError
let cert_chain = if args.tls_ca.is_empty() {
None
} else {
Some(
tokio::fs::read_to_string(&args.tls_ca)
.await
.map_err(|e| TargetError::Configuration(format!("Failed to read {AMQP_TLS_CA}: {e}")))?,
)
let certs_der = load_cert_bundle_der_bytes(&args.tls_ca)
.map_err(|e| TargetError::Configuration(format!("Failed to parse {AMQP_TLS_CA}: {e}")))?;
if certs_der.is_empty() {
return Err(TargetError::Configuration(format!(
"{AMQP_TLS_CA} did not contain any parsable certificates"
)));
}
let pem = tokio::fs::read_to_string(&args.tls_ca)
.await
.map_err(|e| TargetError::Configuration(format!("Failed to read {AMQP_TLS_CA}: {e}")))?;
Some(pem)
};
let identity = if args.tls_client_cert.is_empty() {
None
} else {
let _ = load_cert_bundle_der_bytes(&args.tls_client_cert)
.map_err(|e| TargetError::Configuration(format!("Failed to parse {AMQP_TLS_CLIENT_CERT}: {e}")))?;
let pem = tokio::fs::read(&args.tls_client_cert)
.await
.map_err(|e| TargetError::Configuration(format!("Failed to read {AMQP_TLS_CLIENT_CERT}: {e}")))?;
@@ -290,6 +303,9 @@ where
id: TargetID,
args: AMQPArgs,
connection: Arc<Mutex<Option<Arc<AMQPConnection>>>>,
tls_state: Arc<Mutex<TargetTlsState>>,
/// When set, the coordinator drives TLS reload; inline fingerprint check is skipped.
tls_adapter: Option<TlsReloadAdapter<AMQPConnection>>,
connect_lock: Arc<AsyncMutex<()>>,
store: Option<Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync>>,
delivery_counters: Arc<TargetDeliveryCounters>,
@@ -305,6 +321,8 @@ where
id: self.id.clone(),
args: self.args.clone(),
connection: Arc::clone(&self.connection),
tls_state: Arc::clone(&self.tls_state),
tls_adapter: self.tls_adapter.clone(),
connect_lock: Arc::clone(&self.connect_lock),
store: self.store.as_ref().map(|s| s.boxed_clone()),
delivery_counters: Arc::clone(&self.delivery_counters),
@@ -329,6 +347,8 @@ where
id: target_id,
args,
connection: Arc::new(Mutex::new(None)),
tls_state: Arc::new(Mutex::new(TargetTlsState::default())),
tls_adapter: None,
connect_lock: Arc::new(AsyncMutex::new(())),
store: queue_store,
delivery_counters: Arc::new(TargetDeliveryCounters::default()),
@@ -341,6 +361,27 @@ where
}
async fn get_or_connect(&self) -> Result<Arc<AMQPConnection>, TargetError> {
// When a TLS reload adapter is attached, it drives connection rebuilds
// in the background. The inline per-send fingerprint check is skipped.
if let Some(adapter) = &self.tls_adapter {
let material = adapter.current_material();
if material.connection.status().connected() && material.channel.status().connected() {
return Ok(material);
}
self.clear_connection_handle();
} else {
let next_fingerprint =
build_target_tls_fingerprint(&self.args.tls_ca, &self.args.tls_client_cert, &self.args.tls_client_key).await?;
let tls_changed = {
let tls_state_guard = self.tls_state.lock();
tls_state_guard.needs_update(&next_fingerprint)
};
if tls_changed {
self.clear_connection_handle();
self.tls_state.lock().refresh(next_fingerprint);
}
}
if let Some(connection) = self.connection.lock().clone()
&& connection.connection.status().connected()
&& connection.channel.status().connected()
@@ -362,10 +403,19 @@ where
Ok(connection)
}
fn clear_connection(&self) {
fn clear_connection_handle(&self) {
*self.connection.lock() = None;
}
fn clear_connection_cache(&self) {
self.clear_connection_handle();
self.tls_state.lock().reset();
}
fn clear_connection(&self) {
self.clear_connection_cache();
}
async fn send_body(&self, body: &[u8]) -> Result<(), TargetError> {
let connection = self.get_or_connect().await?;
let publish = connection
@@ -407,6 +457,46 @@ where
}
}
/// Coordinated TLS hot-reload implementation for AMQP targets.
///
/// The coordinator calls these methods on a background poll loop to detect
/// TLS file changes and rebuild the connection without restarting.
#[async_trait]
impl<E> ReloadableTargetTls for AMQPTarget<E>
where
E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
{
type Material = AMQPConnection;
fn tls_input_set(&self) -> TargetTlsInputSet {
TargetTlsInputSet {
ca_path: self.args.tls_ca.clone(),
client_cert_path: self.args.tls_client_cert.clone(),
client_key_path: self.args.tls_client_key.clone(),
target_label: format!("amqp:{}", self.id.id),
}
}
async fn build_tls_material(&self) -> Result<Self::Material, TargetError> {
connect_amqp(&self.args).await
}
async fn apply_tls_material(
&self,
_generation: TargetTlsGeneration,
material: Arc<Self::Material>,
_mode: ReloadApplyMode,
) -> Result<(), TargetError> {
let mut guard = self.connection.lock();
*guard = Some(material);
Ok(())
}
async fn validate_tls_files(&self) -> Result<(), TargetError> {
validate_tls_material(&self.args.tls_ca, &self.args.tls_client_cert, &self.args.tls_client_key)
}
}
#[async_trait]
impl<E> Target<E> for AMQPTarget<E>
where
@@ -458,6 +548,12 @@ where
.await
.map_err(|e| map_lapin_error(e, "Failed to close AMQP connection"))?;
}
self.tls_state.lock().reset();
// If a TLS reload adapter is attached, reset its error tracking
// so that a future re-init does not inherit stale failure state.
if let Some(adapter) = &self.tls_adapter {
*adapter.runtime_state().last_error.write() = None;
}
info!(target_id = %self.id, "AMQP target closed");
Ok(())
}
crates/targets/src/target/kafka.rs
+108 -2
View File
@@ -16,16 +16,21 @@ use crate::{
StoreError, Target,
arn::TargetID,
error::TargetError,
runtime::tls::{
ReloadableTargetTls, TargetTlsInputSet, TlsReloadAdapter, config::ReloadApplyMode, fingerprint::TargetTlsGeneration,
validate::validate_tls_material,
},
store::{Key, Store},
target::{
ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
TargetType, build_queued_payload, invalidate_cache_on_connectivity_error, open_target_queue_store,
persist_queued_payload_to_store,
TargetTlsState, TargetType, build_queued_payload, build_target_tls_fingerprint, invalidate_cache_on_connectivity_error,
open_target_queue_store, persist_queued_payload_to_store,
},
};
use async_trait::async_trait;
use rustfs_kafka_async::error::{ConnectionError, Error as KafkaError};
use rustfs_kafka_async::{AsyncProducer, AsyncProducerConfig, Record, RequiredAcks, SecurityConfig};
use rustfs_tls_runtime::{load_cert_bundle_der_bytes, load_private_key};
use serde::Serialize;
use serde::de::DeserializeOwned;
use std::{marker::PhantomData, sync::Arc, time::Duration};
@@ -104,6 +109,11 @@ where
args: KafkaArgs,
store: Option<Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync>>,
producer: Arc<Mutex<Option<Arc<AsyncProducer>>>>,
tls_state: Arc<Mutex<TargetTlsState>>,
/// Adapter that bridges this target to the TLS reload coordinator.
/// When `Some`, the target uses coordinator-managed material; when `None`,
/// it falls back to inline fingerprint-based change detection.
tls_adapter: Option<TlsReloadAdapter<Arc<AsyncProducer>>>,
delivery_counters: Arc<TargetDeliveryCounters>,
_phantom: PhantomData<E>,
}
@@ -144,6 +154,8 @@ where
args,
store: queue_store,
producer: Arc::new(Mutex::new(None)),
tls_state: Arc::new(Mutex::new(TargetTlsState::default())),
tls_adapter: None,
delivery_counters: Arc::new(TargetDeliveryCounters::default()),
_phantom: PhantomData,
})
@@ -164,9 +176,27 @@ where
if self.args.tls_enable {
let mut security = SecurityConfig::new();
if !self.args.tls_ca.is_empty() {
let certs = load_cert_bundle_der_bytes(&self.args.tls_ca)
.map_err(|e| Self::map_kafka_error(KafkaError::Config(e.to_string()), "Failed to parse Kafka tls_ca"))?;
if certs.is_empty() {
return Err(TargetError::Configuration(
"Kafka tls_ca did not contain any parsable certificates".to_string(),
));
}
security = security.with_ca_cert(self.args.tls_ca.clone());
}
if !self.args.tls_client_cert.is_empty() && !self.args.tls_client_key.is_empty() {
let certs = load_cert_bundle_der_bytes(&self.args.tls_client_cert).map_err(|e| {
Self::map_kafka_error(KafkaError::Config(e.to_string()), "Failed to parse Kafka tls_client_cert")
})?;
if certs.is_empty() {
return Err(TargetError::Configuration(
"Kafka tls_client_cert did not contain any parsable certificates".to_string(),
));
}
let _ = load_private_key(&self.args.tls_client_key).map_err(|e| {
Self::map_kafka_error(KafkaError::Config(e.to_string()), "Failed to parse Kafka tls_client_key")
})?;
security = security.with_client_cert(self.args.tls_client_cert.clone(), self.args.tls_client_key.clone());
}
config = config.with_security(security);
@@ -178,6 +208,31 @@ where
}
async fn get_or_build_producer(&self) -> Result<Arc<AsyncProducer>, TargetError> {
// Adapter-managed path: use the material directly from the TLS reload adapter.
if let Some(adapter) = &self.tls_adapter {
let producer: Arc<AsyncProducer> = (*adapter.current_material()).clone();
// Ensure the producer is also stored locally so that close() can drain it.
{
let mut guard = self.producer.lock().await;
*guard = Some(Arc::clone(&producer));
}
return Ok(producer);
}
// Inline fingerprint fallback path (no coordinator).
let next_fingerprint =
build_target_tls_fingerprint(&self.args.tls_ca, &self.args.tls_client_cert, &self.args.tls_client_key).await?;
let tls_changed = {
let tls_state_guard = self.tls_state.lock().await;
tls_state_guard.needs_update(&next_fingerprint)
};
if tls_changed {
let mut cached = self.producer.lock().await;
*cached = None;
self.tls_state.lock().await.refresh(next_fingerprint);
}
let mut cached = self.producer.lock().await;
if let Some(producer) = cached.as_ref() {
return Ok(Arc::clone(producer));
@@ -191,6 +246,7 @@ where
async fn invalidate_cached_producer(&self) {
let mut cached = self.producer.lock().await;
*cached = None;
self.tls_state.lock().await.reset();
}
/// Serializes the event and builds a QueuedPayload
@@ -230,6 +286,8 @@ where
args: self.args.clone(),
store: self.store.as_ref().map(|s| s.boxed_clone()),
producer: Arc::clone(&self.producer),
tls_state: Arc::clone(&self.tls_state),
tls_adapter: self.tls_adapter.clone(),
delivery_counters: Arc::clone(&self.delivery_counters),
_phantom: PhantomData,
})
@@ -292,6 +350,13 @@ where
}
async fn close(&self) -> Result<(), TargetError> {
{
let mut guard = self.producer.lock().await;
*guard = None;
}
self.tls_state.lock().await.reset();
info!("Kafka target closed: {}", self.id);
Ok(())
}
@@ -318,6 +383,47 @@ where
}
}
/// Coordinated TLS hot-reload implementation for Kafka targets.
///
/// The coordinator calls these methods on a background poll loop to detect
/// TLS file changes and rebuild the producer without restarting.
#[async_trait]
impl<E> ReloadableTargetTls for KafkaTarget<E>
where
E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
{
type Material = Arc<AsyncProducer>;
fn tls_input_set(&self) -> TargetTlsInputSet {
TargetTlsInputSet {
ca_path: self.args.tls_ca.clone(),
client_cert_path: self.args.tls_client_cert.clone(),
client_key_path: self.args.tls_client_key.clone(),
target_label: format!("kafka:{}", self.id.id),
}
}
async fn build_tls_material(&self) -> Result<Self::Material, TargetError> {
let producer = self.build_producer().await?;
Ok(Arc::new(producer))
}
async fn apply_tls_material(
&self,
_generation: TargetTlsGeneration,
material: Arc<Self::Material>,
_mode: ReloadApplyMode,
) -> Result<(), TargetError> {
let mut guard = self.producer.lock().await;
*guard = Some((*material).clone());
Ok(())
}
async fn validate_tls_files(&self) -> Result<(), TargetError> {
validate_tls_material(&self.args.tls_ca, &self.args.tls_client_cert, &self.args.tls_client_key)
}
}
#[cfg(test)]
mod tests {
use super::*;
crates/targets/src/target/mod.rs
+49
View File
@@ -37,6 +37,13 @@ pub mod pulsar;
pub mod redis;
pub mod webhook;
#[cfg(test)]
pub(crate) use crate::runtime::tls::fingerprint::TargetTlsFingerprint as TargetTlsFingerprintState;
#[cfg(test)]
pub(crate) use crate::runtime::tls::fingerprint::TargetTlsGeneration;
pub(crate) use crate::runtime::tls::fingerprint::TargetTlsState;
pub(crate) use crate::runtime::tls::fingerprint::build_target_tls_fingerprint;
/// A read-only snapshot of delivery counters for a target.
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct TargetDeliverySnapshot {
@@ -531,6 +538,48 @@ pub(crate) fn ensure_rustls_provider_installed() {
}
}
#[cfg(test)]
mod tls_state_tests {
use super::{TargetTlsFingerprintState, TargetTlsGeneration, TargetTlsState};
#[test]
fn refresh_increments_generation_only_when_fingerprint_changes() {
let mut state = TargetTlsState::default();
let first = TargetTlsFingerprintState {
ca_sha256: Some([1; 32]),
client_cert_sha256: None,
client_key_sha256: None,
};
let second = TargetTlsFingerprintState {
ca_sha256: Some([2; 32]),
client_cert_sha256: None,
client_key_sha256: None,
};
assert!(state.refresh(first.clone()));
assert_eq!(state.generation, TargetTlsGeneration(1));
assert!(!state.refresh(first));
assert_eq!(state.generation, TargetTlsGeneration(1));
assert!(state.refresh(second));
assert_eq!(state.generation, TargetTlsGeneration(2));
}
#[test]
fn reset_clears_generation_and_fingerprint() {
let mut state = TargetTlsState {
generation: TargetTlsGeneration(5),
fingerprint: Some(TargetTlsFingerprintState {
ca_sha256: Some([9; 32]),
client_cert_sha256: None,
client_key_sha256: None,
}),
};
state.reset();
assert_eq!(state, TargetTlsState::default());
}
}
#[cfg(test)]
mod tests {
use super::*;
crates/targets/src/target/mqtt.rs
+96 -15
View File
@@ -16,6 +16,10 @@ use crate::{
StoreError, Target,
arn::TargetID,
error::TargetError,
runtime::tls::{
ReloadableTargetTls, TargetTlsGeneration, TargetTlsInputSet, TargetTlsState, TlsReloadAdapter, config::ReloadApplyMode,
validate_tls_material,
},
store::{Key, Store},
target::{
ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
@@ -23,6 +27,7 @@ use crate::{
persist_queued_payload_to_store,
},
};
use arc_swap::ArcSwap;
use async_trait::async_trait;
use hyper_rustls::ConfigBuilderExt;
use rumqttc::{
@@ -32,6 +37,7 @@ use rumqttc::{
use rustfs_config::{
EnableState, MQTT_TLS_CA, MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY, MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_WS_PATH_ALLOWLIST,
};
use rustfs_tls_runtime::{load_certs, load_private_key};
use rustls::ClientConfig;
use serde::Serialize;
use serde::de::DeserializeOwned;
@@ -185,8 +191,7 @@ fn validate_path_is_absolute(path: &str, field: &str) -> Result<(), TargetError>
}
fn build_root_store(ca_path: &str, trust_leaf_as_ca: bool) -> Result<rustls::RootCertStore, TargetError> {
let certs =
rustfs_utils::load_certs(ca_path).map_err(|e| TargetError::Configuration(format!("Failed to load MQTT tls_ca: {e}")))?;
let certs = load_certs(ca_path).map_err(|e| TargetError::Configuration(format!("Failed to load MQTT tls_ca: {e}")))?;
let mut store = rustls::RootCertStore::empty();
if trust_leaf_as_ca {
@@ -222,9 +227,9 @@ fn build_mqtt_tls_transport(broker: &Url, tls: &MQTTTlsConfig) -> Result<Transpo
if tls.client_cert_path.is_empty() {
builder.with_no_client_auth()
} else {
let certs = rustfs_utils::load_certs(&tls.client_cert_path)
let certs = load_certs(&tls.client_cert_path)
.map_err(|e| TargetError::Configuration(format!("Failed to load MQTT tls_client_cert: {e}")))?;
let key = rustfs_utils::load_private_key(&tls.client_key_path)
let key = load_private_key(&tls.client_key_path)
.map_err(|e| TargetError::Configuration(format!("Failed to load MQTT tls_client_key: {e}")))?;
builder
.with_client_auth_cert(certs, key)
@@ -237,9 +242,9 @@ fn build_mqtt_tls_transport(broker: &Url, tls: &MQTTTlsConfig) -> Result<Transpo
if tls.client_cert_path.is_empty() {
builder.with_no_client_auth()
} else {
let certs = rustfs_utils::load_certs(&tls.client_cert_path)
let certs = load_certs(&tls.client_cert_path)
.map_err(|e| TargetError::Configuration(format!("Failed to load MQTT tls_client_cert: {e}")))?;
let key = rustfs_utils::load_private_key(&tls.client_key_path)
let key = load_private_key(&tls.client_key_path)
.map_err(|e| TargetError::Configuration(format!("Failed to load MQTT tls_client_key: {e}")))?;
builder
.with_client_auth_cert(certs, key)
@@ -490,6 +495,12 @@ where
store: Option<Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync>>,
connected: Arc<AtomicBool>,
bg_task_manager: Arc<BgTaskManager>,
/// TLS fingerprint tracking for inline fallback path.
tls_state: Arc<parking_lot::Mutex<TargetTlsState>>,
/// When set, the coordinator drives TLS reload; inline fingerprint check is skipped.
tls_adapter: Option<TlsReloadAdapter<MqttOptions>>,
/// Updated MqttOptions from coordinator for use on next reconnection.
pending_mqtt_options: Arc<ArcSwap<MqttOptions>>,
delivery_counters: Arc<TargetDeliveryCounters>,
_phantom: PhantomData<E>,
}
@@ -519,6 +530,17 @@ where
initial_cancel_rx: Mutex::new(Some(cancel_rx)),
});
// Build the initial MqttOptions for TLS reload support.
let initial_mqtt_options = build_mqtt_options(
format!("rustfs_notify_{}", uuid::Uuid::new_v4()),
&args.broker,
Some(args.username.as_str()),
Some(args.password.as_str()),
&args.tls,
args.keep_alive,
Some(MAX_MQTT_PACKET_SIZE_BYTES),
)?;
info!(target_id = %target_id, "MQTT target created");
Ok(MQTTTarget::<E> {
id: target_id,
@@ -527,6 +549,9 @@ where
store: queue_store,
connected: Arc::new(AtomicBool::new(false)),
bg_task_manager,
tls_state: Arc::new(parking_lot::Mutex::new(TargetTlsState::default())),
tls_adapter: None,
pending_mqtt_options: Arc::new(ArcSwap::from(Arc::new(initial_mqtt_options))),
delivery_counters: Arc::new(TargetDeliveryCounters::default()),
_phantom: PhantomData,
})
@@ -544,20 +569,15 @@ where
let connected_arc = Arc::clone(&self.connected);
let target_id_clone = self.id.clone();
let args_clone = self.args.clone();
let pending_mqtt_options = Arc::clone(&self.pending_mqtt_options);
let _ = bg_task_manager
.init_cell
.get_or_try_init(|| async {
debug!(target_id = %target_id_clone, "Initializing MQTT background task.");
let mqtt_options = build_mqtt_options(
format!("rustfs_notify_{}", uuid::Uuid::new_v4()),
&args_clone.broker,
Some(args_clone.username.as_str()),
Some(args_clone.password.as_str()),
&args_clone.tls,
args_clone.keep_alive,
Some(MAX_MQTT_PACKET_SIZE_BYTES),
)?;
// Use the latest MqttOptions (may have been updated by TLS reload coordinator).
let mqtt_options: MqttOptions = (**pending_mqtt_options.load()).clone();
let (new_client, eventloop) = AsyncClient::builder(mqtt_options).capacity(10).build();
@@ -662,12 +682,66 @@ where
store: self.store.as_ref().map(|s| s.boxed_clone()),
connected: self.connected.clone(),
bg_task_manager: self.bg_task_manager.clone(),
tls_state: Arc::clone(&self.tls_state),
tls_adapter: self.tls_adapter.clone(),
pending_mqtt_options: Arc::clone(&self.pending_mqtt_options),
delivery_counters: self.delivery_counters.clone(),
_phantom: PhantomData,
})
}
}
/// Coordinated TLS hot-reload implementation for MQTT targets.
///
/// MQTT uses `MqttOptions` as the material type. The coordinator rebuilds
/// `MqttOptions` on TLS file changes, and `apply_tls_material` stores it in
/// an `ArcSwap` for use on the next reconnection. The running event loop is
/// not interrupted; rumqttc handles reconnection internally.
#[async_trait]
impl<E> ReloadableTargetTls for MQTTTarget<E>
where
E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
{
type Material = MqttOptions;
fn tls_input_set(&self) -> TargetTlsInputSet {
TargetTlsInputSet {
ca_path: self.args.tls.ca_path.clone(),
client_cert_path: self.args.tls.client_cert_path.clone(),
client_key_path: self.args.tls.client_key_path.clone(),
target_label: format!("mqtt:{}", self.id.id),
}
}
async fn build_tls_material(&self) -> Result<Self::Material, TargetError> {
build_mqtt_options(
format!("rustfs_notify_{}", uuid::Uuid::new_v4()),
&self.args.broker,
Some(self.args.username.as_str()),
Some(self.args.password.as_str()),
&self.args.tls,
self.args.keep_alive,
Some(MAX_MQTT_PACKET_SIZE_BYTES),
)
}
async fn apply_tls_material(
&self,
_generation: TargetTlsGeneration,
material: Arc<Self::Material>,
_mode: ReloadApplyMode,
) -> Result<(), TargetError> {
// Store the new MqttOptions for use on next reconnection.
// The running event loop is not interrupted; rumqttc handles reconnection.
self.pending_mqtt_options.store(material);
Ok(())
}
async fn validate_tls_files(&self) -> Result<(), TargetError> {
validate_tls_material(&self.args.tls.ca_path, &self.args.tls.client_cert_path, &self.args.tls.client_key_path)
}
}
async fn run_mqtt_event_loop(
mut eventloop: EventLoop,
connected_status: Arc<AtomicBool>,
@@ -968,6 +1042,13 @@ where
}
}
self.tls_state.lock().reset();
// If a TLS reload adapter is attached, reset its error tracking
// so that a future re-init does not inherit stale failure state.
if let Some(adapter) = &self.tls_adapter {
*adapter.runtime_state().last_error.write() = None;
}
self.connected.store(false, Ordering::SeqCst);
info!(target_id = %self.id, "MQTT target close method finished.");
Ok(())
crates/targets/src/target/mysql.rs
+162 -62
View File
@@ -16,6 +16,10 @@ use crate::{
StoreError, Target,
arn::TargetID,
error::TargetError,
runtime::tls::{
ReloadableTargetTls, TargetTlsInputSet, TlsReloadAdapter, config::ReloadApplyMode, fingerprint::TargetTlsGeneration,
validate::validate_tls_material,
},
store::{Key, Store},
target::{
ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
@@ -26,6 +30,7 @@ use crate::{
use async_trait::async_trait;
use mysql_async::{Conn, Opts, OptsBuilder, Pool, PoolConstraints, PoolOpts, SslOpts, prelude::Queryable};
use rustfs_config::{MYSQL_TLS_CA, MYSQL_TLS_CLIENT_CERT, MYSQL_TLS_CLIENT_KEY};
use rustfs_tls_runtime::{load_certs, load_private_key};
use serde::Serialize;
use serde::de::DeserializeOwned;
use std::marker::PhantomData;
@@ -478,6 +483,11 @@ where
store: Option<Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync>>,
/// Lazily-initialized MySQL connection pool
pool: Arc<Mutex<Option<Pool>>>,
/// TLS fingerprint tracking for hot reload (inline fallback path)
tls_state: Arc<parking_lot::Mutex<super::TargetTlsState>>,
/// When present, the adapter provides coordinator-managed TLS material;
/// otherwise the inline fingerprint path is used as a fallback.
tls_adapter: Option<TlsReloadAdapter<Pool>>,
/// Success/failure counters exposed via `delivery_snapshot`
delivery_counters: Arc<TargetDeliveryCounters>,
/// Zero-sized marker for the event type `E`
@@ -489,6 +499,9 @@ where
E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
{
/// Creates a new MySqlTarget.
///
/// The target starts without a TLS reload coordinator. Use
/// `TlsReloadAdapter::try_register` to opt into coordinated TLS hot-reload.
pub fn new(id: String, args: MySqlArgs) -> Result<Self, TargetError> {
args.validate()?;
@@ -511,6 +524,8 @@ where
store: queue_store,
// Pool is lazily initialized on first use to avoid unnecessary connections at startup and allow for better error handling
pool: Arc::new(Mutex::new(None)),
tls_state: Arc::new(parking_lot::Mutex::new(super::TargetTlsState::default())),
tls_adapter: None,
delivery_counters: Arc::new(TargetDeliveryCounters::default()),
_phantom: PhantomData,
})
@@ -518,6 +533,10 @@ where
/// Returns or lazily initializes the MySQL connection pool.
///
/// When `tls_adapter` is present (coordinator-managed), the pool
/// is sourced from the coordinator's published material.
/// Otherwise, the inline fingerprint-based path is used as a fallback.
///
/// # Errors
///
/// | Scenario | Error variant |
@@ -528,6 +547,31 @@ where
/// | Existing table has incompatible schema | `Initialization` |
/// | DSN parse failure / invalid config | `Configuration` |
async fn get_or_init_pool(&self) -> Result<Pool, TargetError> {
// Adapter-managed path: use the material directly from the coordinator.
if let Some(adapter) = &self.tls_adapter {
let pool: Pool = (*adapter.current_material()).clone();
// Ensure the pool is also stored locally so that close() can drain it.
{
let mut guard = self.pool.lock().await;
*guard = Some(pool.clone());
}
return Ok(pool);
}
// Inline fingerprint fallback path (no coordinator).
let next_fingerprint =
super::build_target_tls_fingerprint(&self.args.tls_ca, &self.args.tls_client_cert, &self.args.tls_client_key).await?;
let tls_changed = {
let tls_state_guard = self.tls_state.lock();
tls_state_guard.needs_update(&next_fingerprint)
};
if tls_changed {
let mut guard = self.pool.lock().await;
*guard = None;
self.tls_state.lock().refresh(next_fingerprint);
}
{
let guard = self.pool.lock().await;
if let Some(pool) = guard.as_ref() {
@@ -535,68 +579,7 @@ where
}
}
let dsn = MySqlDsn::parse(&self.args.dsn_string)?;
let mut builder = OptsBuilder::default()
.user(Some(dsn.user.clone()))
.pass(Some(dsn.password.clone()))
.ip_or_hostname(dsn.host.clone())
.tcp_port(dsn.port)
.db_name(Some(dsn.database.clone()));
if dsn.tls {
super::ensure_rustls_provider_installed();
let mut ssl_opts = SslOpts::default();
if !self.args.tls_ca.is_empty() {
ssl_opts = ssl_opts.with_root_certs(vec![PathBuf::from(self.args.tls_ca.clone()).into()]);
}
if !self.args.tls_client_cert.is_empty() && !self.args.tls_client_key.is_empty() {
let identity = mysql_async::ClientIdentity::new(
PathBuf::from(self.args.tls_client_cert.clone()).into(),
PathBuf::from(self.args.tls_client_key.clone()).into(),
);
ssl_opts = ssl_opts.with_client_identity(Some(identity));
}
builder = builder.ssl_opts(Some(ssl_opts));
} else {
warn!(
"MySQL target '{}' is configured without TLS. This is insecure and should not be used in production.",
self.id
);
}
// When max_open_connections is 0, no explicit upper bound is set —
// mysql_async uses its default pool constraints (10–100).
if self.args.max_open_connections > 0 {
let constraints = PoolConstraints::new(1, self.args.max_open_connections).ok_or_else(|| {
TargetError::Configuration(format!(
"MySQL max_open_connections must be >= 1, got {}",
self.args.max_open_connections
))
})?;
builder = builder.pool_opts(PoolOpts::default().with_constraints(constraints));
}
let opts = Opts::from(builder);
let pool = Pool::new(opts);
// Uses a double-check pattern: the mutex guard is only held for
// short reads/writes to the pool cache. All I/O (connecting,
// DDL, schema validation) happens outside the lock so that
// concurrent callers are not blocked by a slow MySQL server.
let mut conn = pool.get_conn().await.map_err(|_| TargetError::NotConnected)?;
conn.query_drop("SELECT 1").await.map_err(|_| TargetError::NotConnected)?;
let ddl = format!(
"CREATE TABLE IF NOT EXISTS {} (event_time DATETIME(6) NOT NULL, event_data JSON NOT NULL)",
quote_table_name(&self.args.table)?
);
conn.query_drop(ddl)
.await
.map_err(|e| TargetError::Initialization(format!("Failed to create MySQL table: {e}")))?;
validate_existing_schema(&mut conn, &self.args.table).await?;
let pool = build_mysql_pool_from_args(&self.args).await?;
// Double-check: another caller may have initialized the pool
// while we were doing I/O.
@@ -653,12 +636,87 @@ where
args: self.args.clone(),
store: self.store.as_ref().map(|s| s.boxed_clone()),
pool: Arc::clone(&self.pool),
tls_state: Arc::clone(&self.tls_state),
tls_adapter: self.tls_adapter.clone(),
delivery_counters: Arc::clone(&self.delivery_counters),
_phantom: PhantomData,
})
}
}
/// Builds a MySQL connection pool from the given args, including TLS setup,
/// DDL table creation, and schema validation.
///
/// This is a standalone function so it can be called both from
/// `get_or_init_pool` (inline fallback) and from `build_tls_material`
/// (coordinator path).
async fn build_mysql_pool_from_args(args: &MySqlArgs) -> Result<Pool, TargetError> {
let dsn = MySqlDsn::parse(&args.dsn_string)?;
let mut builder = OptsBuilder::default()
.user(Some(dsn.user.clone()))
.pass(Some(dsn.password.clone()))
.ip_or_hostname(dsn.host.clone())
.tcp_port(dsn.port)
.db_name(Some(dsn.database.clone()));
if dsn.tls {
super::ensure_rustls_provider_installed();
let mut ssl_opts = SslOpts::default();
if !args.tls_ca.is_empty() {
let _ =
load_certs(&args.tls_ca).map_err(|e| TargetError::Configuration(format!("Failed to load MySQL tls_ca: {e}")))?;
ssl_opts = ssl_opts.with_root_certs(vec![PathBuf::from(args.tls_ca.clone()).into()]);
}
if !args.tls_client_cert.is_empty() && !args.tls_client_key.is_empty() {
let _ = load_certs(&args.tls_client_cert)
.map_err(|e| TargetError::Configuration(format!("Failed to load MySQL tls_client_cert: {e}")))?;
let _ = load_private_key(&args.tls_client_key)
.map_err(|e| TargetError::Configuration(format!("Failed to load MySQL tls_client_key: {e}")))?;
let identity = mysql_async::ClientIdentity::new(
PathBuf::from(args.tls_client_cert.clone()).into(),
PathBuf::from(args.tls_client_key.clone()).into(),
);
ssl_opts = ssl_opts.with_client_identity(Some(identity));
}
builder = builder.ssl_opts(Some(ssl_opts));
} else {
warn!("MySQL target is configured without TLS. This is insecure and should not be used in production.");
}
// When max_open_connections is 0, no explicit upper bound is set —
// mysql_async uses its default pool constraints (10–100).
if args.max_open_connections > 0 {
let constraints = PoolConstraints::new(1, args.max_open_connections).ok_or_else(|| {
TargetError::Configuration(format!("MySQL max_open_connections must be >= 1, got {}", args.max_open_connections))
})?;
builder = builder.pool_opts(PoolOpts::default().with_constraints(constraints));
}
let opts = Opts::from(builder);
let pool = Pool::new(opts);
// Uses a double-check pattern: the mutex guard is only held for
// short reads/writes to the pool cache. All I/O (connecting,
// DDL, schema validation) happens outside the lock so that
// concurrent callers are not blocked by a slow MySQL server.
let mut conn = pool.get_conn().await.map_err(|_| TargetError::NotConnected)?;
conn.query_drop("SELECT 1").await.map_err(|_| TargetError::NotConnected)?;
let ddl = format!(
"CREATE TABLE IF NOT EXISTS {} (event_time DATETIME(6) NOT NULL, event_data JSON NOT NULL)",
quote_table_name(&args.table)?
);
conn.query_drop(ddl)
.await
.map_err(|e| TargetError::Initialization(format!("Failed to create MySQL table: {e}")))?;
validate_existing_schema(&mut conn, &args.table).await?;
Ok(pool)
}
/// Maps a mysql_async error to `TargetError`:
/// - `Io`/`Driver` → `NotConnected` (connection lost, fixed-delay retry)
/// - `Server(1213|1205|1040)` → `Timeout` (deadlock/lock timeout/too
@@ -793,6 +851,8 @@ where
.map_err(|err| TargetError::Network(format!("Failed to disconnect MySQL pool: {err}")))?;
}
// Adapter cleanup is done by the coordinator; no local state to reset.
info!("MySQL target closed: {}", self.id);
Ok(())
}
@@ -828,6 +888,46 @@ where
}
}
/// Coordinated TLS hot-reload implementation for MySQL targets.
///
/// The coordinator calls these methods on a background poll loop to detect
/// TLS file changes and rebuild the connection pool without restarting.
#[async_trait]
impl<E> ReloadableTargetTls for MySqlTarget<E>
where
E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
{
type Material = Pool;
fn tls_input_set(&self) -> TargetTlsInputSet {
TargetTlsInputSet {
ca_path: self.args.tls_ca.clone(),
client_cert_path: self.args.tls_client_cert.clone(),
client_key_path: self.args.tls_client_key.clone(),
target_label: format!("mysql:{}", self.id.id),
}
}
async fn build_tls_material(&self) -> Result<Self::Material, TargetError> {
build_mysql_pool_from_args(&self.args).await
}
async fn apply_tls_material(
&self,
_generation: TargetTlsGeneration,
material: Arc<Self::Material>,
_mode: ReloadApplyMode,
) -> Result<(), TargetError> {
let mut guard = self.pool.lock().await;
*guard = Some((*material).clone());
Ok(())
}
async fn validate_tls_files(&self) -> Result<(), TargetError> {
validate_tls_material(&self.args.tls_ca, &self.args.tls_client_cert, &self.args.tls_client_key)
}
}
#[cfg(test)]
mod tests {
use super::*;
crates/targets/src/target/nats.rs
+143 -9
View File
@@ -16,10 +16,15 @@ use crate::{
StoreError, Target,
arn::TargetID,
error::TargetError,
runtime::tls::{
ReloadableTargetTls, TargetTlsGeneration, TargetTlsInputSet, TlsReloadAdapter, config::ReloadApplyMode,
validate_tls_material,
},
store::{Key, Store},
target::{
ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
TargetType, build_queued_payload_with_records, open_target_queue_store, persist_queued_payload_to_store,
TargetTlsState, TargetType, build_queued_payload_with_records, build_target_tls_fingerprint, open_target_queue_store,
persist_queued_payload_to_store,
},
};
use async_trait::async_trait;
@@ -28,9 +33,10 @@ use serde::Serialize;
use serde::de::DeserializeOwned;
use std::path::{Path, PathBuf};
use std::str::FromStr;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Mutex};
use tracing::{info, instrument};
use tokio::sync::Mutex;
use tracing::{info, instrument, warn};
#[derive(Debug, Clone)]
pub struct NATSArgs {
@@ -168,7 +174,12 @@ where
{
id: TargetID,
args: NATSArgs,
client: Mutex<Option<async_nats::Client>>,
client: Arc<Mutex<Option<async_nats::Client>>>,
tls_state: Arc<parking_lot::Mutex<TargetTlsState>>,
/// Adapter that bridges this target to the TLS reload coordinator.
/// When `Some`, the target uses coordinator-managed material; when `None`,
/// it falls back to inline fingerprint-based change detection.
tls_adapter: Option<TlsReloadAdapter<async_nats::Client>>,
store: Option<Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync>>,
connected: AtomicBool,
delivery_counters: Arc<TargetDeliveryCounters>,
@@ -183,7 +194,9 @@ where
Box::new(NATSTarget::<E> {
id: self.id.clone(),
args: self.args.clone(),
client: Mutex::new(self.client.lock().unwrap().clone()),
client: Arc::clone(&self.client),
tls_state: Arc::clone(&self.tls_state),
tls_adapter: self.tls_adapter.clone(),
store: self.store.as_ref().map(|s| s.boxed_clone()),
connected: AtomicBool::new(self.connected.load(Ordering::SeqCst)),
delivery_counters: Arc::clone(&self.delivery_counters),
@@ -207,7 +220,9 @@ where
Ok(Self {
id: target_id,
args,
client: Mutex::new(None),
client: Arc::new(Mutex::new(None)),
tls_state: Arc::new(parking_lot::Mutex::new(TargetTlsState::default())),
tls_adapter: None,
store: queue_store,
connected: AtomicBool::new(false),
delivery_counters: Arc::new(TargetDeliveryCounters::default()),
@@ -215,11 +230,42 @@ where
})
}
async fn invalidate_cached_client_connection(&self) {
*self.client.lock().await = None;
}
async fn get_or_connect(&self) -> Result<async_nats::Client, TargetError> {
if let Some(client) = self.client.lock().unwrap().clone() {
// Adapter-managed path: use the material directly from the TLS reload adapter.
if let Some(adapter) = &self.tls_adapter {
let client: async_nats::Client = (*adapter.current_material()).clone();
// Ensure the client is also stored locally so that close() can drain it.
{
let mut guard = self.client.lock().await;
*guard = Some(client.clone());
}
return Ok(client);
}
// Inline fingerprint fallback path (no coordinator).
let next_fingerprint =
build_target_tls_fingerprint(&self.args.tls_ca, &self.args.tls_client_cert, &self.args.tls_client_key).await?;
let tls_changed = {
let tls_state_guard = self.tls_state.lock();
tls_state_guard.needs_update(&next_fingerprint)
};
if tls_changed {
self.invalidate_cached_client_connection().await;
self.tls_state.lock().refresh(next_fingerprint);
}
{
let guard = self.client.lock().await;
if let Some(client) = guard.as_ref() {
return Ok(client.clone());
}
}
let client = connect_nats(&self.args).await?;
client
.flush()
@@ -227,7 +273,7 @@ where
.map_err(|e| TargetError::Network(format!("Failed to flush NATS connection: {e}")))?;
self.connected.store(true, Ordering::SeqCst);
let mut guard = self.client.lock().unwrap();
let mut guard = self.client.lock().await;
let shared = guard.get_or_insert_with(|| client.clone()).clone();
Ok(shared)
}
@@ -294,7 +340,11 @@ where
}
async fn close(&self) -> Result<(), TargetError> {
let client = self.client.lock().unwrap().take();
let client = {
let mut guard = self.client.lock().await;
guard.take()
};
self.tls_state.lock().reset();
self.connected.store(false, Ordering::SeqCst);
if let Some(client) = client {
client
@@ -335,3 +385,87 @@ where
self.delivery_counters.record_final_failure();
}
}
/// Coordinated TLS hot-reload implementation for NATS targets.
///
/// The coordinator calls these methods on a background poll loop to detect
/// TLS file changes and rebuild the NATS client without restarting.
#[async_trait]
impl<E> ReloadableTargetTls for NATSTarget<E>
where
E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
{
type Material = async_nats::Client;
fn tls_input_set(&self) -> TargetTlsInputSet {
TargetTlsInputSet {
ca_path: self.args.tls_ca.clone(),
client_cert_path: self.args.tls_client_cert.clone(),
client_key_path: self.args.tls_client_key.clone(),
target_label: format!("nats:{}", self.id.id),
}
}
async fn build_tls_material(&self) -> Result<Self::Material, TargetError> {
connect_nats(&self.args).await
}
async fn apply_tls_material(
&self,
_generation: TargetTlsGeneration,
material: Arc<Self::Material>,
_mode: ReloadApplyMode,
) -> Result<(), TargetError> {
let mut guard = self.client.lock().await;
*guard = Some((*material).clone());
Ok(())
}
async fn validate_tls_files(&self) -> Result<(), TargetError> {
validate_tls_material(&self.args.tls_ca, &self.args.tls_client_cert, &self.args.tls_client_key)
}
}
#[cfg(test)]
mod tests {
use super::*;
fn base_args() -> NATSArgs {
NATSArgs {
enable: true,
address: "nats://127.0.0.1:4222".to_string(),
subject: "rustfs.events".to_string(),
username: String::new(),
password: String::new(),
token: String::new(),
credentials_file: String::new(),
tls_ca: String::new(),
tls_client_cert: String::new(),
tls_client_key: String::new(),
tls_required: false,
queue_dir: String::new(),
queue_limit: 0,
target_type: TargetType::NotifyEvent,
}
}
#[test]
fn validate_nats_rejects_multiple_auth_methods() {
let args = NATSArgs {
token: "abc".to_string(),
username: "user".to_string(),
password: "pass".to_string(),
..base_args()
};
assert!(args.validate().is_err());
}
#[test]
fn validate_nats_rejects_relative_queue_dir() {
let args = NATSArgs {
queue_dir: "relative/path".to_string(),
..base_args()
};
assert!(args.validate().is_err());
}
}
crates/targets/src/target/postgres.rs
+86 -27
View File
@@ -29,6 +29,10 @@ use crate::{
StoreError, Target,
arn::TargetID,
error::TargetError,
runtime::tls::{
ReloadableTargetTls, TargetTlsGeneration, TargetTlsInputSet, TlsReloadAdapter, config::ReloadApplyMode,
validate_tls_material,
},
store::{Key, Store},
target::{
ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
@@ -38,12 +42,10 @@ use crate::{
use async_trait::async_trait;
use deadpool_postgres::{Manager, ManagerConfig, Pool, RecyclingMethod};
use rustfs_config::{POSTGRES_DSN_STRING, POSTGRES_TLS_CA, POSTGRES_TLS_CLIENT_CERT, POSTGRES_TLS_CLIENT_KEY};
use rustls_pki_types::pem::PemObject;
use rustls_pki_types::{CertificateDer, PrivateKeyDer};
use rustfs_tls_runtime::{load_certs, load_private_key};
use serde::Serialize;
use serde::de::DeserializeOwned;
use std::fmt;
use std::io::BufReader;
use std::path::Path;
use std::sync::Arc;
use tokio_postgres::Config;
@@ -425,11 +427,9 @@ pub fn build_tls_config(args: &PostgresArgs) -> Result<rustls::ClientConfig, Tar
let _ = root_store.add(cert);
}
} else {
let pem = std::fs::read(&args.tls_ca)
.map_err(|e| TargetError::Configuration(format!("failed to read {POSTGRES_TLS_CA}: {e}")))?;
let mut reader = BufReader::new(pem.as_slice());
for cert in CertificateDer::pem_reader_iter(&mut reader) {
let cert = cert.map_err(|e| TargetError::Configuration(format!("invalid {POSTGRES_TLS_CA}: {e}")))?;
let certs =
load_certs(&args.tls_ca).map_err(|e| TargetError::Configuration(format!("invalid {POSTGRES_TLS_CA}: {e}")))?;
for cert in certs {
root_store
.add(cert)
.map_err(|e| TargetError::Configuration(format!("failed to add CA cert: {e}")))?;
@@ -439,16 +439,9 @@ pub fn build_tls_config(args: &PostgresArgs) -> Result<rustls::ClientConfig, Tar
let builder = rustls::ClientConfig::builder().with_root_certificates(root_store);
let client_config = if !args.tls_client_cert.is_empty() && !args.tls_client_key.is_empty() {
let cert_pem = std::fs::read(&args.tls_client_cert)
.map_err(|e| TargetError::Configuration(format!("failed to read {POSTGRES_TLS_CLIENT_CERT}: {e}")))?;
let key_pem = std::fs::read(&args.tls_client_key)
.map_err(|e| TargetError::Configuration(format!("failed to read {POSTGRES_TLS_CLIENT_KEY}: {e}")))?;
let certs: Vec<_> = CertificateDer::pem_reader_iter(&mut BufReader::new(cert_pem.as_slice()))
.collect::<Result<_, _>>()
let certs = load_certs(&args.tls_client_cert)
.map_err(|e| TargetError::Configuration(format!("invalid {POSTGRES_TLS_CLIENT_CERT}: {e}")))?;
let key = PrivateKeyDer::from_pem_reader(&mut BufReader::new(key_pem.as_slice()))
let key = load_private_key(&args.tls_client_key)
.map_err(|e| TargetError::Configuration(format!("invalid {POSTGRES_TLS_CLIENT_KEY}: {e}")))?;
builder
@@ -548,13 +541,22 @@ fn resolve_payload_key(payload: &serde_json::Value, meta: &QueuedPayloadMeta) ->
/// so that `clone_box` does not duplicate connection state. The optional
/// `QueueStore` provides at-least-once delivery semantics consistent with the
/// other built-in targets.
///
/// When `tls_adapter` is `Some`, the target participates in the
/// coordinated TLS hot-reload system driven by `TlsReloadAdapter`,
/// and the inline fingerprint check in `send_body` is skipped. When `None`,
/// the legacy inline fingerprint check is used as a fallback.
pub struct PostgresTarget<E>
where
E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
{
id: TargetID,
args: PostgresArgs,
pool: Pool,
pool: Arc<parking_lot::Mutex<Pool>>,
tls_state: Arc<parking_lot::Mutex<super::TargetTlsState>>,
/// When present, the adapter provides coordinator-managed TLS material;
/// otherwise the inline fingerprint path is used as a fallback.
tls_adapter: Option<TlsReloadAdapter<Pool>>,
namespace_sql: String,
access_sql: String,
store: Option<Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync>>,
@@ -570,7 +572,9 @@ where
Box::new(PostgresTarget::<E> {
id: self.id.clone(),
args: self.args.clone(),
pool: self.pool.clone(),
pool: Arc::clone(&self.pool),
tls_state: Arc::clone(&self.tls_state),
tls_adapter: self.tls_adapter.clone(),
namespace_sql: self.namespace_sql.clone(),
access_sql: self.access_sql.clone(),
store: self.store.as_ref().map(|s| s.boxed_clone()),
@@ -599,7 +603,9 @@ where
namespace_sql: namespace_upsert_sql(&args.schema, &args.table),
access_sql: access_insert_sql(&args.schema, &args.table),
args,
pool,
pool: Arc::new(parking_lot::Mutex::new(pool)),
tls_state: Arc::new(parking_lot::Mutex::new(super::TargetTlsState::default())),
tls_adapter: None,
store: queue_store,
delivery_counters: Arc::new(TargetDeliveryCounters::default()),
_phantom: std::marker::PhantomData,
@@ -611,8 +617,25 @@ where
/// Identifier validation has already happened in `PostgresArgs::validate()`,
/// so `qualified_table` cannot produce a malformed SQL string here.
async fn send_body(&self, body: &[u8], event_id: &str, meta: &QueuedPayloadMeta) -> Result<(), TargetError> {
let client = self
.pool
// When a TLS reload adapter is attached, it drives pool rebuilds in
// the background. The inline per-send fingerprint check is skipped.
if self.tls_adapter.is_none() {
let next_fingerprint =
super::build_target_tls_fingerprint(&self.args.tls_ca, &self.args.tls_client_cert, &self.args.tls_client_key)
.await?;
let tls_changed = {
let tls_state_guard = self.tls_state.lock();
tls_state_guard.fingerprint.as_ref() != Some(&next_fingerprint)
};
if tls_changed {
let new_pool = build_pool(&self.args)?;
*self.pool.lock() = new_pool;
self.tls_state.lock().refresh(next_fingerprint);
}
}
let pool = self.pool.lock().clone();
let client = pool
.get()
.await
.map_err(|e| map_pool_error(e, "PostgreSQL pool checkout failed"))?;
@@ -645,8 +668,8 @@ where
/// Probes the table from `init()`. Failure is non-fatal when a queue is
/// configured: events buffer in the store until the schema is fixed.
async fn probe_table(&self) -> Result<(), TargetError> {
let client = self
.pool
let pool = self.pool.lock().clone();
let client = pool
.get()
.await
.map_err(|e| map_pool_error(e, "PostgreSQL pool checkout failed during init probe"))?;
@@ -659,6 +682,41 @@ where
}
}
#[async_trait]
impl<E> ReloadableTargetTls for PostgresTarget<E>
where
E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
{
type Material = Pool;
fn tls_input_set(&self) -> TargetTlsInputSet {
TargetTlsInputSet {
ca_path: self.args.tls_ca.clone(),
client_cert_path: self.args.tls_client_cert.clone(),
client_key_path: self.args.tls_client_key.clone(),
target_label: format!("postgres:{}", self.id.id),
}
}
async fn build_tls_material(&self) -> Result<Self::Material, TargetError> {
build_pool(&self.args)
}
async fn apply_tls_material(
&self,
_generation: TargetTlsGeneration,
material: Arc<Self::Material>,
_mode: ReloadApplyMode,
) -> Result<(), TargetError> {
*self.pool.lock() = (*material).clone();
Ok(())
}
async fn validate_tls_files(&self) -> Result<(), TargetError> {
validate_tls_material(&self.args.tls_ca, &self.args.tls_client_cert, &self.args.tls_client_key)
}
}
#[async_trait]
impl<E> Target<E> for PostgresTarget<E>
where
@@ -674,8 +732,8 @@ where
}
match tokio::time::timeout(std::time::Duration::from_secs(10), async {
let client = self
.pool
let pool = self.pool.lock().clone();
let client = pool
.get()
.await
.map_err(|e| map_pool_error(e, "PostgreSQL pool checkout failed"))?;
@@ -728,7 +786,8 @@ where
}
async fn close(&self) -> Result<(), TargetError> {
self.pool.close();
self.pool.lock().close();
// Adapter cleanup is done by the coordinator; no local state to reset.
info!(target_id = %self.id, "PostgreSQL target closed");
Ok(())
}
crates/targets/src/target/pulsar.rs
+148 -2
View File
@@ -16,14 +16,20 @@ use crate::{
StoreError, Target,
arn::TargetID,
error::TargetError,
runtime::tls::{
ReloadableTargetTls, TargetTlsGeneration, TargetTlsInputSet, TlsReloadAdapter, config::ReloadApplyMode,
validate_tls_material,
},
store::{Key, Store},
target::{
ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
TargetType, build_queued_payload_with_records, open_target_queue_store, persist_queued_payload_to_store,
TargetTlsState, TargetType, build_queued_payload_with_records, build_target_tls_fingerprint, open_target_queue_store,
persist_queued_payload_to_store,
},
};
use async_trait::async_trait;
use pulsar::{Authentication, Producer, Pulsar, TokioExecutor};
use rustfs_tls_runtime::load_cert_bundle_der_bytes;
use serde::Serialize;
use serde::de::DeserializeOwned;
use std::path::Path;
@@ -136,6 +142,13 @@ pub async fn connect_pulsar(args: &PulsarArgs) -> Result<Pulsar<TokioExecutor>,
}
if !args.tls_ca.is_empty() {
let certs = load_cert_bundle_der_bytes(&args.tls_ca)
.map_err(|e| TargetError::Configuration(format!("Failed to parse Pulsar tls_ca: {e}")))?;
if certs.is_empty() {
return Err(TargetError::Configuration(
"Pulsar tls_ca did not contain any parsable certificates".to_string(),
));
}
builder = builder
.with_certificate_chain_file(&args.tls_ca)
.map_err(|e| TargetError::Configuration(format!("Failed to load Pulsar tls_ca: {e}")))?;
@@ -158,6 +171,9 @@ where
id: TargetID,
args: PulsarArgs,
client: Mutex<Option<Pulsar<TokioExecutor>>>,
tls_state: Mutex<TargetTlsState>,
/// When set, the coordinator drives TLS reload; inline fingerprint check is skipped.
tls_adapter: Option<TlsReloadAdapter<Pulsar<TokioExecutor>>>,
producer: AsyncMutex<Option<Producer<TokioExecutor>>>,
store: Option<Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync>>,
connected: AtomicBool,
@@ -174,6 +190,8 @@ where
id: self.id.clone(),
args: self.args.clone(),
client: Mutex::new(self.client.lock().unwrap().clone()),
tls_state: Mutex::new(self.tls_state.lock().unwrap().clone()),
tls_adapter: self.tls_adapter.clone(),
producer: AsyncMutex::new(None),
store: self.store.as_ref().map(|s| s.boxed_clone()),
connected: AtomicBool::new(self.connected.load(Ordering::SeqCst)),
@@ -199,6 +217,8 @@ where
id: target_id,
args,
client: Mutex::new(None),
tls_state: Mutex::new(TargetTlsState::default()),
tls_adapter: None,
producer: AsyncMutex::new(None),
store: queue_store,
connected: AtomicBool::new(false),
@@ -207,7 +227,36 @@ where
})
}
fn clear_cached_client_connection(&self) {
self.client.lock().unwrap().take();
}
fn clear_cached_client(&self) {
self.clear_cached_client_connection();
self.tls_state.lock().unwrap().reset();
}
async fn get_or_connect_client(&self) -> Result<Pulsar<TokioExecutor>, TargetError> {
// When a TLS reload adapter is attached, it drives client rebuilds
// in the background. The inline per-send fingerprint check is skipped.
if let Some(adapter) = &self.tls_adapter {
let material = adapter.current_material();
{
let mut guard = self.client.lock().unwrap();
*guard = Some((*material).clone());
}
} else {
let next_fingerprint = build_target_tls_fingerprint(&self.args.tls_ca, "", "").await?;
let tls_changed = {
let tls_state_guard = self.tls_state.lock().unwrap();
tls_state_guard.needs_update(&next_fingerprint)
};
if tls_changed {
self.clear_cached_client_connection();
self.tls_state.lock().unwrap().refresh(next_fingerprint);
}
}
if let Some(client) = self.client.lock().unwrap().clone() {
return Ok(client);
}
@@ -262,6 +311,56 @@ where
}
}
/// Coordinated TLS hot-reload implementation for Pulsar targets.
///
/// Pulsar only uses a CA certificate (no client cert/key).
/// The coordinator calls these methods on a background poll loop to detect
/// TLS file changes and rebuild the client without restarting.
#[async_trait]
impl<E> ReloadableTargetTls for PulsarTarget<E>
where
E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
{
type Material = Pulsar<TokioExecutor>;
fn tls_input_set(&self) -> TargetTlsInputSet {
TargetTlsInputSet {
ca_path: self.args.tls_ca.clone(),
client_cert_path: String::new(),
client_key_path: String::new(),
target_label: format!("pulsar:{}", self.id.id),
}
}
async fn build_tls_material(&self) -> Result<Self::Material, TargetError> {
connect_pulsar(&self.args).await
}
async fn apply_tls_material(
&self,
_generation: TargetTlsGeneration,
material: Arc<Self::Material>,
_mode: ReloadApplyMode,
) -> Result<(), TargetError> {
// Pulsar client is Clone, so we clone from the Arc and store it.
{
let mut guard = self.client.lock().unwrap();
*guard = Some((*material).clone());
}
// Producer is bound to the old client; clear it so next send rebuilds.
{
let mut producer = self.producer.lock().await;
*producer = None;
}
Ok(())
}
async fn validate_tls_files(&self) -> Result<(), TargetError> {
// Pulsar only uses CA, no client cert/key.
validate_tls_material(&self.args.tls_ca, "", "")
}
}
#[async_trait]
impl<E> Target<E> for PulsarTarget<E>
where
@@ -321,8 +420,13 @@ where
.map_err(|e| TargetError::Network(format!("Failed to close Pulsar producer: {e}")))?;
}
*producer = None;
self.client.lock().unwrap().take();
self.clear_cached_client();
self.connected.store(false, Ordering::SeqCst);
// If a TLS reload adapter is attached, reset its error tracking
// so that a future re-init does not inherit stale failure state.
if let Some(adapter) = &self.tls_adapter {
*adapter.runtime_state().last_error.write() = None;
}
info!(target_id = %self.id, "Pulsar target closed");
Ok(())
}
@@ -355,3 +459,45 @@ where
self.delivery_counters.record_final_failure();
}
}
#[cfg(test)]
mod tests {
use super::*;
fn base_args() -> PulsarArgs {
PulsarArgs {
enable: true,
broker: "pulsar://127.0.0.1:6650".to_string(),
topic: "persistent://public/default/rustfs-events".to_string(),
auth_token: String::new(),
username: String::new(),
password: String::new(),
tls_ca: String::new(),
tls_allow_insecure: false,
tls_hostname_verification: true,
queue_dir: String::new(),
queue_limit: 0,
target_type: TargetType::NotifyEvent,
}
}
#[test]
fn validate_pulsar_rejects_mixed_auth_methods() {
let args = PulsarArgs {
auth_token: "token".to_string(),
username: "user".to_string(),
password: "pass".to_string(),
..base_args()
};
assert!(args.validate().is_err());
}
#[test]
fn validate_pulsar_rejects_relative_queue_dir() {
let args = PulsarArgs {
queue_dir: "relative/path".to_string(),
..base_args()
};
assert!(args.validate().is_err());
}
}
crates/targets/src/target/redis.rs
+116 -9
View File
@@ -16,6 +16,10 @@ use crate::{
StoreError, Target,
arn::TargetID,
error::TargetError,
runtime::tls::{
ReloadableTargetTls, TargetTlsGeneration, TargetTlsInputSet, TlsReloadAdapter, config::ReloadApplyMode,
validate_tls_material,
},
store::{Key, Store},
target::{
ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
@@ -31,9 +35,12 @@ use redis::{
io::tcp::{TcpSettings, socket2},
};
use rustfs_config::{REDIS_TLS_CA, REDIS_TLS_CLIENT_CERT, REDIS_TLS_CLIENT_KEY, REDIS_TLS_POLICY};
use rustls::pki_types::CertificateDer;
use rustls::pki_types::pem::PemObject;
use serde::Serialize;
use serde::de::DeserializeOwned;
use std::fmt;
use std::io::BufReader;
use std::path::Path;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
@@ -297,7 +304,8 @@ where
{
id: TargetID,
args: RedisArgs,
publisher_client: Client,
/// Redis client, wrapped in a lock so TLS hot-reload can atomically replace it.
publisher_client: Arc<parking_lot::Mutex<Client>>,
publisher: Arc<Mutex<Option<ConnectionManager>>>,
store: Option<Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync>>,
/// Business-level liveness flag.
@@ -306,6 +314,12 @@ where
/// publish exhausted retries, or the target was explicitly closed). Temporary reconnectable
/// errors only invalidate the cached publisher so that a later request can lazily rebuild it.
connected: Arc<AtomicBool>,
/// TLS fingerprint tracking for hot reload (inline fallback path).
tls_state: Arc<parking_lot::Mutex<super::TargetTlsState>>,
/// Adapter that bridges this target to the TLS reload coordinator.
/// When `Some`, the target uses coordinator-managed material; when `None`,
/// it falls back to inline fingerprint-based change detection.
tls_adapter: Option<TlsReloadAdapter<Client>>,
delivery_counters: Arc<TargetDeliveryCounters>,
_phantom: std::marker::PhantomData<E>,
}
@@ -334,10 +348,12 @@ where
Ok(Self {
id: target_id,
args,
publisher_client,
publisher_client: Arc::new(parking_lot::Mutex::new(publisher_client)),
publisher: Arc::new(Mutex::new(None)),
store: queue_store,
connected: Arc::new(AtomicBool::new(false)),
tls_state: Arc::new(parking_lot::Mutex::new(super::TargetTlsState::default())),
tls_adapter: None,
delivery_counters: Arc::new(TargetDeliveryCounters::default()),
_phantom: std::marker::PhantomData,
})
@@ -347,23 +363,61 @@ where
Box::new(Self {
id: self.id.clone(),
args: self.args.clone(),
publisher_client: self.publisher_client.clone(),
publisher_client: Arc::clone(&self.publisher_client),
publisher: Arc::clone(&self.publisher),
store: self.store.as_ref().map(|s| s.boxed_clone()),
connected: Arc::clone(&self.connected),
tls_state: Arc::clone(&self.tls_state),
tls_adapter: self.tls_adapter.clone(),
delivery_counters: Arc::clone(&self.delivery_counters),
_phantom: std::marker::PhantomData,
})
}
async fn get_or_create_publisher(&self) -> Result<ConnectionManager, TargetError> {
// Adapter-managed path: use the material directly from the TLS reload adapter.
if let Some(adapter) = &self.tls_adapter {
let client: Client = (*adapter.current_material()).clone();
// Ensure the client is also stored locally so close() can drain it.
*self.publisher_client.lock() = client.clone();
let manager = client
.get_connection_manager_lazy(build_redis_connection_manager_config(&self.args))
.map_err(map_redis_error)?;
*self.publisher.lock().await = Some(manager.clone());
return Ok(manager);
}
// Inline fingerprint fallback path (no coordinator).
let secure_scheme = matches!(self.args.url.scheme(), "rediss" | "valkeys");
if secure_scheme {
let next_fingerprint = super::build_target_tls_fingerprint(
&self.args.tls.ca_path,
&self.args.tls.client_cert_path,
&self.args.tls.client_key_path,
)
.await?;
let tls_changed = {
let tls_state_guard = self.tls_state.lock();
tls_state_guard.needs_update(&next_fingerprint)
};
if tls_changed {
let new_client = build_redis_client(&self.args)?;
*self.publisher_client.lock() = new_client;
self.invalidate_cached_publisher().await;
self.tls_state.lock().refresh(next_fingerprint);
}
}
let mut guard = self.publisher.lock().await;
if let Some(manager) = guard.clone() {
return Ok(manager);
}
let manager = self
.publisher_client
let client = self.publisher_client.lock().clone();
let manager = client
.get_connection_manager_lazy(build_redis_connection_manager_config(&self.args))
.map_err(map_redis_error)?;
@@ -475,7 +529,8 @@ where
return Ok(false);
}
match tokio::time::timeout(Duration::from_secs(5), ping_redis_server(&self.publisher_client, &self.args)).await {
let client = self.publisher_client.lock().clone();
match tokio::time::timeout(Duration::from_secs(5), ping_redis_server(&client, &self.args)).await {
Ok(Ok(())) => {
self.connected.store(true, Ordering::SeqCst);
Ok(true)
@@ -557,6 +612,7 @@ where
async fn close(&self) -> Result<(), TargetError> {
self.invalidate_cached_publisher().await;
self.tls_state.lock().reset();
self.connected.store(false, Ordering::SeqCst);
info!(target_id = %self.id, "Redis target closed");
Ok(())
@@ -696,9 +752,20 @@ fn read_root_cert(tls: &RedisTlsConfig) -> Result<Option<Vec<u8>>, TargetError>
return Ok(None);
}
std::fs::read(&tls.ca_path)
.map(Some)
.map_err(|e| TargetError::Configuration(format!("Failed to read Redis root CA cert: {e}")))
let pem =
std::fs::read(&tls.ca_path).map_err(|e| TargetError::Configuration(format!("Failed to read Redis root CA cert: {e}")))?;
let mut reader = BufReader::new(pem.as_slice());
let certs_der = CertificateDer::pem_reader_iter(&mut reader)
.collect::<Result<Vec<_>, _>>()
.map_err(|e| TargetError::Configuration(format!("Failed to parse Redis root CA cert: {e}")))?;
if certs_der.is_empty() {
return Err(TargetError::Configuration(
"Redis root CA cert did not contain any parsable certificates".to_string(),
));
}
Ok(Some(pem))
}
fn map_redis_error(err: RedisError) -> TargetError {
@@ -722,6 +789,46 @@ fn compute_retry_delay(attempt: usize, min_delay: Duration, max_delay: Duration)
min_delay.saturating_mul(factor).min(max_delay)
}
/// Coordinated TLS hot-reload implementation for Redis targets.
///
/// The coordinator calls these methods on a background poll loop to detect
/// TLS file changes and rebuild the Redis client without restarting.
#[async_trait]
impl<E> ReloadableTargetTls for RedisTarget<E>
where
E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
{
type Material = Client;
fn tls_input_set(&self) -> TargetTlsInputSet {
TargetTlsInputSet {
ca_path: self.args.tls.ca_path.clone(),
client_cert_path: self.args.tls.client_cert_path.clone(),
client_key_path: self.args.tls.client_key_path.clone(),
target_label: format!("redis:{}", self.id.id),
}
}
async fn build_tls_material(&self) -> Result<Self::Material, TargetError> {
build_redis_client(&self.args)
}
async fn apply_tls_material(
&self,
_generation: TargetTlsGeneration,
material: Arc<Self::Material>,
_mode: ReloadApplyMode,
) -> Result<(), TargetError> {
*self.publisher_client.lock() = (*material).clone();
self.invalidate_cached_publisher().await;
Ok(())
}
async fn validate_tls_files(&self) -> Result<(), TargetError> {
validate_tls_material(&self.args.tls.ca_path, &self.args.tls.client_cert_path, &self.args.tls.client_key_path)
}
}
#[cfg(test)]
mod tests {
use super::*;
crates/targets/src/target/webhook.rs
+105 -10
View File
@@ -16,14 +16,21 @@ use crate::{
StoreError, Target,
arn::TargetID,
error::TargetError,
runtime::tls::{
ReloadableTargetTls, TargetTlsInputSet, TlsReloadAdapter, config::ReloadApplyMode, fingerprint::TargetTlsGeneration,
validate::validate_tls_material,
},
store::{Key, Store},
target::{
ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
TargetType, build_queued_payload, open_target_queue_store, persist_queued_payload_to_store,
TargetTlsState, TargetType, build_queued_payload, build_target_tls_fingerprint, open_target_queue_store,
persist_queued_payload_to_store,
},
};
use async_trait::async_trait;
use parking_lot::Mutex;
use reqwest::{Client, StatusCode, Url};
use rustfs_tls_runtime::load_cert_bundle_der_bytes;
use serde::Serialize;
use serde::de::DeserializeOwned;
use std::{
@@ -104,7 +111,11 @@ where
id: TargetID,
args: WebhookArgs,
health_check_url: Option<Url>,
http_client: Arc<Client>,
http_client: Arc<Mutex<Client>>,
tls_state: Arc<Mutex<TargetTlsState>>,
/// When present, the adapter provides coordinator-managed TLS material;
/// otherwise the inline fingerprint path is used as a fallback.
tls_adapter: Option<TlsReloadAdapter<Client>>,
// Add Send + Sync constraints to ensure thread safety
store: Option<Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync>>,
initialized: AtomicBool,
@@ -124,6 +135,8 @@ where
args: self.args.clone(),
health_check_url: self.health_check_url.clone(),
http_client: Arc::clone(&self.http_client),
tls_state: Arc::clone(&self.tls_state),
tls_adapter: self.tls_adapter.clone(),
store: self.store.as_ref().map(|s| s.boxed_clone()),
initialized: AtomicBool::new(self.initialized.load(Ordering::SeqCst)),
cancel_sender: self.cancel_sender.clone(),
@@ -146,7 +159,7 @@ where
};
// Build HTTP client using the helper function
let http_client = Arc::new(Self::build_http_client(&args)?);
let http_client = Arc::new(Mutex::new(Self::build_http_client(&args)?));
let queue_store = open_target_queue_store(
&args.queue_dir,
@@ -165,6 +178,8 @@ where
args,
health_check_url,
http_client,
tls_state: Arc::new(Mutex::new(TargetTlsState::default())),
tls_adapter: None,
store: queue_store,
initialized: AtomicBool::new(false),
cancel_sender,
@@ -188,11 +203,18 @@ where
);
} else if !args.client_ca.is_empty() {
// Use user-provided custom CA certificate
let ca_cert_pem = std::fs::read(&args.client_ca)
.map_err(|e| TargetError::Configuration(format!("Failed to read root CA cert: {e}")))?;
let ca_cert = reqwest::Certificate::from_pem(&ca_cert_pem)
let certs_der = load_cert_bundle_der_bytes(&args.client_ca)
.map_err(|e| TargetError::Configuration(format!("Failed to parse root CA cert: {e}")))?;
client_builder = client_builder.add_root_certificate(ca_cert);
if certs_der.is_empty() {
return Err(TargetError::Configuration(
"Webhook client_ca did not contain any parsable certificates".to_string(),
));
}
for cert_der in certs_der {
let ca_cert = reqwest::Certificate::from_der(&cert_der)
.map_err(|e| TargetError::Configuration(format!("Failed to load root CA cert: {e}")))?;
client_builder = client_builder.add_root_certificate(ca_cert);
}
}
// If neither is set, use the system's default trust store
@@ -213,6 +235,29 @@ where
.map_err(|e| TargetError::Configuration(format!("Failed to build HTTP client: {e}")))
}
async fn refresh_tls(&self) -> Result<(), TargetError> {
let next_fingerprint =
build_target_tls_fingerprint(&self.args.client_ca, &self.args.client_cert, &self.args.client_key).await?;
let tls_changed = {
let tls_state_guard = self.tls_state.lock();
tls_state_guard.fingerprint.as_ref() != Some(&next_fingerprint)
};
if !tls_changed {
return Ok(());
}
let new_client = Self::build_http_client(&self.args)?;
{
let mut tls_state_guard = self.tls_state.lock();
if tls_state_guard.fingerprint.as_ref() == Some(&next_fingerprint) {
return Ok(());
}
*self.http_client.lock() = new_client;
tls_state_guard.refresh(next_fingerprint);
}
Ok(())
}
fn health_check_url(endpoint: &Url) -> Result<Url, TargetError> {
endpoint
.host()
@@ -230,7 +275,8 @@ where
return Ok(false);
};
match tokio::time::timeout(Duration::from_secs(5), self.http_client.head(health_check_url.as_str()).send()).await {
let client = self.http_client.lock().clone();
match tokio::time::timeout(Duration::from_secs(5), client.head(health_check_url.as_str()).send()).await {
Ok(Ok(resp)) => {
debug!(
target = %self.id,
@@ -299,8 +345,14 @@ where
"Sending webhook payload"
);
let mut req_builder = self
.http_client
// When a TLS reload adapter is attached, it drives client rebuilds in
// the background. The inline per-send fingerprint check is skipped.
if self.tls_adapter.is_none() {
self.refresh_tls().await?;
}
let client = self.http_client.lock().clone();
let mut req_builder = client
.post(self.args.endpoint.as_str())
.header("Content-Type", meta.content_type.as_str());
@@ -425,6 +477,7 @@ where
async fn close(&self) -> Result<(), TargetError> {
// Send cancel signal to background tasks
let _ = self.cancel_sender.try_send(());
// Adapter cleanup is done by the coordinator; no local state to reset.
info!("Webhook target closed: {}", self.id);
Ok(())
}
@@ -460,6 +513,48 @@ where
}
}
/// Coordinated TLS hot-reload implementation for Webhook targets.
///
/// The coordinator calls these methods on a background poll loop to detect
/// TLS file changes and rebuild the HTTP client without restarting.
#[async_trait]
impl<E> ReloadableTargetTls for WebhookTarget<E>
where
E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
{
type Material = Client;
fn tls_input_set(&self) -> TargetTlsInputSet {
TargetTlsInputSet {
ca_path: self.args.client_ca.clone(),
client_cert_path: self.args.client_cert.clone(),
client_key_path: self.args.client_key.clone(),
target_label: format!("webhook:{}", self.id.id),
}
}
async fn build_tls_material(&self) -> Result<Self::Material, TargetError> {
// build_http_client is synchronous (reads files + configures reqwest).
// The coordinator already runs this in a background task, so the
// synchronous file I/O does not block the send path.
Self::build_http_client(&self.args)
}
async fn apply_tls_material(
&self,
_generation: TargetTlsGeneration,
material: Arc<Self::Material>,
_mode: ReloadApplyMode,
) -> Result<(), TargetError> {
*self.http_client.lock() = (*material).clone();
Ok(())
}
async fn validate_tls_files(&self) -> Result<(), TargetError> {
validate_tls_material(&self.args.client_ca, &self.args.client_cert, &self.args.client_key)
}
}
#[cfg(test)]
mod tests {
use super::{WebhookArgs, WebhookTarget};
crates/tls-runtime/Cargo.toml
+50
View File
@@ -0,0 +1,50 @@
# Copyright 2024 RustFS Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
[package]
name = "rustfs-tls-runtime"
version.workspace = true
edition.workspace = true
license.workspace = true
repository.workspace = true
rust-version.workspace = true
homepage.workspace = true
description = "Project-wide TLS runtime foundation for RustFS."
keywords = ["tls", "runtime", "rustls", "hot-reload", "rustfs"]
categories = ["network-programming", "web-programming", "development-tools"]
[lints]
workspace = true
[dependencies]
rustfs-common.workspace = true
rustfs-config.workspace = true
arc-swap.workspace = true
metrics.workspace = true
rustls.workspace = true
rustls-pki-types.workspace = true
serde = { workspace = true, features = ["derive"] }
sha2.workspace = true
thiserror.workspace = true
tokio = { workspace = true, features = ["fs", "rt-multi-thread", "sync", "time"] }
tracing.workspace = true
[dev-dependencies]
rcgen.workspace = true
serde_json.workspace = true
tempfile.workspace = true
tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }
[lib]
doctest = false
crates/utils/src/certs.rs → crates/tls-runtime/src/certs.rs
+21 -337
View File
@@ -25,7 +25,6 @@ use std::sync::Arc;
use std::{fs, io};
use tracing::{debug, warn};
/// Options for loading certificate/key pairs from a directory tree.
#[derive(Debug, Clone)]
pub struct CertDirectoryLoadOptions {
dir_path: PathBuf,
@@ -34,7 +33,6 @@ pub struct CertDirectoryLoadOptions {
}
impl CertDirectoryLoadOptions {
/// Create a builder with explicit certificate and private key filenames.
pub fn builder(
dir_path: impl Into<PathBuf>,
cert_filename: impl Into<String>,
@@ -58,7 +56,6 @@ impl CertDirectoryLoadOptions {
}
}
/// Builder for [`CertDirectoryLoadOptions`].
#[derive(Debug, Clone)]
pub struct CertDirectoryLoadOptionsBuilder {
dir_path: PathBuf,
@@ -67,19 +64,16 @@ pub struct CertDirectoryLoadOptionsBuilder {
}
impl CertDirectoryLoadOptionsBuilder {
/// Override the certificate filename searched in the directory.
pub fn cert_filename(mut self, cert_filename: impl Into<String>) -> Self {
self.cert_filename = cert_filename.into();
self
}
/// Override the private key filename searched in the directory.
pub fn key_filename(mut self, key_filename: impl Into<String>) -> Self {
self.key_filename = key_filename.into();
self
}
/// Build the load options value.
pub fn build(self) -> CertDirectoryLoadOptions {
CertDirectoryLoadOptions {
dir_path: self.dir_path,
@@ -89,7 +83,6 @@ impl CertDirectoryLoadOptionsBuilder {
}
}
/// Options for building an mTLS WebPki client verifier.
#[derive(Debug, Clone)]
pub struct WebPkiClientVerifierOptions {
tls_path: PathBuf,
@@ -99,7 +92,6 @@ pub struct WebPkiClientVerifierOptions {
}
impl WebPkiClientVerifierOptions {
/// Create a builder with explicit CA bundle filenames.
pub fn builder(
tls_path: impl Into<PathBuf>,
client_ca_cert_filename: impl Into<String>,
@@ -114,7 +106,6 @@ impl WebPkiClientVerifierOptions {
}
}
/// Builder for [`WebPkiClientVerifierOptions`].
#[derive(Debug, Clone)]
pub struct WebPkiClientVerifierOptionsBuilder {
tls_path: PathBuf,
@@ -124,25 +115,21 @@ pub struct WebPkiClientVerifierOptionsBuilder {
}
impl WebPkiClientVerifierOptionsBuilder {
/// Set whether mTLS verification should be enabled.
pub fn enabled(mut self, enabled: bool) -> Self {
self.enabled = enabled;
self
}
/// Override the preferred client CA bundle filename.
pub fn client_ca_cert_filename(mut self, client_ca_cert_filename: impl Into<String>) -> Self {
self.client_ca_cert_filename = client_ca_cert_filename.into();
self
}
/// Override the fallback CA bundle filename.
pub fn fallback_ca_cert_filename(mut self, fallback_ca_cert_filename: impl Into<String>) -> Self {
self.fallback_ca_cert_filename = fallback_ca_cert_filename.into();
self
}
/// Build the verifier options value.
pub fn build(self) -> WebPkiClientVerifierOptions {
WebPkiClientVerifierOptions {
tls_path: self.tls_path,
@@ -153,21 +140,10 @@ impl WebPkiClientVerifierOptionsBuilder {
}
}
/// Load public certificate from file.
/// This function loads a public certificate from the specified file.
///
/// # Arguments
/// * `filename` - A string slice that holds the name of the file containing the public certificate.
///
/// # Returns
/// * An io::Result containing a vector of CertificateDer if successful, or an io::Error if an error occurs during loading.
///
pub fn load_certs(filename: &str) -> io::Result<Vec<CertificateDer<'static>>> {
// Open certificate file.
let cert_file = fs::File::open(filename).map_err(|e| certs_error(format!("failed to open {filename}: {e}")))?;
let mut reader = io::BufReader::new(cert_file);
// Load and return certificate.
let certs = CertificateDer::pem_reader_iter(&mut reader)
.collect::<Result<Vec<_>, _>>()
.map_err(|e| certs_error(format!("certificate file {filename} format error:{e:?}")))?;
@@ -177,16 +153,6 @@ pub fn load_certs(filename: &str) -> io::Result<Vec<CertificateDer<'static>>> {
Ok(certs)
}
/// Load a PEM certificate bundle and return each certificate as DER bytes.
///
/// This is a low-level helper intended for TLS clients (reqwest/hyper-rustls) that
/// need to add root certificates one-by-one.
///
/// - Input: a PEM file that may contain multiple cert blocks.
/// - Output: Vec of DER-encoded cert bytes, one per cert.
///
/// NOTE: This intentionally returns raw bytes to avoid forcing downstream crates
/// to depend on rustls types.
pub fn load_cert_bundle_der_bytes(path: &str) -> io::Result<Vec<Vec<u8>>> {
let pem = fs::read(path)?;
let mut reader = io::BufReader::new(&pem[..]);
@@ -198,15 +164,6 @@ pub fn load_cert_bundle_der_bytes(path: &str) -> io::Result<Vec<Vec<u8>>> {
Ok(certs.into_iter().map(|c| c.to_vec()).collect())
}
/// Builds a WebPkiClientVerifier for mTLS when enabled by the caller.
///
/// # Arguments
/// * `options` - mTLS verifier options, including the TLS directory and CA bundle filenames
///
/// # Returns
/// * `Ok(Some(verifier))` if mTLS is enabled and CA certs are found
/// * `Ok(None)` if mTLS is disabled
/// * `Err` if mTLS is enabled but configuration is invalid
pub fn build_webpki_client_verifier(options: WebPkiClientVerifierOptions) -> io::Result<Option<Arc<dyn ClientCertVerifier>>> {
if !options.enabled {
return Ok(None);
@@ -243,7 +200,6 @@ pub fn build_webpki_client_verifier(options: WebPkiClientVerifierOptions) -> io:
Ok(Some(verifier))
}
/// Locate the mTLS client CA bundle in the specified TLS path
fn mtls_ca_bundle_path(options: &WebPkiClientVerifierOptions) -> Option<PathBuf> {
let p1 = options.tls_path.join(&options.client_ca_cert_filename);
if p1.exists() {
@@ -256,34 +212,14 @@ fn mtls_ca_bundle_path(options: &WebPkiClientVerifierOptions) -> Option<PathBuf>
None
}
/// Load private key from file.
/// This function loads a private key from the specified file.
///
/// # Arguments
/// * `filename` - A string slice that holds the name of the file containing the private key.
///
/// # Returns
/// * An io::Result containing the PrivateKeyDer if successful, or an io::Error if an error occurs during loading.
///
pub fn load_private_key(filename: &str) -> io::Result<PrivateKeyDer<'static>> {
// Open keyfile.
let keyfile = fs::File::open(filename).map_err(|e| certs_error(format!("failed to open {filename}: {e}")))?;
let mut reader = io::BufReader::new(keyfile);
// Load and return a single private key.
PrivateKeyDer::from_pem_reader(&mut reader)
.map_err(|e| certs_error(format!("failed to parse private key in {filename}: {e}")))
}
/// error function
/// This function creates a new io::Error with the provided error message.
///
/// # Arguments
/// * `err` - A string containing the error message.
///
/// # Returns
/// * An io::Error instance with the specified error message.
///
pub fn certs_error(err: String) -> Error {
Error::other(err)
}
@@ -292,17 +228,6 @@ fn is_discoverable_cert_domain_dir(domain_name: &str) -> bool {
!domain_name.starts_with('.')
}
/// Load all certificates and private keys in the directory
/// This function loads all certificate and private key pairs from the specified directory.
/// It looks for files named `options.cert_filename` and `options.key_filename` in each subdirectory.
/// The root directory can also contain a default certificate/private key pair.
///
/// # Arguments
/// * `options` - Directory and filename options for discovering certificates and private keys.
///
/// # Returns
/// * An io::Result containing a HashMap where the keys are domain names (or "default" for the root certificate) and the values are tuples of (Vec<CertificateDer>, PrivateKeyDer). If no valid certificate/private key pairs are found, an io::Error is returned.
///
pub fn load_all_certs_from_directory(
options: CertDirectoryLoadOptions,
) -> io::Result<HashMap<String, (Vec<CertificateDer<'static>>, PrivateKeyDer<'static>)>> {
@@ -318,7 +243,6 @@ pub fn load_all_certs_from_directory(
)));
}
// 1. First check whether there is a certificate/private key pair in the root directory
let root_cert_path = dir.join(&options.cert_filename);
let root_key_path = dir.join(&options.key_filename);
@@ -332,7 +256,6 @@ pub fn load_all_certs_from_directory(
.ok_or_else(|| certs_error(format!("Invalid UTF-8 in root key path: {root_key_path:?}")))?;
match load_cert_key_pair(root_cert_str, root_key_str) {
Ok((certs, key)) => {
// The root directory certificate is used as the default certificate and is stored using special keys.
cert_key_pairs.insert("default".to_string(), (certs, key));
}
Err(e) => {
@@ -341,7 +264,6 @@ pub fn load_all_certs_from_directory(
}
}
// 2.iterate through all folders in the directory
for entry in fs::read_dir(dir)? {
let entry = entry?;
let path = entry.path();
@@ -356,9 +278,8 @@ pub fn load_all_certs_from_directory(
continue;
}
// find certificate and private key files
let cert_path = path.join(&options.cert_filename); // e.g., rustfs_cert.pem
let key_path = path.join(&options.key_filename); // e.g., rustfs_key.pem
let cert_path = path.join(&options.cert_filename);
let key_path = path.join(&options.key_filename);
if cert_path.exists() && key_path.exists() {
debug!("find the domain name certificate: {} in {:?}", domain_name, cert_path);
@@ -391,43 +312,21 @@ pub fn load_all_certs_from_directory(
}
if cert_key_pairs.is_empty() {
return Err(certs_error(format!(
"No valid certificate/private key pair found in directory {}",
dir.display()
)));
return Err(io::Error::new(
io::ErrorKind::NotFound,
format!("No valid certificate/private key pair found in directory {}", dir.display()),
));
}
Ok(cert_key_pairs)
}
/// loading a single certificate private key pair
/// This function loads a certificate and private key from the specified paths.
/// It returns a tuple containing the certificate and private key.
///
/// # Arguments
/// * `cert_path` - A string slice that holds the path to the certificate file.
/// * `key_path` - A string slice that holds the path to the private key file
///
/// # Returns
/// * An io::Result containing a tuple of (Vec<CertificateDer>, PrivateKeyDer) if successful, or an io::Error if an error occurs during loading.
///
fn load_cert_key_pair(cert_path: &str, key_path: &str) -> io::Result<(Vec<CertificateDer<'static>>, PrivateKeyDer<'static>)> {
let certs = load_certs(cert_path)?;
let key = load_private_key(key_path)?;
Ok((certs, key))
}
/// Create a multi-cert resolver
/// This function loads all certificates and private keys from the specified directory.
/// It uses the first certificate/private key pair found in the root directory as the default certificate.
/// The rest of the certificates/private keys are used for SNI resolution.
///
/// # Arguments
/// * `cert_key_pairs` - A HashMap where the keys are domain names (or "default" for the root certificate) and the values are tuples of (Vec<CertificateDer>, PrivateKeyDer).
///
/// # Returns
/// * An io::Result containing an implementation of ResolvesServerCert if successful, or an io::Error if an error occurs during loading.
///
pub fn create_multi_cert_resolver(
cert_key_pairs: HashMap<String, (Vec<CertificateDer<'static>>, PrivateKeyDer<'static>)>,
) -> io::Result<impl ResolvesServerCert> {
@@ -436,14 +335,13 @@ pub fn create_multi_cert_resolver(
cert_resolver: ResolvesServerCertUsingSni,
default_cert: Option<Arc<CertifiedKey>>,
}
impl ResolvesServerCert for MultiCertResolver {
fn resolve(&self, client_hello: ClientHello) -> Option<Arc<CertifiedKey>> {
// try matching certificates with sni
fn resolve(&self, client_hello: ClientHello<'_>) -> Option<Arc<CertifiedKey>> {
if let Some(cert) = self.cert_resolver.resolve(client_hello) {
return Some(cert);
}
// If there is no matching SNI certificate, use the default certificate
self.default_cert.clone()
}
}
@@ -452,16 +350,13 @@ pub fn create_multi_cert_resolver(
let mut default_cert = None;
for (domain, (certs, key)) in cert_key_pairs {
// create a signature
let signing_key = rustls::crypto::aws_lc_rs::sign::any_supported_type(&key)
.map_err(|e| certs_error(format!("unsupported private key types:{domain}, err:{e:?}")))?;
// create a CertifiedKey
let certified_key = CertifiedKey::new(certs, signing_key);
if domain == "default" {
default_cert = Some(Arc::new(certified_key.clone()));
} else {
// add certificate to resolver
resolver
.add(&domain, certified_key)
.map_err(|e| certs_error(format!("failed to add a domain name certificate:{domain},err: {e:?}")))?;
@@ -479,6 +374,7 @@ mod tests {
use super::*;
use std::fs;
use std::io::ErrorKind;
use std::path::PathBuf;
use tempfile::TempDir;
fn default_load_options(path: impl Into<PathBuf>) -> CertDirectoryLoadOptions {
@@ -487,9 +383,9 @@ mod tests {
fn write_test_cert_pair(dir: &std::path::Path) {
let rcgen::CertifiedKey { cert, signing_key } =
rcgen::generate_simple_self_signed(vec!["example.com".to_string()]).unwrap();
fs::write(dir.join("rustfs_cert.pem"), cert.pem()).unwrap();
fs::write(dir.join("rustfs_key.pem"), signing_key.serialize_pem()).unwrap();
rcgen::generate_simple_self_signed(vec!["example.com".to_string()]).expect("cert should generate");
fs::write(dir.join("rustfs_cert.pem"), cert.pem()).expect("cert should write");
fs::write(dir.join("rustfs_key.pem"), signing_key.serialize_pem()).expect("key should write");
}
#[test]
@@ -506,7 +402,7 @@ mod tests {
let result = load_certs("non_existent_file.pem");
assert!(result.is_err());
let error = result.unwrap_err();
let error = result.expect_err("missing cert should error");
assert_eq!(error.kind(), ErrorKind::Other);
assert!(error.to_string().contains("failed to open"));
}
@@ -516,252 +412,40 @@ mod tests {
let result = load_private_key("non_existent_key.pem");
assert!(result.is_err());
let error = result.unwrap_err();
let error = result.expect_err("missing key should error");
assert_eq!(error.kind(), ErrorKind::Other);
assert!(error.to_string().contains("failed to open"));
}
#[test]
fn test_load_certs_empty_file() {
let temp_dir = TempDir::new().unwrap();
let cert_path = temp_dir.path().join("empty.pem");
fs::write(&cert_path, "").unwrap();
let result = load_certs(cert_path.to_str().unwrap());
assert!(result.is_err());
let error = result.unwrap_err();
assert!(error.to_string().contains("No valid certificate was found"));
}
#[test]
fn test_load_certs_invalid_format() {
let temp_dir = TempDir::new().unwrap();
let cert_path = temp_dir.path().join("invalid.pem");
fs::write(&cert_path, "invalid certificate content").unwrap();
let result = load_certs(cert_path.to_str().unwrap());
assert!(result.is_err());
let error = result.unwrap_err();
assert!(error.to_string().contains("No valid certificate was found"));
}
#[test]
fn test_load_private_key_empty_file() {
let temp_dir = TempDir::new().unwrap();
let key_path = temp_dir.path().join("empty_key.pem");
fs::write(&key_path, "").unwrap();
let result = load_private_key(key_path.to_str().unwrap());
assert!(result.is_err());
let error = result.unwrap_err();
assert!(error.to_string().contains("failed to parse private key in"));
}
#[test]
fn test_load_private_key_invalid_format() {
let temp_dir = TempDir::new().unwrap();
let key_path = temp_dir.path().join("invalid_key.pem");
fs::write(&key_path, "invalid private key content").unwrap();
let result = load_private_key(key_path.to_str().unwrap());
assert!(result.is_err());
let error = result.unwrap_err();
assert!(error.to_string().contains("failed to parse private key in"));
}
#[test]
fn test_load_all_certs_from_directory_not_exists() {
let result = load_all_certs_from_directory(default_load_options("/non/existent/directory"));
assert!(result.is_err());
let error = result.unwrap_err();
assert!(error.to_string().contains("does not exist or is not a directory"));
}
#[test]
fn test_load_all_certs_from_directory_empty() {
let temp_dir = TempDir::new().unwrap();
let temp_dir = TempDir::new().expect("tempdir should create");
let result = load_all_certs_from_directory(default_load_options(temp_dir.path()));
assert!(result.is_err());
let error = result.unwrap_err();
let error = result.expect_err("empty directory should error");
assert_eq!(error.kind(), ErrorKind::NotFound);
assert!(error.to_string().contains("No valid certificate/private key pair found"));
}
#[test]
fn test_load_all_certs_from_directory_file_instead_of_dir() {
let temp_dir = TempDir::new().unwrap();
let file_path = temp_dir.path().join("not_a_directory.txt");
fs::write(&file_path, "content").unwrap();
let result = load_all_certs_from_directory(default_load_options(&file_path));
assert!(result.is_err());
let error = result.unwrap_err();
assert!(error.to_string().contains("does not exist or is not a directory"));
}
#[test]
fn test_load_cert_key_pair_missing_cert() {
let temp_dir = TempDir::new().unwrap();
let key_path = temp_dir.path().join("test_key.pem");
fs::write(&key_path, "dummy key content").unwrap();
let result = load_cert_key_pair("non_existent_cert.pem", key_path.to_str().unwrap());
assert!(result.is_err());
}
#[test]
fn test_load_cert_key_pair_missing_key() {
let temp_dir = TempDir::new().unwrap();
let cert_path = temp_dir.path().join("test_cert.pem");
fs::write(&cert_path, "dummy cert content").unwrap();
let result = load_cert_key_pair(cert_path.to_str().unwrap(), "non_existent_key.pem");
assert!(result.is_err());
}
#[test]
fn test_create_multi_cert_resolver_empty_map() {
let empty_map = HashMap::new();
let result = create_multi_cert_resolver(empty_map);
// Should succeed even with empty map
assert!(result.is_ok());
}
#[test]
fn test_error_message_formatting() {
let test_cases = vec![
("file not found", "failed to open test.pem: file not found"),
("permission denied", "failed to open key.pem: permission denied"),
("invalid format", "certificate file cert.pem format error:invalid format"),
];
for (input, _expected_pattern) in test_cases {
let error1 = certs_error(format!("failed to open test.pem: {input}"));
assert!(error1.to_string().contains(input));
let error2 = certs_error(format!("failed to open key.pem: {input}"));
assert!(error2.to_string().contains(input));
}
}
#[test]
fn test_path_handling_edge_cases() {
// Test with various path formats
let path_cases = vec![
"", // Empty path
".", // Current directory
"..", // Parent directory
"/", // Root directory (Unix)
"relative/path", // Relative path
"/absolute/path", // Absolute path
];
for path in path_cases {
let result = load_all_certs_from_directory(default_load_options(path));
// All should fail since these are not valid cert directories
assert!(result.is_err());
}
}
#[test]
fn test_directory_structure_validation() {
let temp_dir = TempDir::new().unwrap();
// Create a subdirectory without certificates
let sub_dir = temp_dir.path().join("example.com");
fs::create_dir(&sub_dir).unwrap();
// Should fail because no certificates found
let result = load_all_certs_from_directory(default_load_options(temp_dir.path()));
assert!(result.is_err());
assert!(
result
.unwrap_err()
.to_string()
.contains("No valid certificate/private key pair found")
);
}
#[test]
fn test_load_all_certs_skips_kubernetes_secret_projection_dirs() {
let temp_dir = TempDir::new().unwrap();
let temp_dir = TempDir::new().expect("tempdir should create");
write_test_cert_pair(temp_dir.path());
let domain_dir = temp_dir.path().join("example.com");
fs::create_dir(&domain_dir).unwrap();
fs::create_dir(&domain_dir).expect("domain dir should create");
write_test_cert_pair(&domain_dir);
for internal_dir_name in ["..data", "..2026_04_28_18_33_53.4209048473"] {
let internal_dir = temp_dir.path().join(internal_dir_name);
fs::create_dir(&internal_dir).unwrap();
fs::create_dir(&internal_dir).expect("internal dir should create");
write_test_cert_pair(&internal_dir);
}
let certs = load_all_certs_from_directory(default_load_options(temp_dir.path())).unwrap();
let certs = load_all_certs_from_directory(default_load_options(temp_dir.path())).expect("certs should load");
assert!(certs.contains_key("default"));
assert!(certs.contains_key("example.com"));
assert!(!certs.contains_key("..data"));
assert!(!certs.contains_key("..2026_04_28_18_33_53.4209048473"));
assert_eq!(certs.len(), 2);
}
#[test]
fn test_unicode_path_handling() {
let temp_dir = TempDir::new().unwrap();
// Create directory with Unicode characters
let unicode_dir = temp_dir.path().join("test_directory");
fs::create_dir(&unicode_dir).unwrap();
let result = load_all_certs_from_directory(default_load_options(&unicode_dir));
assert!(result.is_err());
assert!(
result
.unwrap_err()
.to_string()
.contains("No valid certificate/private key pair found")
);
}
#[test]
fn test_concurrent_access_safety() {
use std::sync::Arc;
use std::thread;
let temp_dir = TempDir::new().unwrap();
let dir_path = Arc::new(temp_dir.path().to_string_lossy().to_string());
let handles: Vec<_> = (0..5)
.map(|_| {
let path = Arc::clone(&dir_path);
thread::spawn(move || {
let result = load_all_certs_from_directory(default_load_options(path.as_str()));
// All should fail since directory is empty
assert!(result.is_err());
})
})
.collect();
for handle in handles {
handle.join().expect("Thread should complete successfully");
}
}
#[test]
fn test_memory_efficiency() {
let error = certs_error("test".to_string());
let error_size = std::mem::size_of_val(&error);
// Error should not be excessively large
assert!(error_size < 1024, "Error size should be reasonable, got {error_size} bytes");
}
}
crates/tls-runtime/src/config.rs
+51
View File
@@ -0,0 +1,51 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::time::Duration;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ReloadDetectMode {
Poll,
Watch, // TODO: implement fs::watch-based reload
Hybrid, // TODO: implement poll + fs::watch hybrid
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ReloadApplyHint {
Lazy,
SoftReconnect,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct TlsReloadOptions {
pub enabled: bool,
pub detect_mode: ReloadDetectMode,
pub interval: Duration,
pub debounce: Duration,
pub min_stable_age: Duration,
pub apply_hint: ReloadApplyHint,
}
impl Default for TlsReloadOptions {
fn default() -> Self {
Self {
enabled: true,
detect_mode: ReloadDetectMode::Poll,
interval: Duration::from_secs(15),
debounce: Duration::from_secs(2),
min_stable_age: Duration::from_secs(1),
apply_hint: ReloadApplyHint::Lazy,
}
}
}
crates/tls-runtime/src/coordinator.rs
+226
View File
@@ -0,0 +1,226 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::config::{ReloadDetectMode, TlsReloadOptions};
use crate::error::TlsRuntimeError;
use crate::material::TlsMaterialSnapshot;
use crate::metrics::{
TLS_RUNTIME_FOUNDATION_CONSUMER, record_tls_generation, record_tls_publication_fail, record_tls_reload_result,
record_tls_reload_skipped,
};
use crate::source::TlsSource;
use crate::state::{
TlsGeneration, TlsPublishedState, TlsReloadRuntimeState, TlsRuntimeConsumerSection, TlsRuntimeOutboundSection,
TlsRuntimeRuntimeSection, TlsRuntimeServerSection, TlsRuntimeStatusSnapshot, detect_mode_label,
};
use std::sync::Arc;
use std::time::{SystemTime, UNIX_EPOCH};
use tokio::task::JoinHandle;
use tracing::{debug, info, warn};
pub trait TlsConsumer<M>: Send + Sync + 'static {
fn on_publish(&self, generation: TlsGeneration, state: Arc<TlsPublishedState<M>>) -> Result<(), TlsRuntimeError>;
}
#[derive(Debug)]
pub struct TlsReloadCoordinator {
source: TlsSource,
options: TlsReloadOptions,
}
impl TlsReloadCoordinator {
pub fn new(source: TlsSource, options: TlsReloadOptions) -> Self {
Self { source, options }
}
pub fn source(&self) -> &TlsSource {
&self.source
}
pub fn options(&self) -> &TlsReloadOptions {
&self.options
}
pub async fn status_snapshot(&self, runtime_state: &TlsReloadRuntimeState<TlsMaterialSnapshot>) -> TlsRuntimeStatusSnapshot {
let current = runtime_state.current.load();
let last_attempt = runtime_state.last_attempt_unix_ms();
let last_success = runtime_state.last_success_unix_ms();
TlsRuntimeStatusSnapshot {
runtime: TlsRuntimeRuntimeSection {
generation: current.generation.0,
reload_enabled: self.options.enabled,
detect_mode: detect_mode_label(self.options.detect_mode),
last_attempt_time: (last_attempt != 0).then_some(last_attempt),
last_success_time: (last_success != 0).then_some(last_success),
last_error: runtime_state.last_error.read().await.clone(),
source_path: self.source.base_dir.display().to_string(),
},
outbound: TlsRuntimeOutboundSection {
has_roots: !current.material.outbound.root_ca_pem.is_empty(),
has_mtls_identity: current.material.outbound.mtls_identity.is_some(),
},
server: TlsRuntimeServerSection {
has_material: current.material.server.is_some(),
},
consumer: TlsRuntimeConsumerSection { stale_generation: false },
}
}
pub async fn load_initial_snapshot(&self) -> Result<TlsMaterialSnapshot, TlsRuntimeError> {
TlsMaterialSnapshot::load(&self.source).await
}
pub async fn publish_initial_state(&self, snapshot: TlsMaterialSnapshot) -> Arc<TlsPublishedState<TlsMaterialSnapshot>> {
let published = Arc::new(TlsPublishedState {
generation: TlsGeneration(1),
fingerprint: snapshot.fingerprint.clone(),
material: Arc::new(snapshot),
loaded_at_unix_ms: unix_time_ms(),
});
record_tls_generation(TLS_RUNTIME_FOUNDATION_CONSUMER, published.generation.0);
published
}
pub async fn reload_once<C>(
&self,
runtime_state: &TlsReloadRuntimeState<TlsMaterialSnapshot>,
consumer: &C,
) -> Result<Option<Arc<TlsPublishedState<TlsMaterialSnapshot>>>, TlsRuntimeError>
where
C: TlsConsumer<TlsMaterialSnapshot>,
{
runtime_state.mark_attempt(unix_time_ms());
let started_at = std::time::Instant::now();
let snapshot = self.load_initial_snapshot().await?;
let current = runtime_state.current.load();
if current.fingerprint == snapshot.fingerprint {
debug!(source = %self.source.base_dir.display(), "TLS material unchanged; skipping publication");
record_tls_reload_skipped(TLS_RUNTIME_FOUNDATION_CONSUMER, "unchanged");
return Ok(None);
}
let published = Arc::new(TlsPublishedState {
generation: runtime_state.bump_generation(),
fingerprint: snapshot.fingerprint.clone(),
material: Arc::new(snapshot),
loaded_at_unix_ms: unix_time_ms(),
});
if let Err(err) = consumer.on_publish(published.generation, published.clone()) {
record_tls_publication_fail(TLS_RUNTIME_FOUNDATION_CONSUMER);
return Err(err);
}
runtime_state.current.store(published.clone());
runtime_state.last_good.store(published.clone());
runtime_state.mark_success(unix_time_ms());
*runtime_state.last_error.write().await = None;
record_tls_reload_result(
TLS_RUNTIME_FOUNDATION_CONSUMER,
"ok",
Some(started_at.elapsed().as_secs_f64()),
Some(published.generation.0),
);
Ok(Some(published))
}
pub fn spawn_poll_loop<C>(
self: Arc<Self>,
runtime_state: Arc<TlsReloadRuntimeState<TlsMaterialSnapshot>>,
consumer: Arc<C>,
) -> Option<JoinHandle<()>>
where
C: TlsConsumer<TlsMaterialSnapshot>,
{
if !self.options.enabled {
debug!(source = %self.source.base_dir.display(), "TLS reload disabled; poll loop not started");
return None;
}
if !matches!(self.options.detect_mode, ReloadDetectMode::Poll | ReloadDetectMode::Hybrid) {
debug!(source = %self.source.base_dir.display(), "TLS poll loop skipped for non-poll detect mode");
return None;
}
let interval_duration = self.options.interval;
info!(
source = %self.source.base_dir.display(),
interval_secs = interval_duration.as_secs(),
"TLS poll reload loop enabled"
);
Some(tokio::spawn(async move {
let mut interval = tokio::time::interval(interval_duration);
interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
interval.tick().await;
loop {
interval.tick().await;
if let Err(err) = self.reload_once(runtime_state.as_ref(), consumer.as_ref()).await {
warn!(
source = %self.source.base_dir.display(),
error = %err,
"TLS reload failed (will retry)"
);
*runtime_state.last_error.write().await = Some(err.to_string());
}
}
}))
}
}
fn unix_time_ms() -> u64 {
SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_millis() as u64
}
#[cfg(test)]
mod tests {
use super::*;
use crate::source::TlsSource;
use std::sync::Arc;
struct NopConsumer;
impl TlsConsumer<crate::material::TlsMaterialSnapshot> for NopConsumer {
fn on_publish(
&self,
_generation: TlsGeneration,
_state: Arc<TlsPublishedState<crate::material::TlsMaterialSnapshot>>,
) -> Result<(), TlsRuntimeError> {
Ok(())
}
}
#[tokio::test]
async fn reload_once_skips_when_fingerprint_unchanged() {
let temp = tempfile::tempdir().expect("tempdir");
let source = TlsSource::from_directory(temp.path().to_path_buf());
let options = TlsReloadOptions::default();
let coordinator = TlsReloadCoordinator::new(source.clone(), options);
// Load the actual snapshot from the (empty) temp dir so its fingerprint
// matches what reload_once will observe on the next load.
let initial_snapshot = coordinator.load_initial_snapshot().await.expect("initial load");
let initial = coordinator.publish_initial_state(initial_snapshot).await;
let runtime_state = TlsReloadRuntimeState::new(initial);
let consumer = NopConsumer;
let result = coordinator.reload_once(&runtime_state, &consumer).await;
// Fingerprint has not changed → should skip and return Ok(None).
assert!(result.is_ok(), "reload_once should succeed: {:?}", result.err());
assert!(result.unwrap().is_none(), "should skip when fingerprint unchanged");
}
}
crates/tls-runtime/src/debug.rs
+106
View File
@@ -0,0 +1,106 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::state::TlsRuntimeStatusSnapshot;
use serde::Serialize;
#[derive(Debug, Clone, Serialize, PartialEq, Eq)]
pub struct TlsConsumerStatusItem {
pub consumer: &'static str,
pub generation: u64,
pub has_root_ca: bool,
pub has_mtls_identity: bool,
}
#[derive(Debug, Clone, Serialize, PartialEq, Eq)]
pub struct TlsDebugStatusResponse {
pub foundation: TlsRuntimeStatusSnapshot,
pub consumers: Vec<TlsConsumerStatusItem>,
}
#[derive(Debug, Clone)]
pub struct TlsDebugStatusResponseBuilder {
foundation: TlsRuntimeStatusSnapshot,
consumers: Vec<TlsConsumerStatusItem>,
}
impl TlsDebugStatusResponse {
pub fn builder(foundation: TlsRuntimeStatusSnapshot) -> TlsDebugStatusResponseBuilder {
TlsDebugStatusResponseBuilder {
foundation,
consumers: Vec::new(),
}
}
}
impl TlsDebugStatusResponseBuilder {
pub fn push_consumers<I>(mut self, sources: I) -> Self
where
I: IntoIterator<Item = TlsConsumerStatusItem>,
{
self.consumers.extend(sources);
self
}
pub fn build(self) -> TlsDebugStatusResponse {
TlsDebugStatusResponse {
foundation: self.foundation,
consumers: self.consumers,
}
}
}
#[cfg(test)]
mod tests {
use super::{TlsConsumerStatusItem, TlsDebugStatusResponse};
use crate::state::{
TlsRuntimeConsumerSection, TlsRuntimeOutboundSection, TlsRuntimeRuntimeSection, TlsRuntimeServerSection,
TlsRuntimeStatusSnapshot,
};
#[test]
fn builder_produces_structured_response() {
let foundation = TlsRuntimeStatusSnapshot {
runtime: TlsRuntimeRuntimeSection {
generation: 5,
reload_enabled: true,
detect_mode: "poll",
last_attempt_time: Some(1),
last_success_time: Some(2),
last_error: None,
source_path: "/tmp/tls".to_string(),
},
outbound: TlsRuntimeOutboundSection {
has_roots: true,
has_mtls_identity: false,
},
server: TlsRuntimeServerSection { has_material: true },
consumer: TlsRuntimeConsumerSection { stale_generation: false },
};
let response = TlsDebugStatusResponse::builder(foundation)
.push_consumers([TlsConsumerStatusItem {
consumer: "test_consumer",
generation: 7,
has_root_ca: true,
has_mtls_identity: false,
}])
.build();
let json = serde_json::to_value(response).expect("response should serialize");
assert!(json.get("foundation").is_some());
assert!(json.get("consumers").is_some());
assert!(json["consumers"].is_array());
}
}
crates/tls-runtime/src/error.rs
+32
View File
@@ -0,0 +1,32 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::path::PathBuf;
use thiserror::Error;
#[derive(Debug, Error)]
pub enum TlsRuntimeError {
#[error("TLS source path is empty")]
EmptySourcePath,
#[error("TLS directory does not exist: {path}")]
DirectoryNotFound { path: PathBuf },
#[error("TLS path is not a directory: {path}")]
NotADirectory { path: PathBuf },
#[error("TLS material error: {0}")]
Material(String),
#[error("TLS publication error: {0}")]
Publication(String),
#[error("I/O error: {0}")]
Io(#[from] std::io::Error),
}
crates/tls-runtime/src/fingerprint.rs
+48
View File
@@ -0,0 +1,48 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use sha2::{Digest, Sha256};
#[derive(Debug, Clone, PartialEq, Eq, Default)]
pub struct TlsFingerprint {
pub server_sha256: Option<[u8; 32]>,
pub public_ca_sha256: Option<[u8; 32]>,
pub client_ca_sha256: Option<[u8; 32]>,
pub client_cert_sha256: Option<[u8; 32]>,
pub client_key_sha256: Option<[u8; 32]>,
}
fn digest_bytes(bytes: &[u8]) -> [u8; 32] {
let mut hasher = Sha256::new();
hasher.update(bytes);
hasher.finalize().into()
}
impl TlsFingerprint {
pub fn from_optional_bytes(
server: Option<&[u8]>,
public_ca: Option<&[u8]>,
client_ca: Option<&[u8]>,
client_cert: Option<&[u8]>,
client_key: Option<&[u8]>,
) -> Self {
Self {
server_sha256: server.map(digest_bytes),
public_ca_sha256: public_ca.map(digest_bytes),
client_ca_sha256: client_ca.map(digest_bytes),
client_cert_sha256: client_cert.map(digest_bytes),
client_key_sha256: client_key.map(digest_bytes),
}
}
}
crates/tls-runtime/src/lib.rs
+126
View File
@@ -0,0 +1,126 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
pub mod certs;
pub mod config;
pub mod coordinator;
pub mod debug;
pub mod error;
pub mod fingerprint;
pub mod material;
pub mod metrics;
pub mod outbound;
pub mod server;
pub mod source;
pub mod state;
pub use certs::{
CertDirectoryLoadOptions, WebPkiClientVerifierOptions, build_webpki_client_verifier, create_multi_cert_resolver,
load_all_certs_from_directory, load_cert_bundle_der_bytes, load_certs, load_private_key,
};
pub use config::{ReloadApplyHint, ReloadDetectMode, TlsReloadOptions};
pub use coordinator::{TlsConsumer, TlsReloadCoordinator};
pub use debug::{TlsConsumerStatusItem, TlsDebugStatusResponse, TlsDebugStatusResponseBuilder};
pub use error::TlsRuntimeError;
pub use fingerprint::TlsFingerprint;
pub use material::{OutboundTlsMaterial, ServerTlsMaterial, TlsMaterialSnapshot};
pub use metrics::{
TLS_OUTBOUND_GLOBAL_CONSUMER, TLS_RUNTIME_FOUNDATION_CONSUMER, init_tls_metrics, record_tls_consumer_stale_generation,
record_tls_generation, record_tls_publication_fail, record_tls_reload_result, record_tls_reload_skipped,
};
pub use outbound::{
GlobalOutboundTlsStateSummary, GlobalPublishedOutboundTlsState, load_global_outbound_tls_generation,
load_global_outbound_tls_state, publish_global_outbound_tls_state, summarize_global_outbound_tls_state,
};
pub use server::{ReloadableServerCertResolver, spawn_server_cert_reload_loop};
pub use source::{TlsFileLayout, TlsSource, TlsSourceKind};
pub use state::OutboundOnlySnapshotArgs;
pub use state::{TlsGeneration, TlsPublishedState, TlsReloadRuntimeState, TlsRuntimeStatusSnapshot};
pub use state::{TlsRuntimeConsumerSection, TlsRuntimeOutboundSection, TlsRuntimeRuntimeSection, TlsRuntimeServerSection};
#[cfg(test)]
mod tests {
use super::*;
use rcgen::generate_simple_self_signed;
use std::collections::HashMap;
#[test]
fn tls_source_requires_existing_directory() {
let source = TlsSource::from_directory("/definitely/missing/rustfs/tls-runtime-test");
let err = source.validate_directory().expect_err("missing directory should fail");
assert!(matches!(err, TlsRuntimeError::DirectoryNotFound { .. }));
}
#[test]
fn fingerprint_changes_when_server_material_changes() {
let cert_a = generate_simple_self_signed(vec!["a.example.com".to_string()]).expect("cert A should generate");
let cert_b = generate_simple_self_signed(vec!["b.example.com".to_string()]).expect("cert B should generate");
let single_a = ServerTlsMaterial::SingleCert {
certs: vec![cert_a.cert.der().clone()],
key: rustls::pki_types::PrivateKeyDer::try_from(cert_a.signing_key.serialize_der()).expect("key A should convert"),
};
let single_b = ServerTlsMaterial::SingleCert {
certs: vec![cert_b.cert.der().clone()],
key: rustls::pki_types::PrivateKeyDer::try_from(cert_b.signing_key.serialize_der()).expect("key B should convert"),
};
let bytes_a = match &single_a {
ServerTlsMaterial::SingleCert { certs, key } => {
let mut bytes = Vec::new();
for cert in certs {
bytes.extend_from_slice(cert.as_ref());
}
bytes.extend_from_slice(key.secret_der());
bytes
}
ServerTlsMaterial::MultiCert { .. } => unreachable!(),
};
let bytes_b = match &single_b {
ServerTlsMaterial::SingleCert { certs, key } => {
let mut bytes = Vec::new();
for cert in certs {
bytes.extend_from_slice(cert.as_ref());
}
bytes.extend_from_slice(key.secret_der());
bytes
}
ServerTlsMaterial::MultiCert { .. } => unreachable!(),
};
let fp_a = TlsFingerprint::from_optional_bytes(Some(&bytes_a), None, None, None, None);
let fp_b = TlsFingerprint::from_optional_bytes(Some(&bytes_b), None, None, None, None);
assert_ne!(fp_a, fp_b);
}
#[tokio::test]
async fn coordinator_can_publish_initial_state() {
let source = TlsSource::from_directory(std::env::temp_dir());
let coordinator = TlsReloadCoordinator::new(source.clone(), TlsReloadOptions::default());
let snapshot = TlsMaterialSnapshot {
source,
server: Some(ServerTlsMaterial::MultiCert {
cert_key_pairs: HashMap::new(),
}),
outbound: OutboundTlsMaterial {
root_ca_pem: Vec::new(),
mtls_identity: None,
},
fingerprint: TlsFingerprint::default(),
};
let published = coordinator.publish_initial_state(snapshot).await;
assert_eq!(published.generation, TlsGeneration(1));
}
}
crates/tls-runtime/src/material.rs
+200
View File
@@ -0,0 +1,200 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::certs::{CertDirectoryLoadOptions, load_all_certs_from_directory, load_certs, load_private_key};
use crate::error::TlsRuntimeError;
use crate::fingerprint::TlsFingerprint;
use crate::source::TlsSource;
use rustfs_common::MtlsIdentityPem;
use rustls::pki_types::{CertificateDer, PrivateKeyDer, pem::PemObject};
use std::collections::HashMap;
use std::io::Cursor;
use std::io::ErrorKind;
use std::path::Path;
#[derive(Debug, Clone)]
pub struct OutboundTlsMaterial {
pub root_ca_pem: Vec<u8>,
pub mtls_identity: Option<MtlsIdentityPem>,
}
#[derive(Debug)]
pub enum ServerTlsMaterial {
SingleCert {
certs: Vec<CertificateDer<'static>>,
key: PrivateKeyDer<'static>,
},
MultiCert {
cert_key_pairs: HashMap<String, (Vec<CertificateDer<'static>>, PrivateKeyDer<'static>)>,
},
}
#[derive(Debug)]
pub struct TlsMaterialSnapshot {
pub source: TlsSource,
pub server: Option<ServerTlsMaterial>,
pub outbound: OutboundTlsMaterial,
pub fingerprint: TlsFingerprint,
}
impl TlsMaterialSnapshot {
pub async fn load(source: &TlsSource) -> Result<Self, TlsRuntimeError> {
let base_dir = source.validate_directory()?.to_path_buf();
let server = load_server_material(source, &base_dir)?;
let public_ca_path = base_dir.join(&source.layout.public_ca_filename);
let client_ca_path = base_dir.join(&source.fallback_ca_filename);
let client_cert_path = base_dir.join(&source.layout.client_cert_filename);
let client_key_path = base_dir.join(&source.layout.client_key_filename);
let public_ca_pem = tokio::fs::read(&public_ca_path).await.ok();
let client_ca_pem = tokio::fs::read(&client_ca_path).await.ok();
let root_ca_pem = combine_optional_pem(public_ca_pem.as_deref(), client_ca_pem.as_deref());
let mtls_identity = match (
tokio::fs::read(&client_cert_path).await.ok(),
tokio::fs::read(&client_key_path).await.ok(),
) {
(Some(cert_pem), Some(key_pem)) => {
let mut cert_reader = Cursor::new(&cert_pem);
if CertificateDer::pem_reader_iter(&mut cert_reader).next().is_none() {
return Err(TlsRuntimeError::Material("no valid certificate in client cert PEM".to_string()));
}
let mut key_reader = Cursor::new(&key_pem);
PrivateKeyDer::from_pem_reader(&mut key_reader)
.map_err(|e| TlsRuntimeError::Material(format!("invalid client key PEM: {e}")))?;
Some(MtlsIdentityPem { cert_pem, key_pem })
}
_ => None,
};
let outbound = OutboundTlsMaterial {
root_ca_pem: root_ca_pem.clone(),
mtls_identity: mtls_identity.clone(),
};
let server_fingerprint_bytes = server.as_ref().map(serialize_server_material_for_fingerprint);
let fingerprint = TlsFingerprint::from_optional_bytes(
server_fingerprint_bytes.as_deref(),
public_ca_pem.as_deref(),
client_ca_pem.as_deref(),
mtls_identity.as_ref().map(|identity| identity.cert_pem.as_slice()),
mtls_identity.as_ref().map(|identity| identity.key_pem.as_slice()),
);
Ok(Self {
source: source.clone(),
server,
outbound,
fingerprint,
})
}
}
fn load_server_material(source: &TlsSource, base_dir: &Path) -> Result<Option<ServerTlsMaterial>, TlsRuntimeError> {
let root_cert = base_dir.join(&source.layout.server_cert_filename);
let root_key = base_dir.join(&source.layout.server_key_filename);
let has_root_pair = root_cert.exists() && root_key.exists();
let cert_key_pairs = load_all_certs_from_directory(
CertDirectoryLoadOptions::builder(base_dir, &source.layout.server_cert_filename, &source.layout.server_key_filename)
.build(),
);
match cert_key_pairs {
Ok(cert_key_pairs) if cert_key_pairs.len() > 1 || cert_key_pairs.keys().any(|key| key != "default") => {
Ok(Some(ServerTlsMaterial::MultiCert { cert_key_pairs }))
}
Ok(cert_key_pairs) if !cert_key_pairs.is_empty() => {
if let Some((certs, key)) = cert_key_pairs.get("default") {
return Ok(Some(ServerTlsMaterial::SingleCert {
certs: certs.clone(),
key: key.clone_key(),
}));
}
Ok(Some(ServerTlsMaterial::MultiCert { cert_key_pairs }))
}
Ok(_) => Ok(None),
Err(_err) if has_root_pair => {
let root_cert_path = path_to_utf8_str(&root_cert, "root TLS certificate")?;
let root_key_path = path_to_utf8_str(&root_key, "root TLS private key")?;
let certs = load_certs(root_cert_path)
.map_err(|e| TlsRuntimeError::Material(format!("load root TLS certificate {}: {e}", root_cert.display())))?;
let key = load_private_key(root_key_path)
.map_err(|e| TlsRuntimeError::Material(format!("load root TLS private key {}: {e}", root_key.display())))?;
Ok(Some(ServerTlsMaterial::SingleCert { certs, key }))
}
Err(err) if err.kind() == ErrorKind::NotFound => Ok(None),
Err(err) => Err(TlsRuntimeError::Material(format!(
"discover server TLS certificates under '{}': {err}",
base_dir.display()
))),
}
}
fn path_to_utf8_str<'a>(path: &'a Path, description: &str) -> Result<&'a str, TlsRuntimeError> {
path.to_str()
.ok_or_else(|| TlsRuntimeError::Material(format!("{description} path '{}' is not valid UTF-8", path.display())))
}
fn combine_optional_pem(primary: Option<&[u8]>, fallback: Option<&[u8]>) -> Vec<u8> {
let mut combined = Vec::new();
for pem in [primary, fallback].into_iter().flatten() {
if pem.iter().all(|&b| b.is_ascii_whitespace()) {
continue;
}
combined.extend_from_slice(pem);
if !combined.ends_with(b"\n") {
combined.push(b'\n');
}
}
combined
}
fn serialize_server_material_for_fingerprint(material: &ServerTlsMaterial) -> Vec<u8> {
match material {
ServerTlsMaterial::SingleCert { certs, key } => {
let mut bytes = Vec::new();
for cert in certs {
bytes.extend_from_slice(cert.as_ref());
}
bytes.extend_from_slice(key.secret_der());
bytes
}
ServerTlsMaterial::MultiCert { cert_key_pairs } => {
let mut entries = cert_key_pairs.iter().collect::<Vec<_>>();
entries.sort_by_key(|(left, _)| *left);
let mut bytes = Vec::new();
for (domain, (certs, key)) in entries {
bytes.extend_from_slice(domain.as_bytes());
for cert in certs {
bytes.extend_from_slice(cert.as_ref());
}
bytes.extend_from_slice(key.secret_der());
}
bytes
}
}
}
pub(crate) fn server_material_fingerprint(material: &ServerTlsMaterial) -> TlsFingerprint {
let bytes = serialize_server_material_for_fingerprint(material);
TlsFingerprint::from_optional_bytes(Some(&bytes), None, None, None, None)
}
crates/tls-runtime/src/metrics.rs
+88
View File
@@ -0,0 +1,88 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use metrics::{counter, describe_counter, describe_gauge, describe_histogram, gauge, histogram};
pub const TLS_RUNTIME_FOUNDATION_CONSUMER: &str = "tls_runtime_foundation";
pub const TLS_OUTBOUND_GLOBAL_CONSUMER: &str = "outbound_global";
const CONSUMER_LABEL: &str = "consumer";
const RESULT_LABEL: &str = "result";
const REASON_LABEL: &str = "reason";
const TLS_OUTBOUND_PUBLICATIONS_TOTAL: &str = "rustfs_tls_outbound_publications_total";
const TLS_OUTBOUND_GENERATION: &str = "rustfs_tls_outbound_generation";
const TLS_OUTBOUND_HAS_ROOT_CA: &str = "rustfs_tls_outbound_has_root_ca";
const TLS_OUTBOUND_HAS_MTLS_IDENTITY: &str = "rustfs_tls_outbound_has_mtls_identity";
const TLS_GENERATION: &str = "rustfs_tls_generation";
const TLS_RELOAD_TOTAL: &str = "rustfs_tls_reload_total";
const TLS_RELOAD_DURATION_SECONDS: &str = "rustfs_tls_reload_duration_seconds";
const TLS_RELOAD_GENERATION: &str = "rustfs_tls_reload_generation";
const TLS_RELOAD_SKIPPED_TOTAL: &str = "rustfs_tls_reload_skipped_total";
const TLS_PUBLICATION_FAIL_TOTAL: &str = "rustfs_tls_publication_fail_total";
const TLS_CONSUMER_STALE_GENERATION_TOTAL: &str = "rustfs_tls_consumer_stale_generation_total";
pub fn record_outbound_tls_publication(generation: u64, has_root_ca: bool, has_mtls_identity: bool) {
counter!(TLS_OUTBOUND_PUBLICATIONS_TOTAL, "result" => "ok").increment(1);
gauge!(TLS_OUTBOUND_GENERATION).set(generation as f64);
gauge!(TLS_OUTBOUND_HAS_ROOT_CA).set(if has_root_ca { 1.0 } else { 0.0 });
gauge!(TLS_OUTBOUND_HAS_MTLS_IDENTITY).set(if has_mtls_identity { 1.0 } else { 0.0 });
record_tls_generation(TLS_OUTBOUND_GLOBAL_CONSUMER, generation);
}
pub fn record_tls_generation(consumer: &'static str, generation: u64) {
gauge!(TLS_GENERATION, CONSUMER_LABEL => consumer).set(generation as f64);
}
pub fn record_tls_reload_result(
consumer: &'static str,
result: &'static str,
duration_secs: Option<f64>,
generation: Option<u64>,
) {
counter!(TLS_RELOAD_TOTAL, CONSUMER_LABEL => consumer, RESULT_LABEL => result).increment(1);
if let Some(duration_secs) = duration_secs {
histogram!(TLS_RELOAD_DURATION_SECONDS, CONSUMER_LABEL => consumer).record(duration_secs);
}
if let Some(generation) = generation {
gauge!(TLS_RELOAD_GENERATION, CONSUMER_LABEL => consumer).set(generation as f64);
record_tls_generation(consumer, generation);
}
}
pub fn record_tls_reload_skipped(consumer: &'static str, reason: &'static str) {
counter!(TLS_RELOAD_SKIPPED_TOTAL, CONSUMER_LABEL => consumer, REASON_LABEL => reason).increment(1);
}
pub fn record_tls_publication_fail(consumer: &'static str) {
counter!(TLS_PUBLICATION_FAIL_TOTAL, CONSUMER_LABEL => consumer).increment(1);
}
pub fn record_tls_consumer_stale_generation(consumer: &'static str) {
counter!(TLS_CONSUMER_STALE_GENERATION_TOTAL, CONSUMER_LABEL => consumer).increment(1);
}
pub fn init_tls_metrics() {
describe_counter!(TLS_OUTBOUND_PUBLICATIONS_TOTAL, "Total TLS outbound publications, labeled by result.");
describe_gauge!(TLS_OUTBOUND_GENERATION, "Current outbound TLS generation.");
describe_gauge!(TLS_OUTBOUND_HAS_ROOT_CA, "Whether outbound TLS roots are configured.");
describe_gauge!(TLS_OUTBOUND_HAS_MTLS_IDENTITY, "Whether outbound mTLS identity is configured.");
describe_gauge!(TLS_GENERATION, "Current TLS generation by consumer.");
describe_counter!(TLS_RELOAD_TOTAL, "Total TLS reload attempts, labeled by consumer and result.");
describe_histogram!(TLS_RELOAD_DURATION_SECONDS, "TLS reload duration by consumer (seconds).");
describe_gauge!(TLS_RELOAD_GENERATION, "TLS generation after reload by consumer.");
describe_counter!(TLS_RELOAD_SKIPPED_TOTAL, "Total skipped TLS reloads, labeled by consumer and reason.");
describe_counter!(TLS_PUBLICATION_FAIL_TOTAL, "Total TLS publication failures by consumer.");
describe_counter!(TLS_CONSUMER_STALE_GENERATION_TOTAL, "Total stale TLS consumer generations observed.");
}
crates/tls-runtime/src/outbound.rs
+67
View File
@@ -0,0 +1,67 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::material::OutboundTlsMaterial;
use crate::metrics::record_outbound_tls_publication;
use crate::state::TlsGeneration;
use rustfs_common::{
GLOBAL_MTLS_IDENTITY, GLOBAL_ROOT_CERT, MtlsIdentityPem, get_global_outbound_tls_generation, set_global_mtls_identity,
set_global_outbound_tls_generation, set_global_root_cert,
};
#[derive(Debug, Clone)]
pub struct GlobalPublishedOutboundTlsState {
pub generation: TlsGeneration,
pub root_ca_pem: Option<Vec<u8>>,
pub mtls_identity: Option<MtlsIdentityPem>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct GlobalOutboundTlsStateSummary {
pub generation: TlsGeneration,
pub has_root_ca: bool,
pub has_mtls_identity: bool,
}
pub async fn publish_global_outbound_tls_state(generation: TlsGeneration, material: &OutboundTlsMaterial) {
if !material.root_ca_pem.is_empty() {
set_global_root_cert(material.root_ca_pem.clone()).await;
} else {
*GLOBAL_ROOT_CERT.write().await = None;
}
set_global_mtls_identity(material.mtls_identity.clone()).await;
set_global_outbound_tls_generation(generation.0);
record_outbound_tls_publication(generation.0, !material.root_ca_pem.is_empty(), material.mtls_identity.is_some());
}
pub async fn load_global_outbound_tls_state() -> GlobalPublishedOutboundTlsState {
GlobalPublishedOutboundTlsState {
generation: TlsGeneration(get_global_outbound_tls_generation()),
root_ca_pem: GLOBAL_ROOT_CERT.read().await.clone(),
mtls_identity: GLOBAL_MTLS_IDENTITY.read().await.clone(),
}
}
pub fn load_global_outbound_tls_generation() -> TlsGeneration {
TlsGeneration(get_global_outbound_tls_generation())
}
pub async fn summarize_global_outbound_tls_state() -> GlobalOutboundTlsStateSummary {
let state = load_global_outbound_tls_state().await;
GlobalOutboundTlsStateSummary {
generation: state.generation,
has_root_ca: state.root_ca_pem.as_ref().is_some_and(|pem| !pem.is_empty()),
has_mtls_identity: state.mtls_identity.is_some(),
}
}
crates/protocols/src/tls_hot_reload.rs → crates/tls-runtime/src/server.rs
+90 -81
View File
@@ -12,19 +12,19 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use rustfs_config::{
DEFAULT_TLS_RELOAD_ENABLE, DEFAULT_TLS_RELOAD_INTERVAL, ENV_TLS_RELOAD_ENABLE, ENV_TLS_RELOAD_INTERVAL, RUSTFS_TLS_CERT,
RUSTFS_TLS_KEY,
};
use crate::certs::{CertDirectoryLoadOptions, load_all_certs_from_directory};
use crate::config::TlsReloadOptions;
use crate::error::TlsRuntimeError;
use crate::material::{ServerTlsMaterial, server_material_fingerprint};
use crate::metrics::{record_tls_generation, record_tls_publication_fail, record_tls_reload_result, record_tls_reload_skipped};
use crate::source::TlsSource;
use rustls::pki_types::{CertificateDer, PrivateKeyDer};
use rustls::server::{ClientHello, ResolvesServerCert, ResolvesServerCertUsingSni};
use rustls::sign::CertifiedKey;
use std::collections::HashMap;
use std::collections::hash_map::DefaultHasher;
use std::hash::Hasher;
use std::io::{self, Error};
use std::io;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Arc, RwLock};
use std::time::Duration;
use tokio::sync::watch;
use tokio::task::JoinHandle;
use tokio::time::MissedTickBehavior;
@@ -35,16 +35,18 @@ struct ResolverState {
cert_resolver: ResolvesServerCertUsingSni,
default_cert: Option<Arc<CertifiedKey>>,
cert_count: usize,
fingerprint: u64,
fingerprint: crate::fingerprint::TlsFingerprint,
}
impl ResolverState {
fn load_from_directory(cert_dir: &str) -> io::Result<Self> {
let cert_key_pairs = rustfs_utils::load_all_certs_from_directory(
rustfs_utils::CertDirectoryLoadOptions::builder(cert_dir, RUSTFS_TLS_CERT, RUSTFS_TLS_KEY).build(),
fn load_from_source(source: &TlsSource) -> Result<Self, TlsRuntimeError> {
let base_dir = source.validate_directory()?;
let cert_key_pairs = load_all_certs_from_directory(
CertDirectoryLoadOptions::builder(base_dir, &source.layout.server_cert_filename, &source.layout.server_key_filename)
.build(),
)?;
if cert_key_pairs.is_empty() {
return Err(Error::other("No valid certificates found in directory"));
return Err(TlsRuntimeError::Material("No valid certificates found in directory".to_string()));
}
Self::from_cert_key_pairs(cert_key_pairs)
@@ -52,17 +54,23 @@ impl ResolverState {
fn from_cert_key_pairs(
cert_key_pairs: HashMap<String, (Vec<CertificateDer<'static>>, PrivateKeyDer<'static>)>,
) -> io::Result<Self> {
) -> Result<Self, TlsRuntimeError> {
let cert_count = cert_key_pairs.len();
let mut cert_resolver = ResolvesServerCertUsingSni::new();
let mut default_cert = None;
let mut entries = cert_key_pairs.into_iter().collect::<Vec<_>>();
entries.sort_by(|(left_domain, _), (right_domain, _)| left_domain.cmp(right_domain));
let fingerprint = fingerprint_tls_entries(&entries);
let material = ServerTlsMaterial::MultiCert {
cert_key_pairs: entries
.iter()
.map(|(domain, (certs, key))| (domain.clone(), (certs.clone(), key.clone_key())))
.collect(),
};
let fingerprint = server_material_fingerprint(&material);
for (domain, (certs, key)) in entries {
let signing_key = rustls::crypto::aws_lc_rs::sign::any_supported_type(&key)
.map_err(|e| Error::other(format!("unsupported private key type for {domain}: {e:?}")))?;
.map_err(|e| io::Error::other(format!("unsupported private key type for {domain}: {e:?}")))?;
let certified_key = CertifiedKey::new(certs, signing_key);
if domain.as_str() == "default" {
@@ -70,7 +78,7 @@ impl ResolverState {
} else {
cert_resolver
.add(&domain, certified_key)
.map_err(|e| Error::other(format!("failed to add certificate for {domain}: {e:?}")))?;
.map_err(|e| io::Error::other(format!("failed to add certificate for {domain}: {e:?}")))?;
}
}
@@ -83,39 +91,30 @@ impl ResolverState {
}
}
fn fingerprint_tls_entries(entries: &[(String, (Vec<CertificateDer<'static>>, PrivateKeyDer<'static>))]) -> u64 {
let mut hasher = DefaultHasher::new();
for (domain, (certs, key)) in entries {
hasher.write_usize(domain.len());
hasher.write(domain.as_bytes());
hasher.write_usize(certs.len());
for cert in certs {
hasher.write_usize(cert.as_ref().len());
hasher.write(cert.as_ref());
}
hasher.write_usize(key.secret_der().len());
hasher.write(key.secret_der());
}
hasher.finish()
}
#[derive(Debug)]
pub(crate) struct ReloadableCertResolver {
pub struct ReloadableServerCertResolver {
source: TlsSource,
current: RwLock<ResolverState>,
generation: AtomicU64,
}
impl ReloadableCertResolver {
pub(crate) fn load_from_directory(cert_dir: &str) -> io::Result<Arc<Self>> {
let state = ResolverState::load_from_directory(cert_dir)?;
impl ReloadableServerCertResolver {
pub fn load_from_source(source: TlsSource) -> Result<Arc<Self>, TlsRuntimeError> {
let state = ResolverState::load_from_source(&source)?;
record_tls_generation("server_resolver", 1);
Ok(Arc::new(Self {
source,
current: RwLock::new(state),
generation: AtomicU64::new(1),
}))
}
pub(crate) fn reload_from_directory(&self, cert_dir: &str) -> io::Result<Option<usize>> {
let new_state = ResolverState::load_from_directory(cert_dir)?;
pub fn load_from_directory(cert_dir: &str) -> Result<Arc<Self>, TlsRuntimeError> {
Self::load_from_source(TlsSource::from_directory(cert_dir))
}
pub fn reload(&self) -> Result<Option<usize>, TlsRuntimeError> {
let new_state = ResolverState::load_from_source(&self.source)?;
match self.current.write() {
Ok(mut guard) => {
@@ -124,6 +123,7 @@ impl ReloadableCertResolver {
}
let cert_count = new_state.cert_count;
*guard = new_state;
self.generation.fetch_add(1, Ordering::Relaxed);
Ok(Some(cert_count))
}
Err(poisoned) => {
@@ -133,14 +133,19 @@ impl ReloadableCertResolver {
}
let cert_count = new_state.cert_count;
*guard = new_state;
self.generation.fetch_add(1, Ordering::Relaxed);
Ok(Some(cert_count))
}
}
}
pub fn generation(&self) -> u64 {
self.generation.load(Ordering::Relaxed)
}
}
impl ResolvesServerCert for ReloadableCertResolver {
fn resolve(&self, client_hello: ClientHello) -> Option<Arc<CertifiedKey>> {
impl ResolvesServerCert for ReloadableServerCertResolver {
fn resolve(&self, client_hello: ClientHello<'_>) -> Option<Arc<CertifiedKey>> {
let guard = match self.current.read() {
Ok(guard) => guard,
Err(poisoned) => poisoned.into_inner(),
@@ -153,31 +158,26 @@ impl ResolvesServerCert for ReloadableCertResolver {
}
}
pub(crate) fn spawn_cert_reload_loop(
pub fn spawn_server_cert_reload_loop(
protocol: &'static str,
cert_dir: String,
resolver: Arc<ReloadableCertResolver>,
resolver: Arc<ReloadableServerCertResolver>,
options: TlsReloadOptions,
mut shutdown_rx: watch::Receiver<bool>,
) -> Option<JoinHandle<()>> {
let enabled = rustfs_utils::get_env_bool(ENV_TLS_RELOAD_ENABLE, DEFAULT_TLS_RELOAD_ENABLE);
if !enabled {
debug!(
protocol,
"TLS certificate hot reload is disabled (set {}=1 to enable)", ENV_TLS_RELOAD_ENABLE
);
if !options.enabled {
debug!(protocol, "TLS certificate hot reload is disabled");
return None;
}
let interval_secs = rustfs_utils::get_env_u64(ENV_TLS_RELOAD_INTERVAL, DEFAULT_TLS_RELOAD_INTERVAL).max(5);
info!(
protocol,
cert_dir = %cert_dir,
cert_dir = %resolver.source.base_dir.display(),
"TLS certificate hot reload enabled, checking every {}s",
interval_secs
options.interval.as_secs()
);
Some(tokio::spawn(async move {
let mut interval = tokio::time::interval(Duration::from_secs(interval_secs));
let mut interval = tokio::time::interval(options.interval);
interval.set_missed_tick_behavior(MissedTickBehavior::Delay);
interval.tick().await;
@@ -187,7 +187,7 @@ pub(crate) fn spawn_cert_reload_loop(
match changed {
Ok(()) => {
if *shutdown_rx.borrow() {
info!(protocol, cert_dir = %cert_dir, "TLS certificate hot reload task stopped");
info!(protocol, cert_dir = %resolver.source.base_dir.display(), "TLS certificate hot reload task stopped");
break;
}
continue;
@@ -195,7 +195,7 @@ pub(crate) fn spawn_cert_reload_loop(
Err(_) => {
info!(
protocol,
cert_dir = %cert_dir,
cert_dir = %resolver.source.base_dir.display(),
"TLS certificate hot reload task stopped because the shutdown channel closed"
);
break;
@@ -205,22 +205,29 @@ pub(crate) fn spawn_cert_reload_loop(
_ = interval.tick() => {}
}
match resolver.reload_from_directory(&cert_dir) {
match resolver.reload() {
Ok(Some(cert_count)) => {
record_tls_reload_result(protocol, "ok", None, Some(resolver.generation()));
info!(
protocol,
cert_dir = %cert_dir,
cert_dir = %resolver.source.base_dir.display(),
cert_count,
"TLS certificates reloaded successfully"
);
}
Ok(None) => {
debug!(protocol, cert_dir = %cert_dir, "TLS certificate material unchanged; skipping reload");
record_tls_reload_skipped(protocol, "unchanged");
debug!(
protocol,
cert_dir = %resolver.source.base_dir.display(),
"TLS certificate material unchanged; skipping reload"
);
}
Err(e) => {
record_tls_publication_fail(protocol);
warn!(
protocol,
cert_dir = %cert_dir,
cert_dir = %resolver.source.base_dir.display(),
"TLS certificate reload failed (will retry): {}",
e
);
@@ -238,10 +245,10 @@ mod tests {
use tempfile::TempDir;
fn cert_key_pair(san: &str) -> (Vec<CertificateDer<'static>>, PrivateKeyDer<'static>) {
let cert = generate_simple_self_signed(vec![san.to_string()]).unwrap();
let cert = generate_simple_self_signed(vec![san.to_string()]).expect("cert should generate");
(
vec![cert.cert.der().clone()],
PrivateKeyDer::try_from(cert.signing_key.serialize_der()).unwrap(),
PrivateKeyDer::try_from(cert.signing_key.serialize_der()).expect("key should convert"),
)
}
@@ -252,42 +259,44 @@ mod tests {
}
fn write_default_cert(dir: &std::path::Path, san: &str) {
let cert = generate_simple_self_signed(vec![san.to_string()]).unwrap();
fs::write(dir.join(RUSTFS_TLS_CERT), cert.cert.pem()).unwrap();
fs::write(dir.join(RUSTFS_TLS_KEY), cert.signing_key.serialize_pem()).unwrap();
let cert = generate_simple_self_signed(vec![san.to_string()]).expect("cert should generate");
fs::write(dir.join(rustfs_config::RUSTFS_TLS_CERT), cert.cert.pem()).expect("cert should write");
fs::write(dir.join(rustfs_config::RUSTFS_TLS_KEY), cert.signing_key.serialize_pem()).expect("key should write");
}
#[test]
fn reload_from_directory_replaces_default_certificate() {
let temp_dir = TempDir::new().unwrap();
fn reload_replaces_default_certificate() {
let temp_dir = TempDir::new().expect("tempdir should create");
write_default_cert(temp_dir.path(), "localhost");
let resolver = ReloadableCertResolver::load_from_directory(temp_dir.path().to_str().unwrap()).unwrap();
let resolver = ReloadableServerCertResolver::load_from_directory(temp_dir.path().to_str().expect("path should utf8"))
.expect("resolver should load");
let before = {
let guard = resolver.current.read().unwrap();
guard.default_cert.as_ref().unwrap().clone()
let guard = resolver.current.read().expect("lock should acquire");
guard.default_cert.as_ref().expect("default cert should exist").clone()
};
write_default_cert(temp_dir.path(), "rotated.local");
let cert_count = resolver.reload_from_directory(temp_dir.path().to_str().unwrap()).unwrap();
let cert_count = resolver.reload().expect("reload should succeed");
assert_eq!(cert_count, Some(1));
let after = {
let guard = resolver.current.read().unwrap();
guard.default_cert.as_ref().unwrap().clone()
let guard = resolver.current.read().expect("lock should acquire");
guard.default_cert.as_ref().expect("default cert should exist").clone()
};
assert_ne!(before.cert[0].as_ref(), after.cert[0].as_ref());
}
#[test]
fn reload_from_directory_skips_when_material_is_unchanged() {
let temp_dir = TempDir::new().unwrap();
fn reload_skips_when_material_is_unchanged() {
let temp_dir = TempDir::new().expect("tempdir should create");
write_default_cert(temp_dir.path(), "localhost");
let resolver = ReloadableCertResolver::load_from_directory(temp_dir.path().to_str().unwrap()).unwrap();
let outcome = resolver.reload_from_directory(temp_dir.path().to_str().unwrap()).unwrap();
let resolver = ReloadableServerCertResolver::load_from_directory(temp_dir.path().to_str().expect("path should utf8"))
.expect("resolver should load");
let outcome = resolver.reload().expect("reload should succeed");
assert_eq!(outcome, None);
}
@@ -307,8 +316,8 @@ mod tests {
second.insert("default".to_string(), clone_cert_key_pair(&default_cert));
second.insert("api.example.com".to_string(), clone_cert_key_pair(&api_cert));
let first_state = ResolverState::from_cert_key_pairs(first).unwrap();
let second_state = ResolverState::from_cert_key_pairs(second).unwrap();
let first_state = ResolverState::from_cert_key_pairs(first).expect("first state should build");
let second_state = ResolverState::from_cert_key_pairs(second).expect("second state should build");
assert_eq!(first_state.cert_count, 3);
assert_eq!(second_state.cert_count, 3);
crates/tls-runtime/src/source.rs
+94
View File
@@ -0,0 +1,94 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::error::TlsRuntimeError;
use rustfs_config::{
RUSTFS_CA_CERT, RUSTFS_CLIENT_CA_CERT_FILENAME, RUSTFS_CLIENT_CERT_FILENAME, RUSTFS_CLIENT_KEY_FILENAME, RUSTFS_PUBLIC_CERT,
RUSTFS_TLS_CERT, RUSTFS_TLS_KEY,
};
use std::path::{Path, PathBuf};
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum TlsSourceKind {
Directory,
ExplicitFiles,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct TlsFileLayout {
pub server_cert_filename: String,
pub server_key_filename: String,
pub public_ca_filename: String,
pub client_ca_filename: String,
pub client_cert_filename: String,
pub client_key_filename: String,
}
impl Default for TlsFileLayout {
fn default() -> Self {
Self {
server_cert_filename: RUSTFS_TLS_CERT.to_string(),
server_key_filename: RUSTFS_TLS_KEY.to_string(),
public_ca_filename: RUSTFS_PUBLIC_CERT.to_string(),
client_ca_filename: RUSTFS_CLIENT_CA_CERT_FILENAME.to_string(),
client_cert_filename: RUSTFS_CLIENT_CERT_FILENAME.to_string(),
client_key_filename: RUSTFS_CLIENT_KEY_FILENAME.to_string(),
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct TlsSource {
pub kind: TlsSourceKind,
pub base_dir: PathBuf,
pub layout: TlsFileLayout,
pub fallback_ca_filename: String,
pub trust_system_ca: bool,
pub trust_leaf_as_ca: bool,
pub server_mtls_enabled: bool,
}
impl TlsSource {
pub fn from_directory(base_dir: impl Into<PathBuf>) -> Self {
Self {
kind: TlsSourceKind::Directory,
base_dir: base_dir.into(),
layout: TlsFileLayout::default(),
fallback_ca_filename: RUSTFS_CA_CERT.to_string(),
trust_system_ca: false,
trust_leaf_as_ca: false,
server_mtls_enabled: false,
}
}
pub fn validate_directory(&self) -> Result<&Path, TlsRuntimeError> {
if self.base_dir.as_os_str().is_empty() {
return Err(TlsRuntimeError::EmptySourcePath);
}
if !self.base_dir.exists() {
return Err(TlsRuntimeError::DirectoryNotFound {
path: self.base_dir.clone(),
});
}
if !self.base_dir.is_dir() {
return Err(TlsRuntimeError::NotADirectory {
path: self.base_dir.clone(),
});
}
Ok(self.base_dir.as_path())
}
}
crates/tls-runtime/src/state.rs
+234
View File
@@ -0,0 +1,234 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::fingerprint::TlsFingerprint;
use arc_swap::ArcSwap;
use serde::Serialize;
use std::sync::Arc;
use std::sync::atomic::{AtomicU64, Ordering};
use tokio::sync::RwLock;
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct TlsGeneration(pub u64);
#[derive(Debug)]
pub struct TlsPublishedState<M> {
pub generation: TlsGeneration,
pub material: Arc<M>,
pub fingerprint: TlsFingerprint,
pub loaded_at_unix_ms: u64,
}
#[derive(Debug)]
pub struct TlsReloadRuntimeState<M> {
pub current: ArcSwap<TlsPublishedState<M>>,
pub last_good: ArcSwap<TlsPublishedState<M>>,
pub last_attempt_unix_ms: AtomicU64,
pub last_success_unix_ms: AtomicU64,
pub last_error: RwLock<Option<String>>,
}
impl<M> TlsReloadRuntimeState<M> {
pub fn new(initial: Arc<TlsPublishedState<M>>) -> Self {
Self {
current: ArcSwap::from(initial.clone()),
last_good: ArcSwap::from(initial),
last_attempt_unix_ms: AtomicU64::new(0),
last_success_unix_ms: AtomicU64::new(0),
last_error: RwLock::new(None),
}
}
pub fn current_generation(&self) -> TlsGeneration {
self.current.load().generation
}
pub fn bump_generation(&self) -> TlsGeneration {
TlsGeneration(self.current_generation().0.saturating_add(1))
}
pub fn mark_attempt(&self, unix_ms: u64) {
self.last_attempt_unix_ms.store(unix_ms, Ordering::Relaxed);
}
pub fn mark_success(&self, unix_ms: u64) {
self.last_success_unix_ms.store(unix_ms, Ordering::Relaxed);
}
pub fn last_attempt_unix_ms(&self) -> u64 {
self.last_attempt_unix_ms.load(Ordering::Relaxed)
}
pub fn last_success_unix_ms(&self) -> u64 {
self.last_success_unix_ms.load(Ordering::Relaxed)
}
}
#[derive(Debug, Clone, Serialize, PartialEq, Eq)]
pub struct TlsRuntimeStatusSnapshot {
pub runtime: TlsRuntimeRuntimeSection,
pub outbound: TlsRuntimeOutboundSection,
pub server: TlsRuntimeServerSection,
pub consumer: TlsRuntimeConsumerSection,
}
impl TlsRuntimeStatusSnapshot {
pub fn is_complete(&self) -> bool {
self.server.has_material || self.outbound.has_roots || self.outbound.has_mtls_identity
}
pub fn from_outbound_only(args: OutboundOnlySnapshotArgs) -> Self {
Self {
runtime: TlsRuntimeRuntimeSection {
generation: args.generation,
reload_enabled: args.reload_enabled,
detect_mode: args.detect_mode,
last_attempt_time: args.last_attempt_time,
last_success_time: args.last_success_time,
last_error: args.last_error,
source_path: args.source_path,
},
outbound: TlsRuntimeOutboundSection {
has_roots: args.has_roots,
has_mtls_identity: args.has_mtls_identity,
},
server: TlsRuntimeServerSection { has_material: false },
consumer: TlsRuntimeConsumerSection { stale_generation: false },
}
}
}
#[derive(Debug, Clone)]
pub struct OutboundOnlySnapshotArgs {
pub source_path: String,
pub generation: u64,
pub reload_enabled: bool,
pub detect_mode: &'static str,
pub last_attempt_time: Option<u64>,
pub last_success_time: Option<u64>,
pub last_error: Option<String>,
pub has_roots: bool,
pub has_mtls_identity: bool,
}
pub fn detect_mode_label(mode: crate::config::ReloadDetectMode) -> &'static str {
match mode {
crate::config::ReloadDetectMode::Poll => "poll",
crate::config::ReloadDetectMode::Watch => "watch",
crate::config::ReloadDetectMode::Hybrid => "hybrid",
}
}
#[derive(Debug, Clone, Serialize, PartialEq, Eq)]
pub struct TlsRuntimeRuntimeSection {
pub generation: u64,
pub reload_enabled: bool,
pub detect_mode: &'static str,
pub last_attempt_time: Option<u64>,
pub last_success_time: Option<u64>,
pub last_error: Option<String>,
pub source_path: String,
}
#[derive(Debug, Clone, Serialize, PartialEq, Eq)]
pub struct TlsRuntimeOutboundSection {
pub has_roots: bool,
pub has_mtls_identity: bool,
}
#[derive(Debug, Clone, Serialize, PartialEq, Eq)]
pub struct TlsRuntimeServerSection {
pub has_material: bool,
}
#[derive(Debug, Clone, Serialize, PartialEq, Eq)]
pub struct TlsRuntimeConsumerSection {
pub stale_generation: bool,
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::Arc;
fn make_published(generation: u64, fingerprint_bytes: &[u8]) -> Arc<TlsPublishedState<String>> {
Arc::new(TlsPublishedState {
generation: TlsGeneration(generation),
material: Arc::new("test".to_string()),
fingerprint: TlsFingerprint::from_optional_bytes(Some(fingerprint_bytes), None, None, None, None),
loaded_at_unix_ms: 0,
})
}
#[test]
fn runtime_state_tracks_generation_and_timestamps() {
let initial = make_published(1, b"aaa");
let state = TlsReloadRuntimeState::new(initial);
assert_eq!(state.current_generation(), TlsGeneration(1));
assert_eq!(state.bump_generation(), TlsGeneration(2));
assert_eq!(state.last_attempt_unix_ms(), 0);
assert_eq!(state.last_success_unix_ms(), 0);
state.mark_attempt(100);
assert_eq!(state.last_attempt_unix_ms(), 100);
state.mark_success(200);
assert_eq!(state.last_success_unix_ms(), 200);
}
#[test]
fn bump_generation_saturates_at_max() {
let initial = Arc::new(TlsPublishedState {
generation: TlsGeneration(u64::MAX),
material: Arc::new("max".to_string()),
fingerprint: TlsFingerprint::default(),
loaded_at_unix_ms: 0,
});
let state = TlsReloadRuntimeState::new(initial);
assert_eq!(state.bump_generation(), TlsGeneration(u64::MAX));
}
#[test]
fn status_snapshot_is_complete_with_outbound_roots() {
let snap = TlsRuntimeStatusSnapshot::from_outbound_only(OutboundOnlySnapshotArgs {
source_path: "/tmp".to_string(),
generation: 1,
reload_enabled: true,
detect_mode: "poll",
last_attempt_time: None,
last_success_time: None,
last_error: None,
has_roots: true,
has_mtls_identity: false,
});
assert!(snap.is_complete());
}
#[test]
fn status_snapshot_is_not_complete_when_empty() {
let snap = TlsRuntimeStatusSnapshot::from_outbound_only(OutboundOnlySnapshotArgs {
source_path: "/tmp".to_string(),
generation: 1,
reload_enabled: true,
detect_mode: "poll",
last_attempt_time: None,
last_success_time: None,
last_error: None,
has_roots: false,
has_mtls_identity: false,
});
assert!(!snap.is_complete());
}
}
crates/utils/Cargo.toml
+2 -11
View File
@@ -32,22 +32,17 @@ bytes = { workspace = true, optional = true }
crc-fast = { workspace = true, optional = true }
flate2 = { workspace = true, optional = true }
futures = { workspace = true, optional = true }
hashbrown = { workspace = true, optional = true }
hex-simd = { workspace = true, optional = true }
highway = { workspace = true, optional = true }
hmac = { workspace = true, optional = true }
http = { workspace = true, optional = true }
hyper = { workspace = true, optional = true }
libc = { workspace = true, optional = true }
local-ip-address = { workspace = true, optional = true }
lz4 = { workspace = true, optional = true }
md-5 = { workspace = true, optional = true }
netif = { workspace = true, optional = true }
regex = { workspace = true, optional = true }
rustix = { workspace = true, optional = true }
rustls = { workspace = true, optional = true }
rustls-pki-types = { workspace = true, optional = true }
s3s = { workspace = true, optional = true }
serde = { workspace = true, optional = true }
sha1 = { workspace = true, optional = true }
sha2 = { workspace = true, optional = true }
@@ -55,7 +50,6 @@ convert_case = { workspace = true, optional = true }
siphasher = { workspace = true, optional = true }
snap = { workspace = true, optional = true }
tempfile = { workspace = true, optional = true }
thiserror = { workspace = true, optional = true }
tokio = { workspace = true, optional = true, features = ["io-util", "macros"] }
tracing = { workspace = true }
transform-stream = { workspace = true, optional = true }
@@ -63,7 +57,6 @@ url = { workspace = true, optional = true }
zstd = { workspace = true, optional = true }
[dev-dependencies]
rcgen = { workspace = true }
tempfile = { workspace = true }
tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }
temp-env = { workspace = true }
@@ -77,11 +70,9 @@ workspace = true
[features]
default = ["ip"] # features that are enabled by default
ip = ["dep:local-ip-address"] # ip characteristics and their dependencies
tls = ["dep:rustls", "dep:rustls-pki-types"] # tls characteristics and their dependencies
net = ["ip", "dep:url", "dep:netif", "dep:futures", "dep:transform-stream", "dep:bytes", "dep:s3s", "dep:hyper", "dep:thiserror", "dep:tokio"] # network features with DNS resolver
net = ["ip", "dep:url", "dep:netif", "dep:futures", "dep:transform-stream", "dep:bytes", "dep:hyper", "dep:tokio"] # network features with DNS resolver
io = ["dep:tokio"]
path = [] # path manipulation features
notify = ["dep:hyper", "dep:s3s", "dep:hashbrown", "dep:thiserror", "dep:serde", "dep:libc", "dep:url", "dep:regex"] # file system notification features
compress = ["dep:flate2", "dep:brotli", "dep:snap", "dep:lz4", "dep:zstd"]
string = ["dep:regex"]
crypto = ["dep:base64-simd", "dep:hex-simd", "dep:hmac", "dep:hyper", "dep:sha1", "dep:sha2"]
@@ -90,4 +81,4 @@ os = ["dep:rustix", "dep:tempfile", "dep:windows"] # operating system utilities
integration = [] # integration test features
http = ["dep:convert_case", "dep:http", "dep:regex"]
obj = ["http"] # object storage features
full = ["ip", "tls", "net", "io", "hash", "os", "integration", "path", "crypto", "string", "compress", "notify", "http", "obj"] # all features
full = ["ip", "net", "io", "hash", "os", "integration", "path", "crypto", "string", "compress", "http", "obj"] # all features
crates/utils/src/lib.rs
-11
View File
@@ -12,8 +12,6 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#[cfg(feature = "tls")]
pub mod certs;
#[cfg(feature = "ip")]
pub mod ip;
#[cfg(feature = "net")]
@@ -52,9 +50,6 @@ pub mod compress;
#[cfg(feature = "path")]
pub mod dirs;
#[cfg(feature = "tls")]
pub use certs::*;
#[cfg(feature = "hash")]
pub use hash::*;
@@ -70,12 +65,6 @@ pub use crypto::*;
#[cfg(feature = "compress")]
pub use compress::*;
#[cfg(feature = "notify")]
mod notify;
#[cfg(feature = "notify")]
pub use notify::*;
#[cfg(feature = "obj")]
pub mod obj;
crates/utils/src/notify/mod.rs
-205
View File
@@ -1,205 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
mod net;
use hashbrown::HashMap;
use hyper::HeaderMap;
use s3s::{S3Request, S3Response};
pub use net::*;
/// Extract request parameters from S3Request, mainly header information.
pub fn extract_req_params<T>(req: &S3Request<T>) -> HashMap<String, String> {
extract_params_header(&req.headers)
}
/// Extract request parameters from hyper::HeaderMap, mainly header information.
/// This function is useful when you have a raw HTTP request and need to extract parameters.
#[deprecated(since = "0.1.0", note = "Use extract_params_header instead")]
pub fn extract_req_params_header(head: &HeaderMap) -> HashMap<String, String> {
extract_params_header(head)
}
/// Extract parameters from hyper::HeaderMap, mainly header information.
/// This function is useful when you have a raw HTTP request and need to extract parameters.
pub fn extract_params_header(head: &HeaderMap) -> HashMap<String, String> {
let mut params = HashMap::new();
for (key, value) in head.iter() {
if let Ok(val_str) = value.to_str() {
params.insert(key.as_str().to_string(), val_str.to_string());
}
}
params
}
/// Extract response elements from S3Response, mainly header information.
pub fn extract_resp_elements<T>(resp: &S3Response<T>) -> HashMap<String, String> {
extract_params_header(&resp.headers)
}
/// Get host from header information.
pub fn get_request_host(headers: &HeaderMap) -> String {
headers
.get("host")
.and_then(|v| v.to_str().ok())
.unwrap_or_default()
.to_string()
}
/// Get Port from header information.
/// Priority:
/// 1. x-forwarded-port
/// 2. host header (parse port)
/// If host has no port, try to deduce from x-forwarded-proto (http->80, https->443)
/// 3. port header
///
/// If the port cannot be determined, returns 0.
pub fn get_request_port(headers: &HeaderMap) -> u16 {
// 1. Try x-forwarded-port
if let Some(port) = headers
.get("x-forwarded-port")
.and_then(|v| v.to_str().ok())
.and_then(|s| s.parse::<u16>().ok())
{
return port;
}
// 2. Try host header
if let Some(host) = headers.get("host").and_then(|v| v.to_str().ok()) {
if let Some(idx) = host.rfind(':') {
// Check if it's an IPv6 address with port, e.g., [::1]:8080
// If ']' is present, the colon must be after it.
let valid_colon = match host.rfind(']') {
Some(close_bracket_idx) => idx > close_bracket_idx,
None => true,
};
if valid_colon
&& let Ok(port) = host[idx + 1..].parse::<u16>()
&& port > 0
{
return port;
}
}
// If host is present but no port found (or parsing failed, or port is 0),
// try to deduce from x-forwarded-proto
if let Some(proto) = headers.get("x-forwarded-proto").and_then(|v| v.to_str().ok()) {
match proto {
"http" => return 80,
"https" => return 443,
_ => {}
}
}
}
// 3. Fallback to "port" header
headers
.get("port")
.and_then(|v| v.to_str().ok())
.and_then(|s| s.parse::<u16>().ok())
.unwrap_or(0)
}
/// Get content-length from header information.
pub fn get_request_content_length(headers: &HeaderMap) -> u64 {
headers
.get("content-length")
.and_then(|v| v.to_str().ok())
.and_then(|s| s.parse::<u64>().ok())
.unwrap_or(0)
}
/// Get referer from header information.
/// If the referer header is not present, returns an empty string.
pub fn get_request_referer(headers: &HeaderMap) -> String {
headers
.get("referer")
.and_then(|v| v.to_str().ok())
.unwrap_or_default()
.to_string()
}
/// Get user-agent from header information.
pub fn get_request_user_agent(headers: &HeaderMap) -> String {
headers
.get("user-agent")
.and_then(|v| v.to_str().ok())
.unwrap_or_default()
.to_string()
}
#[cfg(test)]
mod tests {
use super::*;
use hyper::header::HeaderValue;
#[test]
fn test_get_request_port() {
let mut headers = HeaderMap::new();
// Case 1: No port info
assert_eq!(get_request_port(&headers), 0);
// Case 2: port header
headers.insert("port", HeaderValue::from_static("8080"));
assert_eq!(get_request_port(&headers), 8080);
// Case 3: host header with port
headers.remove("port");
headers.insert("host", HeaderValue::from_static("example.com:9000"));
assert_eq!(get_request_port(&headers), 9000);
// Case 4: host header without port, no proto
headers.insert("host", HeaderValue::from_static("example.com"));
assert_eq!(get_request_port(&headers), 0);
// Case 5: IPv6 host with port
headers.insert("host", HeaderValue::from_static("[::1]:9001"));
assert_eq!(get_request_port(&headers), 9001);
// Case 6: IPv6 host without port
headers.insert("host", HeaderValue::from_static("[::1]"));
assert_eq!(get_request_port(&headers), 0);
// Case 7: x-forwarded-port
headers.insert("x-forwarded-port", HeaderValue::from_static("7000"));
// Even if host is present, x-forwarded-port takes precedence
assert_eq!(get_request_port(&headers), 7000);
// Case 8: host without port, but x-forwarded-proto is http
headers.remove("x-forwarded-port");
headers.insert("host", HeaderValue::from_static("example.com"));
headers.insert("x-forwarded-proto", HeaderValue::from_static("http"));
assert_eq!(get_request_port(&headers), 80);
// Case 9: host without port, but x-forwarded-proto is https
headers.insert("x-forwarded-proto", HeaderValue::from_static("https"));
assert_eq!(get_request_port(&headers), 443);
// Case 10: host without port, unknown proto
headers.insert("x-forwarded-proto", HeaderValue::from_static("ftp"));
assert_eq!(get_request_port(&headers), 0);
// Case 11: host with port 0, should fallback to proto
headers.insert("host", HeaderValue::from_static("example.com:0"));
headers.insert("x-forwarded-proto", HeaderValue::from_static("https"));
assert_eq!(get_request_port(&headers), 443);
// Case 12: host with port 0, no proto
headers.remove("x-forwarded-proto");
assert_eq!(get_request_port(&headers), 0);
}
}
rustfs/Cargo.toml
+3 -3
View File
@@ -85,6 +85,7 @@ rustfs-data-usage = { workspace = true }
rustfs-s3select-api = { workspace = true }
rustfs-s3select-query = { workspace = true }
rustfs-targets = { workspace = true }
rustfs-tls-runtime = { workspace = true }
rustfs-trusted-proxies = { workspace = true }
rustfs-utils = { workspace = true, features = ["full"] }
rustfs-zip = { workspace = true }
@@ -127,6 +128,7 @@ serde_urlencoded = { workspace = true }
# Cryptography and Security
rustls = { workspace = true }
rustls-pki-types = { workspace = true }
subtle = { workspace = true }
jiff = { workspace = true }
time = { workspace = true, features = ["parsing", "formatting", "serde"] }
@@ -173,9 +175,7 @@ libsystemd.workspace = true
[target.'cfg(not(all(target_os = "linux", target_env = "gnu", target_arch = "x86_64")))'.dependencies]
mimalloc = { workspace = true }
libmimalloc-sys = { version = "0.1.48", features = ["extended"] }
libmimalloc-sys = { version = "0.1.49", features = ["extended"] }
# Only enable pprof-based profiling on linux + gnu + x86_64.
[target.'cfg(all(target_os = "linux", target_env = "gnu", target_arch = "x86_64"))'.dependencies]
rustfs/src/admin/handlers/mod.rs
+2
View File
@@ -45,6 +45,7 @@ pub mod sts;
pub mod system;
mod target_descriptor;
pub mod tier;
pub mod tls_debug;
pub mod trace;
pub mod user;
pub mod user_iam;
@@ -75,6 +76,7 @@ mod tests {
let _metrics_handler = metrics::MetricsHandler {};
let _profile_handler = profile_admin::ProfileHandler {};
let _profile_status_handler = profile_admin::ProfileStatusHandler {};
let _tls_status_handler = tls_debug::TlsStatusHandler {};
let _heal_handler = heal::HealHandler {};
let _bg_heal_handler = heal::BackgroundHealStatusHandler {};
let _replication_metrics_handler = replication::GetReplicationMetricsHandler {};
rustfs/src/admin/handlers/site_replication.rs
+30 -53
View File
@@ -29,10 +29,7 @@ use http::header::{CONTENT_TYPE, HOST};
use http::{HeaderMap, HeaderValue, Uri};
use hyper::{Method, StatusCode};
use matchit::Params;
use rustfs_config::{
DEFAULT_DELIMITER, DEFAULT_RUSTFS_TLS_PATH, DEFAULT_TRUST_LEAF_CERT_AS_CA, ENV_RUSTFS_TLS_PATH, ENV_TRUST_LEAF_CERT_AS_CA,
MAX_ADMIN_REQUEST_BODY_SIZE, RUSTFS_CA_CERT, RUSTFS_TLS_CERT,
};
use rustfs_config::{DEFAULT_DELIMITER, DEFAULT_RUSTFS_TLS_PATH, ENV_RUSTFS_TLS_PATH, MAX_ADMIN_REQUEST_BODY_SIZE};
use rustfs_ecstore::bucket::bucket_target_sys::BucketTargetSys;
use rustfs_ecstore::bucket::metadata::{
BUCKET_CORS_CONFIG, BUCKET_LIFECYCLE_CONFIG, BUCKET_POLICY_CONFIG, BUCKET_QUOTA_CONFIG_FILE, BUCKET_REPLICATION_CONFIG,
@@ -67,7 +64,9 @@ use rustfs_policy::policy::{
};
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
use rustfs_signer::sign_v4;
use rustfs_tls_runtime::load_global_outbound_tls_state;
use rustfs_utils::http::get_source_scheme;
use rustls_pki_types::pem::PemObject;
use s3s::dto::{
BucketVersioningStatus, DeleteMarkerReplication, DeleteMarkerReplicationStatus, DeleteReplication, DeleteReplicationStatus,
Destination, ExistingObjectReplication, ExistingObjectReplicationStatus, ReplicationConfiguration, ReplicationRule,
@@ -80,9 +79,9 @@ use serde::de::DeserializeOwned;
use serde_json::Value;
use sha2::{Digest, Sha256};
use std::collections::{BTreeMap, HashMap, HashSet};
use std::sync::OnceLock;
use std::time::{Duration, Instant};
use time::OffsetDateTime;
use tokio::sync::OnceCell;
use tracing::warn;
use url::{Url, form_urlencoded};
use uuid::Uuid;
@@ -103,7 +102,7 @@ const SITE_REPLICATOR_SERVICE_ACCOUNT: &str = "site-replicator-0";
const SITE_REPLICATION_PEER_JOIN_PATH: &str = "/rustfs/admin/v3/site-replication/peer/join";
const SITE_REPLICATION_PEER_EDIT_PATH: &str = "/rustfs/admin/v3/site-replication/peer/edit";
const SITE_REPLICATION_PEER_REMOVE_PATH: &str = "/rustfs/admin/v3/site-replication/peer/remove";
static SITE_REPLICATION_PEER_CLIENT: OnceLock<Result<reqwest::Client, String>> = OnceLock::new();
static SITE_REPLICATION_PEER_CLIENT: OnceCell<Result<reqwest::Client, String>> = OnceCell::const_new();
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
struct SiteReplicationState {
@@ -448,56 +447,32 @@ async fn persist_site_replication_state(state: &SiteReplicationState) -> S3Resul
}
}
fn add_root_certificates_from_file(
mut builder: reqwest::ClientBuilder,
cert_path: &std::path::Path,
description: &str,
) -> S3Result<reqwest::ClientBuilder> {
if !cert_path.exists() {
return Ok(builder);
}
std::fs::read(cert_path).map_err(|e| {
S3Error::with_message(
S3ErrorCode::InternalError,
format!("failed to read {description} {}: {e}", cert_path.display()),
)
})?;
let certs_der = rustfs_utils::load_cert_bundle_der_bytes(cert_path.to_string_lossy().as_ref()).map_err(|e| {
S3Error::with_message(
S3ErrorCode::InternalError,
format!("failed to parse {description} {}: {e}", cert_path.display()),
)
})?;
for cert_der in certs_der {
let cert = reqwest::Certificate::from_der(&cert_der).map_err(|e| {
S3Error::with_message(
S3ErrorCode::InternalError,
format!("failed to load {description} {}: {e}", cert_path.display()),
)
})?;
builder = builder.add_root_certificate(cert);
}
Ok(builder)
}
fn build_site_replication_peer_client() -> S3Result<reqwest::Client> {
async fn build_site_replication_peer_client() -> S3Result<reqwest::Client> {
let mut builder = reqwest::Client::builder()
.timeout(SITE_REPLICATION_PEER_REQUEST_TIMEOUT)
.connect_timeout(SITE_REPLICATION_PEER_CONNECT_TIMEOUT)
.pool_idle_timeout(Some(Duration::from_secs(60)));
let tls_path = rustfs_utils::get_env_str(ENV_RUSTFS_TLS_PATH, DEFAULT_RUSTFS_TLS_PATH);
if !tls_path.is_empty() {
let tls_dir = std::path::Path::new(&tls_path);
builder = add_root_certificates_from_file(builder, &tls_dir.join(RUSTFS_CA_CERT), "site-replication CA cert")?;
let outbound_tls = load_global_outbound_tls_state().await;
if let Some(root_ca_pem) = outbound_tls.root_ca_pem.as_ref() {
let mut reader = std::io::BufReader::new(root_ca_pem.as_slice());
let certs_der = rustls_pki_types::CertificateDer::pem_reader_iter(&mut reader)
.collect::<Result<Vec<_>, _>>()
.map_err(|e| {
S3Error::with_message(
S3ErrorCode::InternalError,
format!("failed to parse published site-replication CA certs: {e}"),
)
})?;
if rustfs_utils::get_env_bool(ENV_TRUST_LEAF_CERT_AS_CA, DEFAULT_TRUST_LEAF_CERT_AS_CA) {
builder =
add_root_certificates_from_file(builder, &tls_dir.join(RUSTFS_TLS_CERT), "site-replication leaf cert as CA")?;
for cert_der in certs_der {
let cert = reqwest::Certificate::from_der(cert_der.as_ref()).map_err(|e| {
S3Error::with_message(
S3ErrorCode::InternalError,
format!("failed to load published site-replication CA cert: {e}"),
)
})?;
builder = builder.add_root_certificate(cert);
}
}
@@ -506,8 +481,10 @@ fn build_site_replication_peer_client() -> S3Result<reqwest::Client> {
.map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("build site replication peer client failed: {e}")))
}
fn site_replication_peer_client() -> S3Result<&'static reqwest::Client> {
let result = SITE_REPLICATION_PEER_CLIENT.get_or_init(|| build_site_replication_peer_client().map_err(|e| e.to_string()));
async fn site_replication_peer_client() -> S3Result<&'static reqwest::Client> {
let result = SITE_REPLICATION_PEER_CLIENT
.get_or_init(|| async { build_site_replication_peer_client().await.map_err(|e| e.to_string()) })
.await;
result.as_ref().map_err(|err| {
S3Error::with_message(
S3ErrorCode::InternalError,
@@ -969,7 +946,7 @@ async fn send_peer_admin_request<T: Serialize>(
.unwrap_or("us-east-1"),
);
let mut req = site_replication_peer_client()?.request(reqwest::Method::PUT, &url);
let mut req = site_replication_peer_client().await?.request(reqwest::Method::PUT, &url);
for (name, value) in signed.headers() {
req = req.header(name, value);
}
rustfs/src/admin/handlers/tls_debug.rs
+154
View File
@@ -0,0 +1,154 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use super::profile::authorize_profile_request;
use crate::admin::router::{AdminOperation, Operation, S3Router};
use crate::server::ADMIN_PREFIX;
use http::StatusCode;
use http::{HeaderMap, HeaderValue};
use hyper::Method;
use matchit::Params;
use rustfs_tls_runtime::{
OutboundOnlySnapshotArgs, TlsConsumerStatusItem, TlsDebugStatusResponse, TlsRuntimeStatusSnapshot,
summarize_global_outbound_tls_state,
};
use s3s::header::CONTENT_TYPE;
use s3s::{Body, S3Error, S3ErrorCode, S3Request, S3Response, S3Result};
pub fn register_tls_debug_route(r: &mut S3Router<AdminOperation>) -> std::io::Result<()> {
r.insert(
Method::GET,
format!("{}{}", ADMIN_PREFIX, "/debug/tls/status").as_str(),
AdminOperation(&TlsStatusHandler {}),
)?;
Ok(())
}
pub struct TlsStatusHandler {}
const PROTOS_GRPC_CHANNEL_CONSUMER: &str = "protos_grpc_channel";
const RIO_HTTP_READER_CONSUMER: &str = "rio_http_reader";
const ECSTORE_TRANSITION_CLIENT_CONSUMER: &str = "ecstore_transition_client";
#[async_trait::async_trait]
impl Operation for TlsStatusHandler {
async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
authorize_profile_request(&req).await?;
let tls_path = rustfs_utils::get_env_opt_str(rustfs_config::ENV_RUSTFS_TLS_PATH).unwrap_or_default();
let outbound = summarize_global_outbound_tls_state().await;
let status = TlsRuntimeStatusSnapshot::from_outbound_only(OutboundOnlySnapshotArgs {
source_path: tls_path,
generation: outbound.generation.0,
reload_enabled: rustfs_utils::get_env_bool(
rustfs_config::ENV_TLS_RELOAD_ENABLE,
rustfs_config::DEFAULT_TLS_RELOAD_ENABLE,
),
detect_mode: "poll",
last_attempt_time: None,
last_success_time: None,
last_error: None,
has_roots: outbound.has_root_ca,
has_mtls_identity: outbound.has_mtls_identity,
});
let payload = TlsDebugStatusResponse::builder(status)
.push_consumers([
TlsConsumerStatusItem {
consumer: PROTOS_GRPC_CHANNEL_CONSUMER,
generation: outbound.generation.0,
has_root_ca: outbound.has_root_ca,
has_mtls_identity: outbound.has_mtls_identity,
},
TlsConsumerStatusItem {
consumer: RIO_HTTP_READER_CONSUMER,
generation: outbound.generation.0,
has_root_ca: outbound.has_root_ca,
has_mtls_identity: outbound.has_mtls_identity,
},
TlsConsumerStatusItem {
consumer: ECSTORE_TRANSITION_CLIENT_CONSUMER,
generation: outbound.generation.0,
has_root_ca: outbound.has_root_ca,
has_mtls_identity: outbound.has_mtls_identity,
},
])
.build();
let body = serde_json::to_vec(&payload)
.map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("serialize tls status failed: {e}")))?;
let mut headers = HeaderMap::new();
headers.insert(CONTENT_TYPE, HeaderValue::from_static("application/json"));
Ok(S3Response::with_headers((StatusCode::OK, Body::from(body)), headers))
}
}
#[cfg(test)]
mod tests {
use super::TlsStatusHandler;
use crate::admin::router::Operation;
use http::{Extensions, HeaderMap, Uri};
use hyper::Method;
use matchit::Params;
use rustfs_tls_runtime::{OutboundOnlySnapshotArgs, TlsConsumerStatusItem, TlsRuntimeStatusSnapshot};
use s3s::{Body, S3ErrorCode, S3Request};
fn build_tls_status_request() -> S3Request<Body> {
S3Request {
input: Body::empty(),
method: Method::GET,
uri: Uri::from_static("/rustfs/admin/debug/tls/status"),
headers: HeaderMap::new(),
extensions: Extensions::new(),
credentials: None,
region: None,
service: None,
trailing_headers: None,
}
}
#[tokio::test]
async fn tls_status_handler_rejects_missing_credentials() {
let result = TlsStatusHandler {}.call(build_tls_status_request(), Params::new()).await;
let err = result.expect_err("tls status handler must reject unauthenticated requests");
assert_eq!(err.code(), &S3ErrorCode::AccessDenied);
}
#[test]
fn tls_debug_response_schema_is_structured() {
let status = TlsRuntimeStatusSnapshot::from_outbound_only(OutboundOnlySnapshotArgs {
source_path: "/tmp/tls".to_string(),
generation: 7,
reload_enabled: true,
detect_mode: "poll",
last_attempt_time: Some(1),
last_success_time: Some(2),
last_error: Some("x".to_string()),
has_roots: true,
has_mtls_identity: false,
});
let payload = rustfs_tls_runtime::TlsDebugStatusResponse::builder(status)
.push_consumers([TlsConsumerStatusItem {
consumer: "protos_grpc_channel",
generation: 7,
has_root_ca: true,
has_mtls_identity: false,
}])
.build();
let json = serde_json::to_value(payload).expect("json should serialize");
assert!(json.get("foundation").is_some());
assert!(json.get("consumers").is_some());
assert!(json["consumers"].is_array());
}
}
rustfs/src/admin/mod.rs
+2 -1
View File
@@ -28,7 +28,7 @@ mod route_registration_test;
use handlers::{
audit, bucket_meta, heal, health, kms, module_switch, oidc, plugins_catalog, plugins_instances, pools, profile_admin, quota,
rebalance, replication, site_replication, sts, system, tier, user,
rebalance, replication, site_replication, sts, system, tier, tls_debug, user,
};
use router::{AdminOperation, S3Router};
use s3s::route::S3Route;
@@ -65,6 +65,7 @@ pub fn make_admin_route(console_enabled: bool) -> std::io::Result<impl S3Route>
replication::register_replication_route(&mut r)?;
site_replication::register_site_replication_route(&mut r)?;
profile_admin::register_profiling_route(&mut r)?;
tls_debug::register_tls_debug_route(&mut r)?;
kms::register_kms_route(&mut r)?;
oidc::register_oidc_route(&mut r)?;
rustfs/src/admin/route_registration_test.rs
+3 -1
View File
@@ -15,7 +15,7 @@
use crate::admin::{
handlers::{
audit, bucket_meta, heal, health, kms, module_switch, oidc, plugins_catalog, plugins_instances, pools, profile_admin,
quota, rebalance, replication, site_replication, sts, system, tier, user,
quota, rebalance, replication, site_replication, sts, system, tier, tls_debug, user,
},
router::{AdminOperation, S3Router},
};
@@ -59,6 +59,7 @@ fn register_admin_routes(router: &mut S3Router<AdminOperation>) {
replication::register_replication_route(router).expect("register replication route");
site_replication::register_site_replication_route(router).expect("register site replication route");
profile_admin::register_profiling_route(router).expect("register profile route");
tls_debug::register_tls_debug_route(router).expect("register tls debug route");
kms::register_kms_route(router).expect("register kms route");
oidc::register_oidc_route(router).expect("register oidc route");
}
@@ -157,6 +158,7 @@ fn test_register_routes_cover_representative_admin_paths() {
assert_route(&router, Method::PUT, &admin_path("/v3/site-replication/resync/op"));
assert_route(&router, Method::PUT, &admin_path("/v3/site-replication/state/edit"));
assert_route(&router, Method::GET, &admin_path("/debug/pprof/profile"));
assert_route(&router, Method::GET, &admin_path("/debug/tls/status"));
assert_route(&router, Method::POST, &admin_path("/v3/kms/create-key"));
assert_route(&router, Method::POST, &admin_path("/v3/kms/key/create"));
rustfs/src/app/object_usecase.rs
+4 -5
View File
@@ -92,7 +92,10 @@ use rustfs_s3select_api::{
query::{Context, Query},
};
use rustfs_s3select_query::get_global_db;
use rustfs_targets::EventName;
use rustfs_targets::{
EventName, extract_params_header, extract_resp_elements, get_request_host, get_request_port, get_request_user_agent,
};
use rustfs_utils::CompressionAlgorithm;
use rustfs_utils::http::{
AMZ_BUCKET_REPLICATION_STATUS, AMZ_CHECKSUM_MODE, AMZ_CHECKSUM_TYPE, AMZ_WEBSITE_REDIRECT_LOCATION, CONTENT_TYPE,
SUFFIX_ACTUAL_SIZE, SUFFIX_COMPRESSION, SUFFIX_COMPRESSION_SIZE, SUFFIX_REPLICATION_STATUS, SUFFIX_REPLICATION_TIMESTAMP,
@@ -108,10 +111,6 @@ use rustfs_utils::http::{
insert_str, remove_str,
};
use rustfs_utils::path::{is_dir_object, path_join_buf};
use rustfs_utils::{
CompressionAlgorithm, extract_params_header, extract_resp_elements, get_request_host, get_request_port,
get_request_user_agent,
};
use rustfs_zip::CompressionFormat;
use s3s::dto::*;
use s3s::header::{X_AMZ_RESTORE, X_AMZ_RESTORE_OUTPUT_PATH};
rustfs/src/main.rs
+11 -4
View File
@@ -221,12 +221,16 @@ async fn async_main() -> Result<()> {
debug!("rustls crypto provider already installed, skipping aws-lc-rs default install");
}
// Initialize TLS outbound material (root CAs, mTLS identity) if configured.
// Server-side TLS acceptor is built separately inside start_http_server()
// using the same TlsMaterialSnapshot loading logic.
// Server-side TLS acceptor is built separately inside start_http_server().
// Single load via tls-runtime; outbound is enriched with platform CAs and
// published to the global state before any HTTP listener starts.
if let Some(tls_path) = config.tls_path.as_deref().map(str::trim).filter(|path| !path.is_empty()) {
match rustfs::server::tls_material::TlsMaterialSnapshot::load(tls_path).await {
match rustfs::server::tls_material::load_tls_material(tls_path).await {
Ok(snapshot) => {
snapshot.apply_outbound().await;
use rustfs_tls_runtime::{TlsGeneration, publish_global_outbound_tls_state, record_tls_generation};
let generation = TlsGeneration(rustfs_common::get_global_outbound_tls_generation().saturating_add(1));
publish_global_outbound_tls_state(generation, &snapshot.outbound).await;
record_tls_generation("rustfs_server_startup", generation.0);
info!(target: "rustfs::main", "TLS outbound material initialized from {}", tls_path);
}
Err(e) => {
@@ -234,6 +238,9 @@ async fn async_main() -> Result<()> {
return Err(Error::other(e.to_string()));
}
}
if rustfs_obs::observability_metric_enabled() {
rustfs_tls_runtime::init_tls_metrics();
}
}
// Run parameters
rustfs/src/server/http.rs
+26 -12
View File
@@ -25,7 +25,9 @@ use crate::server::{
BodylessStatusFixLayer, ConditionalCorsLayer, EmptyBodyContentLengthCompatLayer, HeadRequestBodyFixLayer,
ObjectAttributesEtagFixLayer, PublicHealthEndpointLayer, RedirectLayer, RequestContextLayer, S3ErrorMessageCompatLayer,
},
tls_material::{TlsAcceptorHolder, TlsHandshakeFailureKind, TlsMaterialSnapshot, spawn_reload_loop},
tls_material::{
TlsAcceptorHolder, TlsHandshakeFailureKind, build_acceptor_from_loaded, load_tls_material, spawn_reload_loop,
},
};
use crate::storage;
use crate::storage::rpc::InternodeRpcService;
@@ -217,22 +219,34 @@ pub async fn start_http_server(
let tls_path = config.tls_path.as_deref().map(str::trim).unwrap_or_default();
let tls_path_configured = !tls_path.is_empty();
// Load TLS materials and build server acceptor.
// Note: outbound material (root CAs, mTLS identity) is already applied in main.rs.
let tls_snapshot = TlsMaterialSnapshot::load(tls_path)
.await
.map_err(|e| Error::other(e.to_string()))?;
let tls_acceptor = tls_snapshot.build_tls_acceptor(tls_path).await.map_err(|e| {
if tls_path_configured {
// Load TLS materials and build server acceptor in a single pass.
// Outbound material (root CAs, mTLS identity) was already published in main.rs;
// this load is needed for the server-side TLS acceptor and reload loop.
let tls_acceptor = if tls_path_configured {
let snapshot = load_tls_material(tls_path).await.map_err(|e| {
Error::other(format!(
"TLS is explicitly configured via RUSTFS_TLS_PATH/tls_path='{}' but TLS acceptor initialization failed: {}",
tls_path, e
))
} else {
Error::other(e.to_string())
})?;
let acceptor = build_acceptor_from_loaded(snapshot.server, std::path::Path::new(tls_path))
.await
.map_err(|e| Error::other(e.to_string()))?;
// Fail closed: if TLS was explicitly configured but no server certificates
// were found, refuse to start rather than silently falling back to plain HTTP.
match acceptor {
None => {
return Err(Error::other(format!(
"TLS is explicitly configured via RUSTFS_TLS_PATH/tls_path='{}' but no server certificates were found",
tls_path
)));
}
Some(a) => Some(a),
}
})?;
} else {
None
};
let tls_enabled = tls_acceptor.is_some();
let protocol = if tls_enabled { "https" } else { "http" };
rustfs/src/server/tls_material.rs
+222 -372
View File
@@ -12,32 +12,32 @@
// See the License for the specific language governing permissions and
// limitations under the License.
//! Unified TLS Material Snapshot
//! TLS material loading, enrichment, and acceptor construction.
//!
//! Provides a single loading point for all TLS materials, eliminating duplicate
//! directory scanning and PEM parsing between outbound and inbound paths.
//!
//! Usage:
//! 1. Call `TlsMaterialSnapshot::load(tls_path)` once at startup.
//! 2. Call `snapshot.apply_outbound()` to set global root CAs and mTLS identity.
//! 3. TLS acceptor construction is handled internally during server startup.
//! Single-load architecture: `rustfs_tls_runtime::TlsMaterialSnapshot` loads all
//! TLS materials (server certs + outbound CAs + mTLS identity) from disk in one
//! pass. This module enriches the outbound material with platform-specific CAs
//! (system roots, leaf-as-CA, mTLS env-var path overrides) and builds the server
//! TLS acceptor directly from the pre-loaded server material — no double reads.
use rustfs_common::{MtlsIdentityPem, set_global_mtls_identity, set_global_root_cert};
use rustfs_common::{MtlsIdentityPem, get_global_outbound_tls_generation};
use rustfs_config::{
DEFAULT_SERVER_MTLS_ENABLE, DEFAULT_TLS_KEYLOG, DEFAULT_TLS_RELOAD_ENABLE, DEFAULT_TLS_RELOAD_INTERVAL,
DEFAULT_TRUST_LEAF_CERT_AS_CA, DEFAULT_TRUST_SYSTEM_CA, ENV_MTLS_CLIENT_CERT, ENV_MTLS_CLIENT_KEY, ENV_SERVER_MTLS_ENABLE,
ENV_TLS_KEYLOG, ENV_TLS_RELOAD_ENABLE, ENV_TLS_RELOAD_INTERVAL, ENV_TRUST_LEAF_CERT_AS_CA, ENV_TRUST_SYSTEM_CA,
RUSTFS_CA_CERT, RUSTFS_CLIENT_CA_CERT_FILENAME, RUSTFS_CLIENT_CERT_FILENAME, RUSTFS_CLIENT_KEY_FILENAME, RUSTFS_PUBLIC_CERT,
RUSTFS_TLS_CERT, RUSTFS_TLS_KEY,
RUSTFS_CA_CERT, RUSTFS_CLIENT_CA_CERT_FILENAME, RUSTFS_CLIENT_CERT_FILENAME, RUSTFS_CLIENT_KEY_FILENAME, RUSTFS_TLS_CERT,
};
use rustfs_tls_runtime::{
ServerTlsMaterial as RuntimeServerTlsMaterial, TlsGeneration, TlsSource, WebPkiClientVerifierOptions,
build_webpki_client_verifier, create_multi_cert_resolver, publish_global_outbound_tls_state, record_tls_generation,
record_tls_reload_result, record_tls_reload_skipped,
};
use rustfs_utils::{get_env_bool, get_env_opt_str};
use rustls::pki_types::{CertificateDer, PrivateKeyDer, pem::PemObject};
use std::collections::HashMap;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::sync::RwLock;
use std::time::Duration;
use std::{fs, io};
use tokio_rustls::TlsAcceptor;
use tracing::{debug, info, warn};
@@ -54,310 +54,86 @@ const SYSTEM_CA_PATHS: &[&str] = &[
"/usr/share/pki/ca-trust-legacy/ca-bundle.legacy.crt", // RHEL legacy
];
/// Outbound TLS material for client connections (inter-node RPC).
#[derive(Debug, Clone)]
pub struct OutboundTlsMaterial {
/// Concatenated PEM-encoded root CA certificates.
pub root_ca_pem: Vec<u8>,
/// Optional mTLS client identity.
pub mtls_identity: Option<MtlsIdentityPem>,
}
// ── Public API ──
/// Complete TLS material snapshot loaded once at startup.
#[derive(Debug)]
pub struct TlsMaterialSnapshot {
/// Material for outbound client connections.
pub outbound: OutboundTlsMaterial,
}
impl TlsMaterialSnapshot {
/// Load all TLS materials from the given directory.
///
/// This is the single entry point that replaces both the old
/// `cert.rs::init_cert()` and `http.rs::setup_tls_acceptor()` loading logic.
pub async fn load(tls_path: &str) -> Result<Self, TlsMaterialError> {
if tls_path.is_empty() {
info!("No TLS path configured; skipping TLS material loading");
return Ok(Self::empty());
}
let tls_dir = PathBuf::from(tls_path);
// Load outbound material (root CAs + mTLS identity)
let outbound = load_outbound_material(&tls_dir).await?;
Ok(Self { outbound })
/// Load all TLS materials from the given directory in a single pass.
///
/// Uses `rustfs_tls_runtime::TlsMaterialSnapshot` as the single disk-read point,
/// then enriches the outbound material with platform-specific CAs that the runtime
/// crate does not handle (system roots, leaf-as-CA, mTLS env-var path overrides).
///
/// Returns the fully enriched snapshot ready for both outbound publishing and
/// acceptor construction.
pub async fn load_tls_material(tls_path: &str) -> Result<rustfs_tls_runtime::TlsMaterialSnapshot, TlsMaterialError> {
if tls_path.is_empty() {
return Err(TlsMaterialError::Io("TLS path is empty".into()));
}
/// Apply outbound material to global state (root CAs, mTLS identity).
pub async fn apply_outbound(&self) {
if !self.outbound.root_ca_pem.is_empty() {
set_global_root_cert(self.outbound.root_ca_pem.clone()).await;
info!("Configured custom root certificates for inter-node communication");
}
set_global_mtls_identity(self.outbound.mtls_identity.clone()).await;
}
let tls_source = TlsSource::from_directory(tls_path);
let mut snapshot = rustfs_tls_runtime::TlsMaterialSnapshot::load(&tls_source)
.await
.map_err(map_runtime_tls_error)?;
/// Build a `TlsAcceptorHolder` from the loaded snapshot.
///
/// This is the single place that constructs the server `ServerConfig`,
/// handling both multi-cert (SNI resolver) and single-cert fallback.
/// Returns `None` if no TLS certificates are available.
pub(crate) async fn build_tls_acceptor(&self, tls_path: &str) -> Result<Option<Arc<TlsAcceptorHolder>>, TlsMaterialError> {
if tls_path.is_empty() {
return Ok(None);
}
enrich_outbound(&mut snapshot.outbound, &PathBuf::from(tls_path)).await?;
let tls_dir = validate_server_tls_directory(tls_path)?;
let mtls_verifier = rustfs_utils::build_webpki_client_verifier(
rustfs_utils::WebPkiClientVerifierOptions::builder(&tls_dir, RUSTFS_CLIENT_CA_CERT_FILENAME, RUSTFS_CA_CERT)
.enabled(get_env_bool(ENV_SERVER_MTLS_ENABLE, DEFAULT_SERVER_MTLS_ENABLE))
.build(),
)
.map_err(|e| TlsMaterialError::Io(format!("build mTLS verifier: {e}")))?;
match load_server_certificate_material(&tls_dir)? {
ServerCertificateMaterial::SingleCert { certs, key } => {
let config = build_server_config(ServerCertSource::SingleCert { certs, key }, mtls_verifier)?;
info!("Created TLS acceptor with root single certificate");
let acceptor = Arc::new(TlsAcceptor::from(Arc::new(config)));
Ok(Some(Arc::new(TlsAcceptorHolder::new(acceptor))))
}
ServerCertificateMaterial::MultiCert { cert_key_pairs } => {
let resolver = rustfs_utils::create_multi_cert_resolver(cert_key_pairs)
.map_err(|e| TlsMaterialError::Parse(format!("build multi-cert resolver: {e}")))?;
let config = build_server_config(ServerCertSource::Resolver(Arc::new(resolver)), mtls_verifier)?;
info!("Created TLS acceptor with SNI resolver");
let acceptor = Arc::new(TlsAcceptor::from(Arc::new(config)));
Ok(Some(Arc::new(TlsAcceptorHolder::new(acceptor))))
}
}
}
fn empty() -> Self {
Self {
outbound: OutboundTlsMaterial {
root_ca_pem: Vec::new(),
mtls_identity: None,
},
}
}
Ok(snapshot)
}
fn validate_server_tls_directory(tls_path: &str) -> Result<PathBuf, TlsMaterialError> {
let tls_dir = PathBuf::from(tls_path);
let metadata = fs::metadata(&tls_dir).map_err(|e| {
let kind = if e.kind() == io::ErrorKind::NotFound {
"TLS directory does not exist"
} else {
"Failed to inspect TLS directory"
};
TlsMaterialError::Io(format!("{kind}: {} ({e})", tls_dir.display()))
})?;
if !metadata.is_dir() {
return Err(TlsMaterialError::Io(format!("TLS path is not a directory: {}", tls_dir.display())));
}
Ok(tls_dir)
}
fn load_server_certificate_material(tls_dir: &Path) -> Result<ServerCertificateMaterial, TlsMaterialError> {
let root_cert_pair = inspect_root_server_cert_pair(tls_dir);
if has_discoverable_domain_cert_pair(tls_dir)? {
return load_multi_cert_material(tls_dir);
}
match root_cert_pair {
RootCertPairState::Complete { cert_path, key_path } => load_root_single_cert_material(&cert_path, &key_path),
RootCertPairState::MissingCert { cert_path } => Err(TlsMaterialError::Io(format!(
"TLS root certificate file missing: {}",
cert_path.display()
))),
RootCertPairState::MissingKey { key_path } => {
Err(TlsMaterialError::Io(format!("TLS root private key file missing: {}", key_path.display())))
}
RootCertPairState::Absent => Err(TlsMaterialError::Io(format!(
"No usable TLS certificate material found under '{}'",
tls_dir.display()
))),
}
}
fn inspect_root_server_cert_pair(tls_dir: &Path) -> RootCertPairState {
let cert_path = tls_dir.join(RUSTFS_TLS_CERT);
let key_path = tls_dir.join(RUSTFS_TLS_KEY);
let cert_exists = cert_path.exists();
let key_exists = key_path.exists();
match (cert_exists, key_exists) {
(true, true) => RootCertPairState::Complete { cert_path, key_path },
(false, false) => RootCertPairState::Absent,
(false, true) => RootCertPairState::MissingCert { cert_path },
(true, false) => RootCertPairState::MissingKey { key_path },
}
}
fn has_discoverable_domain_cert_pair(tls_dir: &Path) -> Result<bool, TlsMaterialError> {
for entry in
fs::read_dir(tls_dir).map_err(|e| TlsMaterialError::Io(format!("read TLS directory {}: {e}", tls_dir.display())))?
{
let entry = entry.map_err(|e| TlsMaterialError::Io(format!("read TLS directory entry in {}: {e}", tls_dir.display())))?;
let path = entry.path();
if !path.is_dir() {
continue;
}
let Some(domain_name) = path.file_name().and_then(|name| name.to_str()) else {
continue;
};
if !is_discoverable_cert_domain_dir(domain_name) {
continue;
}
if path.join(RUSTFS_TLS_CERT).exists() && path.join(RUSTFS_TLS_KEY).exists() {
return Ok(true);
}
}
Ok(false)
}
fn is_discoverable_cert_domain_dir(domain_name: &str) -> bool {
!domain_name.starts_with('.')
}
fn load_root_single_cert_material(cert_path: &Path, key_path: &Path) -> Result<ServerCertificateMaterial, TlsMaterialError> {
let cert_path_str = cert_path
.to_str()
.ok_or_else(|| TlsMaterialError::Io(format!("invalid UTF-8 in TLS root certificate path: {cert_path:?}")))?;
let key_path_str = key_path
.to_str()
.ok_or_else(|| TlsMaterialError::Io(format!("invalid UTF-8 in TLS root private key path: {key_path:?}")))?;
let certs = rustfs_utils::load_certs(cert_path_str)
.map_err(|e| TlsMaterialError::Io(format!("load root TLS certificate {}: {e}", cert_path.display())))?;
let key = rustfs_utils::load_private_key(key_path_str)
.map_err(|e| TlsMaterialError::Io(format!("load root TLS private key {}: {e}", key_path.display())))?;
Ok(ServerCertificateMaterial::SingleCert { certs, key })
}
fn load_multi_cert_material(tls_dir: &Path) -> Result<ServerCertificateMaterial, TlsMaterialError> {
let cert_key_pairs = rustfs_utils::load_all_certs_from_directory(
rustfs_utils::CertDirectoryLoadOptions::builder(tls_dir, RUSTFS_TLS_CERT, RUSTFS_TLS_KEY).build(),
/// Build a TLS acceptor from pre-loaded server material.
///
/// Takes the server material already loaded by `load_tls_material` and constructs
/// the `rustls::ServerConfig` without any additional disk reads (except the mTLS
/// client CA verifier, which reads the client CA cert from disk).
pub(crate) async fn build_acceptor_from_loaded(
server: Option<RuntimeServerTlsMaterial>,
tls_dir: &Path,
) -> Result<Option<Arc<TlsAcceptorHolder>>, TlsMaterialError> {
let mtls_verifier = build_webpki_client_verifier(
WebPkiClientVerifierOptions::builder(tls_dir, RUSTFS_CLIENT_CA_CERT_FILENAME, RUSTFS_CA_CERT)
.enabled(get_env_bool(ENV_SERVER_MTLS_ENABLE, DEFAULT_SERVER_MTLS_ENABLE))
.build(),
)
.map_err(|e| TlsMaterialError::Io(format!("discover multi-cert TLS certificates under '{}': {e}", tls_dir.display())))?;
.map_err(|e| TlsMaterialError::Io(format!("build mTLS verifier: {e}")))?;
Ok(ServerCertificateMaterial::MultiCert { cert_key_pairs })
}
enum RootCertPairState {
Complete { cert_path: PathBuf, key_path: PathBuf },
MissingCert { cert_path: PathBuf },
MissingKey { key_path: PathBuf },
Absent,
}
enum ServerCertificateMaterial {
SingleCert {
certs: Vec<CertificateDer<'static>>,
key: PrivateKeyDer<'static>,
},
MultiCert {
cert_key_pairs: HashMap<String, (Vec<CertificateDer<'static>>, PrivateKeyDer<'static>)>,
},
}
// ── Server Config Construction ──
/// Certificate source for building a `ServerConfig`.
enum ServerCertSource {
/// Pre-built SNI resolver from multi-cert directory.
Resolver(Arc<dyn rustls::server::ResolvesServerCert + Send + Sync>),
/// Single certificate/key pair.
SingleCert {
certs: Vec<CertificateDer<'static>>,
key: PrivateKeyDer<'static>,
},
}
/// Build a `ServerConfig` with standardized ALPN, session cache, and key log settings.
///
/// This is the single place for `ServerConfig` construction, used by both
/// initial startup and hot-reload.
fn build_server_config(
cert_source: ServerCertSource,
mtls_verifier: Option<Arc<dyn rustls::server::danger::ClientCertVerifier>>,
) -> Result<rustls::ServerConfig, TlsMaterialError> {
let mut config = match cert_source {
ServerCertSource::Resolver(resolver) => {
if let Some(verifier) = mtls_verifier {
rustls::ServerConfig::builder()
.with_client_cert_verifier(verifier)
.with_cert_resolver(resolver)
} else {
rustls::ServerConfig::builder()
.with_no_client_auth()
.with_cert_resolver(resolver)
}
match server {
Some(RuntimeServerTlsMaterial::SingleCert { certs, key }) => {
let config = build_server_config(ServerCertSource::SingleCert { certs, key }, mtls_verifier)?;
info!("Created TLS acceptor with root single certificate");
let acceptor = Arc::new(TlsAcceptor::from(Arc::new(config)));
Ok(Some(Arc::new(TlsAcceptorHolder::new(acceptor))))
}
ServerCertSource::SingleCert { certs, key } => {
if let Some(verifier) = mtls_verifier {
rustls::ServerConfig::builder()
.with_client_cert_verifier(verifier)
.with_single_cert(certs, key)
.map_err(|e| TlsMaterialError::Io(format!("configure single cert with mTLS: {e}")))?
} else {
rustls::ServerConfig::builder()
.with_no_client_auth()
.with_single_cert(certs, key)
.map_err(|e| TlsMaterialError::Io(format!("configure single cert: {e}")))?
}
Some(RuntimeServerTlsMaterial::MultiCert { cert_key_pairs }) => {
let resolver = create_multi_cert_resolver(cert_key_pairs)
.map_err(|e| TlsMaterialError::Parse(format!("build multi-cert resolver: {e}")))?;
let config = build_server_config(ServerCertSource::Resolver(Arc::new(resolver)), mtls_verifier)?;
info!("Created TLS acceptor with SNI resolver");
let acceptor = Arc::new(TlsAcceptor::from(Arc::new(config)));
Ok(Some(Arc::new(TlsAcceptorHolder::new(acceptor))))
}
};
config.alpn_protocols = vec![b"h2".to_vec(), b"http/1.1".to_vec(), b"http/1.0".to_vec()];
config.session_storage = rustls::server::ServerSessionMemoryCache::new(10000);
if tls_key_log() {
config.key_log = Arc::new(rustls::KeyLogFile::new());
None => Ok(None),
}
Ok(config)
}
/// Checks if TLS key logging is enabled.
///
/// # Returns
/// * A boolean indicating whether TLS key logging is enabled based on the `RUSTFS_TLS_KEYLOG` environment variable.
///
fn tls_key_log() -> bool {
get_env_bool(ENV_TLS_KEYLOG, DEFAULT_TLS_KEYLOG)
}
// ── Outbound Material Loading ──
/// Load root CA certificates and mTLS identity for outbound connections.
async fn load_outbound_material(tls_dir: &Path) -> Result<OutboundTlsMaterial, TlsMaterialError> {
let mut root_ca_pem = Vec::new();
// ── Outbound Enrichment ──
/// Enrich the outbound TLS material with platform-specific additions that the
/// tls-runtime crate does not handle:
/// 1. Optional: server leaf cert as root CA (`RUSTFS_TRUST_LEAF_CERT_AS_CA`)
/// 2. Optional: system root CAs from platform paths (`RUSTFS_TRUST_SYSTEM_CA`)
/// 3. Optional: mTLS identity from env-var-overridden paths
async fn enrich_outbound(outbound: &mut rustfs_tls_runtime::OutboundTlsMaterial, tls_dir: &Path) -> Result<(), TlsMaterialError> {
// 1. Optional: load leaf certs as root CAs
if get_env_bool(ENV_TRUST_LEAF_CERT_AS_CA, DEFAULT_TRUST_LEAF_CERT_AS_CA)
&& load_cert_file_by_name(tls_dir, RUSTFS_TLS_CERT, &mut root_ca_pem).await
&& load_cert_file_by_name(tls_dir, RUSTFS_TLS_CERT, &mut outbound.root_ca_pem).await
{
info!("Loaded leaf certificate(s) as root CA as per RUSTFS_TRUST_LEAF_CERT_AS_CA");
}
// 2. Load public.crt and ca.crt
load_cert_file(&tls_dir.join(RUSTFS_PUBLIC_CERT), &mut root_ca_pem, "CA certificate").await;
load_cert_file(&tls_dir.join(RUSTFS_CA_CERT), &mut root_ca_pem, "CA certificate").await;
// 3. Optional: load system root CAs
// 2. Optional: load system root CAs
if get_env_bool(ENV_TRUST_SYSTEM_CA, DEFAULT_TRUST_SYSTEM_CA) {
let mut system_loaded = false;
for path in SYSTEM_CA_PATHS {
if load_cert_file(Path::new(path), &mut root_ca_pem, "system root certificates").await {
if load_cert_file(Path::new(path), &mut outbound.root_ca_pem, "system root certificates").await {
system_loaded = true;
info!("Loaded system root certificates from {}", path);
break;
@@ -370,23 +146,31 @@ async fn load_outbound_material(tls_dir: &Path) -> Result<OutboundTlsMaterial, T
info!("Loading system root certificates disabled via RUSTFS_TRUST_SYSTEM_CA");
}
// 4. Load optional mTLS identity
let mtls_identity = load_mtls_identity(tls_dir).await?;
// 3. Optional: override mTLS identity from env-var paths
let env_cert = get_env_opt_str(ENV_MTLS_CLIENT_CERT);
let env_key = get_env_opt_str(ENV_MTLS_CLIENT_KEY);
if (env_cert.is_some() || env_key.is_some())
&& let Some(identity) = load_mtls_identity_from_overridden_paths(env_cert.as_deref(), env_key.as_deref(), tls_dir).await?
{
outbound.mtls_identity = Some(identity);
}
Ok(OutboundTlsMaterial {
root_ca_pem,
mtls_identity,
})
Ok(())
}
/// Load mTLS client identity from the TLS directory.
async fn load_mtls_identity(tls_dir: &Path) -> Result<Option<MtlsIdentityPem>, TlsMaterialError> {
let client_cert_path = match get_env_opt_str(ENV_MTLS_CLIENT_CERT) {
/// Load mTLS client identity when env-var path overrides are set.
/// If neither env var is set, returns Ok(None) (the tls-runtime already loaded
/// identity from the default path).
async fn load_mtls_identity_from_overridden_paths(
env_cert: Option<&str>,
env_key: Option<&str>,
tls_dir: &Path,
) -> Result<Option<MtlsIdentityPem>, TlsMaterialError> {
let client_cert_path = match env_cert {
Some(p) => PathBuf::from(p),
None => tls_dir.join(RUSTFS_CLIENT_CERT_FILENAME),
};
let client_key_path = match get_env_opt_str(ENV_MTLS_CLIENT_KEY) {
let client_key_path = match env_key {
Some(p) => PathBuf::from(p),
None => tls_dir.join(RUSTFS_CLIENT_KEY_FILENAME),
};
@@ -406,7 +190,6 @@ async fn load_mtls_identity(tls_dir: &Path) -> Result<Option<MtlsIdentityPem>, T
.await
.map_err(|e| TlsMaterialError::Io(format!("read client key {client_key_path:?}: {e}")))?;
// Validate parse-ability
let mut reader = std::io::Cursor::new(&cert_pem);
if CertificateDer::pem_reader_iter(&mut reader).next().is_none() {
return Err(TlsMaterialError::Parse("no valid certificate in client cert PEM".into()));
@@ -418,6 +201,17 @@ async fn load_mtls_identity(tls_dir: &Path) -> Result<Option<MtlsIdentityPem>, T
Ok(Some(MtlsIdentityPem { cert_pem, key_pem }))
}
// ── Helpers ──
fn map_runtime_tls_error(err: rustfs_tls_runtime::TlsRuntimeError) -> TlsMaterialError {
match &err {
rustfs_tls_runtime::TlsRuntimeError::Material(msg) | rustfs_tls_runtime::TlsRuntimeError::Publication(msg) => {
TlsMaterialError::Parse(msg.clone())
}
_ => TlsMaterialError::Io(err.to_string()),
}
}
/// Load a single certificate file and append PEM data.
/// Returns true if the file was successfully loaded.
async fn load_cert_file(path: &Path, pem_data: &mut Vec<u8>, desc: &str) -> bool {
@@ -457,17 +251,16 @@ async fn load_cert_file_by_name(dir: &Path, cert_name: &str, pem_data: &mut Vec<
if fname == cert_name && load_cert_file(&entry.path(), pem_data, "certificate").await {
loaded = true;
}
} else if ft.is_dir() {
// Only check direct subdirectories (one level deep)
if let Ok(mut sub_rd) = tokio::fs::read_dir(&entry.path()).await {
while let Ok(Some(sub_entry)) = sub_rd.next_entry().await {
if let Ok(sub_ft) = sub_entry.file_type().await
&& sub_ft.is_file()
{
let fname = sub_entry.file_name().to_string_lossy().to_string();
if fname == cert_name && load_cert_file(&sub_entry.path(), pem_data, "certificate").await {
loaded = true;
}
} else if ft.is_dir()
&& let Ok(mut sub_rd) = tokio::fs::read_dir(&entry.path()).await
{
while let Ok(Some(sub_entry)) = sub_rd.next_entry().await {
if let Ok(sub_ft) = sub_entry.file_type().await
&& sub_ft.is_file()
{
let fname = sub_entry.file_name().to_string_lossy().to_string();
if fname == cert_name && load_cert_file(&sub_entry.path(), pem_data, "certificate").await {
loaded = true;
}
}
}
@@ -476,12 +269,66 @@ async fn load_cert_file_by_name(dir: &Path, cert_name: &str, pem_data: &mut Vec<
loaded
}
/// Errors that can occur during TLS material loading.
// ── Server Config Construction ──
enum ServerCertSource {
Resolver(Arc<dyn rustls::server::ResolvesServerCert + Send + Sync>),
SingleCert {
certs: Vec<CertificateDer<'static>>,
key: PrivateKeyDer<'static>,
},
}
fn build_server_config(
cert_source: ServerCertSource,
mtls_verifier: Option<Arc<dyn rustls::server::danger::ClientCertVerifier>>,
) -> Result<rustls::ServerConfig, TlsMaterialError> {
let mut config = match cert_source {
ServerCertSource::Resolver(resolver) => {
if let Some(verifier) = mtls_verifier {
rustls::ServerConfig::builder()
.with_client_cert_verifier(verifier)
.with_cert_resolver(resolver)
} else {
rustls::ServerConfig::builder()
.with_no_client_auth()
.with_cert_resolver(resolver)
}
}
ServerCertSource::SingleCert { certs, key } => {
if let Some(verifier) = mtls_verifier {
rustls::ServerConfig::builder()
.with_client_cert_verifier(verifier)
.with_single_cert(certs, key)
.map_err(|e| TlsMaterialError::Io(format!("configure single cert with mTLS: {e}")))?
} else {
rustls::ServerConfig::builder()
.with_no_client_auth()
.with_single_cert(certs, key)
.map_err(|e| TlsMaterialError::Io(format!("configure single cert: {e}")))?
}
}
};
config.alpn_protocols = vec![b"h2".to_vec(), b"http/1.1".to_vec(), b"http/1.0".to_vec()];
config.session_storage = rustls::server::ServerSessionMemoryCache::new(10000);
if tls_key_log() {
config.key_log = Arc::new(rustls::KeyLogFile::new());
}
Ok(config)
}
fn tls_key_log() -> bool {
get_env_bool(ENV_TLS_KEYLOG, DEFAULT_TLS_KEYLOG)
}
// ── Errors ──
#[derive(Debug)]
pub enum TlsMaterialError {
/// I/O error (file read, directory access).
Io(String),
/// PEM parsing error.
Parse(String),
}
@@ -498,7 +345,6 @@ impl std::error::Error for TlsMaterialError {}
// ── TLS Handshake Error Classification ──
/// Structured classification of TLS handshake failures.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum TlsHandshakeFailureKind {
UnexpectedEof,
@@ -509,7 +355,6 @@ pub(crate) enum TlsHandshakeFailureKind {
}
impl TlsHandshakeFailureKind {
/// Classify a TLS accept error into a structured failure kind.
pub(crate) fn classify(err_msg: &str) -> Self {
if err_msg.contains("unexpected EOF") || err_msg.contains("handshake eof") {
Self::UnexpectedEof
@@ -524,7 +369,6 @@ impl TlsHandshakeFailureKind {
}
}
/// Metric label string for Prometheus.
pub(crate) fn as_str(self) -> &'static str {
match self {
Self::UnexpectedEof => "UNEXPECTED_EOF",
@@ -538,10 +382,6 @@ impl TlsHandshakeFailureKind {
// ── TLS Acceptor Holder (for hot reload) ──
/// Holds the current TLS acceptor and supports atomic swap for certificate rotation.
///
/// Uses `RwLock` so that multiple readers (per-connection `get()` calls)
/// do not block each other. The write lock is held only briefly during swap.
pub(crate) struct TlsAcceptorHolder {
current: RwLock<Arc<TlsAcceptor>>,
}
@@ -553,7 +393,6 @@ impl TlsAcceptorHolder {
}
}
/// Get the current TLS acceptor for handling a new connection.
#[inline]
pub(crate) fn get(&self) -> Arc<TlsAcceptor> {
match self.current.read() {
@@ -562,7 +401,6 @@ impl TlsAcceptorHolder {
}
}
/// Atomically replace the TLS acceptor with a new one.
fn swap(&self, new_holder: &TlsAcceptorHolder) {
let new_acceptor = new_holder.get();
match self.current.write() {
@@ -575,7 +413,11 @@ impl TlsAcceptorHolder {
}
}
// ── Reload Loop ──
/// Spawn a background task that periodically checks for TLS certificate changes.
/// Single load per tick: loads once via tls-runtime, enriches, publishes outbound,
/// and builds acceptor — no double reads.
pub(crate) fn spawn_reload_loop(tls_path: String, holder: Arc<TlsAcceptorHolder>) {
let enabled = get_env_bool(ENV_TLS_RELOAD_ENABLE, DEFAULT_TLS_RELOAD_ENABLE);
if !enabled {
@@ -584,31 +426,49 @@ pub(crate) fn spawn_reload_loop(tls_path: String, holder: Arc<TlsAcceptorHolder>
}
let interval_secs = rustfs_utils::get_env_u64(ENV_TLS_RELOAD_INTERVAL, DEFAULT_TLS_RELOAD_INTERVAL).max(5);
let tls_source = TlsSource::from_directory(&tls_path);
info!("TLS certificate hot reload enabled, checking every {}s", interval_secs);
tokio::spawn(async move {
let mut interval = tokio::time::interval(Duration::from_secs(interval_secs));
let tls_dir = PathBuf::from(&tls_path);
loop {
interval.tick().await;
match TlsMaterialSnapshot::load(&tls_path).await {
Ok(snapshot) => {
// Always refresh outbound material (root CAs, mTLS identity) on reload.
snapshot.apply_outbound().await;
match rustfs_tls_runtime::TlsMaterialSnapshot::load(&tls_source).await {
Ok(mut snapshot) => {
if let Err(e) = enrich_outbound(&mut snapshot.outbound, &tls_dir).await {
record_tls_reload_result("rustfs_server_reload_loop", "enrich_err", None, None);
warn!("TLS outbound enrichment failed (will retry): {}", e);
continue;
}
match snapshot.build_tls_acceptor(&tls_path).await {
let generation = get_global_outbound_tls_generation().saturating_add(1);
publish_global_outbound_tls_state(TlsGeneration(generation), &snapshot.outbound).await;
record_tls_generation("rustfs_server_reload_loop", generation);
if !snapshot.outbound.root_ca_pem.is_empty() {
info!("Configured custom root certificates for inter-node communication");
}
match build_acceptor_from_loaded(snapshot.server, &tls_dir).await {
Ok(Some(new_holder)) => {
info!("TLS certificates reloaded successfully");
holder.swap(&new_holder);
record_tls_reload_result("rustfs_server_reload_loop", "ok", None, Some(generation));
}
Ok(None) => {
record_tls_reload_skipped("rustfs_server_reload_loop", "no_acceptor");
warn!("TLS reload returned no acceptor despite configured TLS path; keeping previous acceptor")
}
Err(e) => warn!("TLS certificate reload failed (will retry): {}", e),
Err(e) => {
record_tls_reload_result("rustfs_server_reload_loop", "acceptor_err", None, Some(generation));
warn!("TLS certificate reload failed (will retry): {}", e)
}
}
}
Err(e) => {
record_tls_reload_result("rustfs_server_reload_loop", "load_err", None, None);
warn!("TLS material reload failed (will retry): {}", e);
}
}
@@ -636,20 +496,19 @@ mod tests {
fn write_test_cert_pair(dir: &Path, subject: &str) {
let CertifiedKey { cert, signing_key } = rcgen::generate_simple_self_signed(vec![subject.to_string()]).unwrap();
fs::write(dir.join(RUSTFS_TLS_CERT), cert.pem()).unwrap();
fs::write(dir.join(RUSTFS_TLS_KEY), signing_key.serialize_pem()).unwrap();
fs::write(dir.join(rustfs_config::RUSTFS_TLS_KEY), signing_key.serialize_pem()).unwrap();
}
#[tokio::test]
async fn build_tls_acceptor_accepts_root_single_cert_with_trailing_slash() {
async fn build_acceptor_accepts_root_single_cert_with_trailing_slash() {
ensure_rustls_crypto_provider();
let temp_dir = TempDir::new().unwrap();
write_test_cert_pair(temp_dir.path(), "localhost");
let snapshot = TlsMaterialSnapshot::load(&format!("{}/", temp_dir.path().display()))
let snapshot = load_tls_material(&format!("{}/", temp_dir.path().display()))
.await
.expect("TLS material load should succeed");
let acceptor = snapshot
.build_tls_acceptor(&format!("{}/", temp_dir.path().display()))
let acceptor = build_acceptor_from_loaded(snapshot.server, temp_dir.path())
.await
.expect("root single-cert TLS acceptor should build");
@@ -657,7 +516,7 @@ mod tests {
}
#[tokio::test]
async fn build_tls_acceptor_accepts_symlinked_root_single_cert_directory() {
async fn build_acceptor_accepts_symlinked_root_single_cert_directory() {
ensure_rustls_crypto_provider();
let temp_dir = TempDir::new().unwrap();
let real_dir = temp_dir.path().join("tls-real");
@@ -670,11 +529,10 @@ mod tests {
#[cfg(windows)]
std::os::windows::fs::symlink_dir(&real_dir, &symlink_dir).unwrap();
let snapshot = TlsMaterialSnapshot::load(symlink_dir.to_str().unwrap())
let snapshot = load_tls_material(symlink_dir.to_str().unwrap())
.await
.expect("TLS material load through symlink should succeed");
let acceptor = snapshot
.build_tls_acceptor(symlink_dir.to_str().unwrap())
let acceptor = build_acceptor_from_loaded(snapshot.server, &symlink_dir)
.await
.expect("TLS acceptor should build through symlink");
@@ -682,62 +540,55 @@ mod tests {
}
#[tokio::test]
async fn build_tls_acceptor_rejects_missing_tls_directory() {
async fn build_acceptor_rejects_missing_tls_directory() {
let temp_dir = TempDir::new().unwrap();
let missing_dir = temp_dir.path().join("missing");
let snapshot = TlsMaterialSnapshot::empty();
let err = match snapshot.build_tls_acceptor(missing_dir.to_str().unwrap()).await {
Ok(_) => panic!("missing TLS directory should fail"),
Err(err) => err,
};
let err = load_tls_material(missing_dir.to_str().unwrap())
.await
.expect_err("missing TLS directory should fail");
assert!(err.to_string().contains("TLS directory does not exist"));
}
#[tokio::test]
async fn build_tls_acceptor_rejects_tls_path_that_is_not_directory() {
async fn build_acceptor_rejects_tls_path_that_is_not_directory() {
let temp_dir = TempDir::new().unwrap();
let file_path = temp_dir.path().join("tls-file");
fs::write(&file_path, "not-a-directory").unwrap();
let snapshot = TlsMaterialSnapshot::empty();
let err = match snapshot.build_tls_acceptor(file_path.to_str().unwrap()).await {
Ok(_) => panic!("regular file TLS path should fail"),
Err(err) => err,
};
let err = load_tls_material(file_path.to_str().unwrap())
.await
.expect_err("regular file TLS path should fail");
assert!(err.to_string().contains("TLS path is not a directory"));
}
#[tokio::test]
async fn build_tls_acceptor_rejects_empty_tls_directory() {
async fn build_acceptor_returns_none_for_empty_tls_directory() {
let temp_dir = TempDir::new().unwrap();
let snapshot = TlsMaterialSnapshot::load(temp_dir.path().to_str().unwrap())
let snapshot = load_tls_material(temp_dir.path().to_str().unwrap())
.await
.expect("outbound TLS load for empty dir should succeed");
.expect("TLS material load for empty dir should succeed");
let err = match snapshot.build_tls_acceptor(temp_dir.path().to_str().unwrap()).await {
Ok(_) => panic!("empty TLS directory should fail"),
Err(err) => err,
};
assert!(err.to_string().contains("No usable TLS certificate material found"));
let acceptor = build_acceptor_from_loaded(snapshot.server, temp_dir.path())
.await
.expect("empty TLS directory should be treated as no acceptor");
assert!(acceptor.is_none());
}
#[tokio::test]
async fn build_tls_acceptor_still_supports_multi_cert_directories() {
async fn build_acceptor_still_supports_multi_cert_directories() {
ensure_rustls_crypto_provider();
let temp_dir = TempDir::new().unwrap();
let domain_dir = temp_dir.path().join("example.com");
fs::create_dir(&domain_dir).unwrap();
write_test_cert_pair(&domain_dir, "example.com");
let snapshot = TlsMaterialSnapshot::load(temp_dir.path().to_str().unwrap())
let snapshot = load_tls_material(temp_dir.path().to_str().unwrap())
.await
.expect("TLS material load for multi-cert dir should succeed");
let acceptor = snapshot
.build_tls_acceptor(temp_dir.path().to_str().unwrap())
let acceptor = build_acceptor_from_loaded(snapshot.server, temp_dir.path())
.await
.expect("multi-cert TLS acceptor should build");
@@ -745,7 +596,7 @@ mod tests {
}
#[tokio::test]
async fn build_tls_acceptor_prefers_multi_cert_when_root_and_domain_pairs_both_exist() {
async fn build_acceptor_prefers_multi_cert_when_root_and_domain_pairs_both_exist() {
ensure_rustls_crypto_provider();
let temp_dir = TempDir::new().unwrap();
write_test_cert_pair(temp_dir.path(), "default.local");
@@ -754,11 +605,10 @@ mod tests {
fs::create_dir(&domain_dir).unwrap();
write_test_cert_pair(&domain_dir, "example.com");
let snapshot = TlsMaterialSnapshot::load(temp_dir.path().to_str().unwrap())
let snapshot = load_tls_material(temp_dir.path().to_str().unwrap())
.await
.expect("TLS material load for mixed root/domain layout should succeed");
let acceptor = snapshot
.build_tls_acceptor(temp_dir.path().to_str().unwrap())
let acceptor = build_acceptor_from_loaded(snapshot.server, temp_dir.path())
.await
.expect("multi-cert TLS acceptor should still build when root pair also exists");
rustfs/src/storage/helper.rs
+2 -2
View File
@@ -27,10 +27,10 @@ use rustfs_io_metrics::record_s3_op;
use rustfs_notify::{EventArgsBuilder, notifier_global};
use rustfs_s3_ops::{S3Operation, operation_matches_event_name};
use rustfs_s3_types::EventName;
use rustfs_utils::{
use rustfs_targets::{
extract_params_header, extract_req_params, extract_resp_elements, get_request_host, get_request_port, get_request_user_agent,
http::headers::AMZ_REQUEST_ID,
};
use rustfs_utils::http::headers::AMZ_REQUEST_ID;
use s3s::{S3Request, S3Response, S3Result};
use serde_json::Value;
use std::future::Future;