From d74e6eb04201f554af3138a3efb8a5b0505d1bb0 Mon Sep 17 00:00:00 2001 From: houseme Date: Sun, 24 May 2026 14:41:15 +0800 Subject: [PATCH] refactor(tls): centralize runtime foundation (#3065) * refactor(targets): move notify net helpers from utils * refactor(tls): centralize runtime foundation * refactor(targets): move notify net helpers from utils * refactor(tls): centralize runtime foundation * feat(tls-runtime): add TLS debug state and admin handler * refactor(tls-runtime): unify TLS debug consumer status view * fix(tls): address PR3065 review feedback * refactor(tls): align debug status payload types * refactor(targets): harden TLS hot reload paths * fix(targets): resolve review-4348251652 findings * fix(targets): finalize tls runtime review follow-ups * fix(targets): harden tls reload and review follow-ups * fix(targets): align tls reload handling across targets * fix(targets): finalize tls reload state and metrics updates * chore(deps): trim unused TLS deps * style(targets): normalize TLS reload formatting * refactor(targets): introduce tls runtime adapter path * chore: update workspace manifests for tls refactor * fix(tls): stabilize material reload and audit workflow * fix(targets): refresh tls fingerprint flow across sinks * fix(tls): align runtime coordinator and http reader updates * fix(sftp): simplify protocol error mapping * fix(tls): harmonize material loading behavior * fix(server): finalize tls material wiring in startup flow * fix(protos): tighten tls generation cache and deps --- .github/workflows/audit.yml | 4 +- Cargo.lock | 224 +++--- Cargo.toml | 6 +- crates/common/src/globals.rs | 12 + crates/ecstore/Cargo.toml | 2 + crates/ecstore/src/client/transition_api.rs | 84 +-- crates/protocols/Cargo.toml | 6 +- crates/protocols/src/ftps/server.rs | 21 +- crates/protocols/src/lib.rs | 3 - crates/protocols/src/sftp/errors.rs | 7 + crates/protocols/src/webdav/server.rs | 22 +- crates/protos/Cargo.toml | 5 +- crates/protos/src/lib.rs | 75 +- crates/rio/Cargo.toml | 4 +- crates/rio/src/http_reader.rs | 169 +++-- crates/targets/Cargo.toml | 9 +- crates/targets/src/check.rs | 4 +- crates/targets/src/lib.rs | 2 + .../{utils/src/notify => targets/src}/net.rs | 380 +++++----- crates/targets/src/runtime/mod.rs | 1 + crates/targets/src/runtime/tls/adapter.rs | 217 ++++++ crates/targets/src/runtime/tls/config.rs | 20 + crates/targets/src/runtime/tls/coordinator.rs | 656 ++++++++++++++++++ crates/targets/src/runtime/tls/fingerprint.rs | 160 +++++ crates/targets/src/runtime/tls/metrics.rs | 63 ++ crates/targets/src/runtime/tls/mod.rs | 41 ++ crates/targets/src/runtime/tls/state.rs | 127 ++++ crates/targets/src/runtime/tls/trait.rs | 68 ++ crates/targets/src/runtime/tls/validate.rs | 58 ++ crates/targets/src/target/amqp.rs | 112 ++- crates/targets/src/target/kafka.rs | 110 ++- crates/targets/src/target/mod.rs | 49 ++ crates/targets/src/target/mqtt.rs | 111 ++- crates/targets/src/target/mysql.rs | 224 ++++-- crates/targets/src/target/nats.rs | 152 +++- crates/targets/src/target/postgres.rs | 113 ++- crates/targets/src/target/pulsar.rs | 150 +++- crates/targets/src/target/redis.rs | 125 +++- crates/targets/src/target/webhook.rs | 115 ++- crates/tls-runtime/Cargo.toml | 50 ++ crates/{utils => tls-runtime}/src/certs.rs | 358 +--------- crates/tls-runtime/src/config.rs | 51 ++ crates/tls-runtime/src/coordinator.rs | 226 ++++++ crates/tls-runtime/src/debug.rs | 106 +++ crates/tls-runtime/src/error.rs | 32 + crates/tls-runtime/src/fingerprint.rs | 48 ++ crates/tls-runtime/src/lib.rs | 126 ++++ crates/tls-runtime/src/material.rs | 200 ++++++ crates/tls-runtime/src/metrics.rs | 88 +++ crates/tls-runtime/src/outbound.rs | 67 ++ .../src/server.rs} | 171 ++--- crates/tls-runtime/src/source.rs | 94 +++ crates/tls-runtime/src/state.rs | 234 +++++++ crates/utils/Cargo.toml | 13 +- crates/utils/src/lib.rs | 11 - crates/utils/src/notify/mod.rs | 205 ------ rustfs/Cargo.toml | 6 +- rustfs/src/admin/handlers/mod.rs | 2 + rustfs/src/admin/handlers/site_replication.rs | 83 +-- rustfs/src/admin/handlers/tls_debug.rs | 154 ++++ rustfs/src/admin/mod.rs | 3 +- rustfs/src/admin/route_registration_test.rs | 4 +- rustfs/src/app/object_usecase.rs | 9 +- rustfs/src/main.rs | 15 +- rustfs/src/server/http.rs | 38 +- rustfs/src/server/tls_material.rs | 594 ++++++---------- rustfs/src/storage/helper.rs | 4 +- 67 files changed, 4978 insertions(+), 1725 deletions(-) rename crates/{utils/src/notify => targets/src}/net.rs (65%) create mode 100644 crates/targets/src/runtime/tls/adapter.rs create mode 100644 crates/targets/src/runtime/tls/config.rs create mode 100644 crates/targets/src/runtime/tls/coordinator.rs create mode 100644 crates/targets/src/runtime/tls/fingerprint.rs create mode 100644 crates/targets/src/runtime/tls/metrics.rs create mode 100644 crates/targets/src/runtime/tls/mod.rs create mode 100644 crates/targets/src/runtime/tls/state.rs create mode 100644 crates/targets/src/runtime/tls/trait.rs create mode 100644 crates/targets/src/runtime/tls/validate.rs create mode 100644 crates/tls-runtime/Cargo.toml rename crates/{utils => tls-runtime}/src/certs.rs (51%) create mode 100644 crates/tls-runtime/src/config.rs create mode 100644 crates/tls-runtime/src/coordinator.rs create mode 100644 crates/tls-runtime/src/debug.rs create mode 100644 crates/tls-runtime/src/error.rs create mode 100644 crates/tls-runtime/src/fingerprint.rs create mode 100644 crates/tls-runtime/src/lib.rs create mode 100644 crates/tls-runtime/src/material.rs create mode 100644 crates/tls-runtime/src/metrics.rs create mode 100644 crates/tls-runtime/src/outbound.rs rename crates/{protocols/src/tls_hot_reload.rs => tls-runtime/src/server.rs} (59%) create mode 100644 crates/tls-runtime/src/source.rs create mode 100644 crates/tls-runtime/src/state.rs delete mode 100644 crates/utils/src/notify/mod.rs create mode 100644 rustfs/src/admin/handlers/tls_debug.rs diff --git a/.github/workflows/audit.yml b/.github/workflows/audit.yml index ade2e6930..dd679ea35 100644 --- a/.github/workflows/audit.yml +++ b/.github/workflows/audit.yml @@ -75,7 +75,7 @@ jobs: uses: actions/checkout@v6 - name: Dependency Review - uses: actions/dependency-review-action@v4 + uses: actions/dependency-review-action@v5 with: fail-on-severity: moderate - comment-summary-in-pr: true + comment-summary-in-pr: always diff --git a/Cargo.lock b/Cargo.lock index 9ace27983..da370aac3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -761,9 +761,9 @@ dependencies = [ [[package]] name = "async-rs" -version = "0.8.7" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53bf71bee8a75907b6e3c81c5476efa7fcbb34df6e12d30b706888abded72091" +checksum = "d096d3e7d6c2cd3ff2bebba9273fef6e0c129ee04ec6c16a8023c19958b2cc90" dependencies = [ "async-compat", "async-global-executor", @@ -840,9 +840,9 @@ dependencies = [ [[package]] name = "autocfg" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" [[package]] name = "aws-config" @@ -1645,9 +1645,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.20.2" +version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" [[package]] name = "bytemuck" @@ -3517,9 +3517,9 @@ dependencies = [ [[package]] name = "dial9-tokio-telemetry" -version = "0.3.10" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45521f70d67ff82b19ffddb55c7f039ee24c44a172aeb03ec2142f1808ae8562" +checksum = "7a8b3e96f10aa94341f08a672cf2bf4df3a0128a396e5d2ea2d41724205fd2c3" dependencies = [ "arc-swap", "bon", @@ -3613,7 +3613,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -3949,7 +3949,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -5426,7 +5426,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -5503,7 +5503,7 @@ dependencies = [ "portable-atomic", "portable-atomic-util", "serde_core", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -5593,9 +5593,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.98" +version = "0.3.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67df7112613f8bfd9150013a0314e196f4800d3201ae742489d999db2f979f08" +checksum = "142bc4740e452c1e57ade0cbc129f139c9093e354346f0872ef985f4f5cf5f11" dependencies = [ "cfg-if", "futures-util", @@ -5867,9 +5867,9 @@ checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" [[package]] name = "libmimalloc-sys" -version = "0.1.48" +version = "0.1.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2892ae4ea6fa2cb7acb0e236a6880d39523239cd9089de71d220910ccc806790" +checksum = "6a45a52f43e1c16f667ccfe4dd8c85b7f7c204fd5e3bf46c5b0db9a5c3c0b8e9" dependencies = [ "cc", "cty", @@ -6287,9 +6287,9 @@ dependencies = [ [[package]] name = "mimalloc" -version = "0.1.51" +version = "0.1.52" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebca48a43116bc25f18a61360f1be98412f50cc218f5e52c823086b999a4a21a" +checksum = "2d4139bb28d14ad1facf21d5eb8825051b326e172d216b39f6d31df53cc97862" dependencies = [ "libmimalloc-sys", ] @@ -6639,7 +6639,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -8415,7 +8415,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.60.2", + "windows-sys 0.59.0", ] [[package]] @@ -9101,9 +9101,9 @@ dependencies = [ [[package]] name = "russh-sftp" -version = "2.2.2" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c826d40e310bbcb377fd8f6c5873c12eaf54da30569d8e8da56914fdac9773cd" +checksum = "9ed8949eca4163c18a8f59ff96d32cf61e9c13b9735e21ef32b3907f4aafa1a9" dependencies = [ "bitflags 2.11.1", "bytes", @@ -9288,10 +9288,12 @@ dependencies = [ "rustfs-scanner", "rustfs-signer", "rustfs-targets", + "rustfs-tls-runtime", "rustfs-trusted-proxies", "rustfs-utils", "rustfs-zip", "rustls", + "rustls-pki-types", "s3s", "serde", "serde_json", @@ -9510,9 +9512,11 @@ dependencies = [ "rustfs-rio", "rustfs-s3-types", "rustfs-signer", + "rustfs-tls-runtime", "rustfs-utils", "rustix 1.1.4", "rustls", + "rustls-pki-types", "s3s", "serde", "serde_json", @@ -9937,7 +9941,6 @@ dependencies = [ "percent-encoding", "proptest", "quick-xml 0.40.1", - "rcgen", "regex", "russh", "russh-sftp", @@ -9948,6 +9951,7 @@ dependencies = [ "rustfs-keystone", "rustfs-policy", "rustfs-rio", + "rustfs-tls-runtime", "rustfs-utils", "rustls", "s3s", @@ -9980,7 +9984,9 @@ dependencies = [ "rustfs-common", "rustfs-config", "rustfs-io-metrics", + "rustfs-tls-runtime", "rustfs-utils", + "tokio", "tonic", "tonic-prost", "tonic-prost-build", @@ -10007,7 +10013,9 @@ dependencies = [ "reqwest", "rustfs-config", "rustfs-io-metrics", + "rustfs-tls-runtime", "rustfs-utils", + "rustls-pki-types", "s3s", "serde", "serde_json", @@ -10130,28 +10138,33 @@ dependencies = [ name = "rustfs-targets" version = "1.0.0-beta.4" dependencies = [ + "arc-swap", "async-nats", "async-trait", "chrono", "criterion", "deadpool-postgres", "hashbrown 0.17.1", + "hyper", "hyper-rustls", "lapin", + "libc", + "metrics", "mysql_async", "parking_lot 0.12.5", "pulsar", "redis", + "regex", "reqwest", "rumqttc-next", "rustfs-config", "rustfs-ecstore", "rustfs-kafka-async", "rustfs-s3-types", - "rustfs-utils", + "rustfs-tls-runtime", "rustls", "rustls-native-certs", - "rustls-pki-types", + "s3s", "serde", "serde_json", "snap", @@ -10167,6 +10180,26 @@ dependencies = [ "uuid", ] +[[package]] +name = "rustfs-tls-runtime" +version = "1.0.0-beta.4" +dependencies = [ + "arc-swap", + "metrics", + "rcgen", + "rustfs-common", + "rustfs-config", + "rustls", + "rustls-pki-types", + "serde", + "serde_json", + "sha2 0.11.0", + "tempfile", + "thiserror 2.0.18", + "tokio", + "tracing", +] + [[package]] name = "rustfs-trusted-proxies" version = "1.0.0-beta.4" @@ -10203,23 +10236,17 @@ dependencies = [ "crc-fast", "flate2", "futures", - "hashbrown 0.17.1", "hex-simd", "highway", "hmac 0.13.0", "http 1.4.0", "hyper", - "libc", "local-ip-address", "lz4", "md-5 0.11.0", "netif", - "rcgen", "regex", "rustix 1.1.4", - "rustls", - "rustls-pki-types", - "s3s", "serde", "sha1 0.11.0", "sha2 0.11.0", @@ -10227,7 +10254,6 @@ dependencies = [ "snap", "temp-env", "tempfile", - "thiserror 2.0.18", "tokio", "tracing", "transform-stream", @@ -10312,7 +10338,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.12.1", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -10333,9 +10359,9 @@ dependencies = [ [[package]] name = "rustls-connector" -version = "0.23.2" +version = "0.23.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "546c32c0a03187814b1e1239eec017778dc9b87d8241a2c5c1954c47ab8ac8fd" +checksum = "1869b2b1c5adf03c1025b23d8c23c36903a6bdd40a92332bfd8f309db5a460a1" dependencies = [ "futures-io", "futures-rustls", @@ -10386,7 +10412,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -11493,9 +11519,9 @@ checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" [[package]] name = "tcp-stream" -version = "0.34.10" +version = "0.34.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8da40490cac3733b85c67b831f64e2132fdaa0929f42a6d4cf458d339777473" +checksum = "04a6c48788336a4223ddc6e4d4f2d623f4e5865058c0999d1502a8994816ec4f" dependencies = [ "async-rs", "cfg-if", @@ -11524,7 +11550,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix 1.1.4", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -12503,9 +12529,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.121" +version = "0.2.122" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49ace1d07c165b0864824eee619580c4689389afa9dc9ed3a4c75040d82e6790" +checksum = "3ed04576f974d2b2fba0f38c51dbc5518011e38c36bf1143164be765528fd409" dependencies = [ "cfg-if", "once_cell", @@ -12516,9 +12542,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.71" +version = "0.4.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96492d0d3ffba25305a7dc88720d250b1401d7edca02cc3bcd50633b424673b8" +checksum = "9473dbd2991ae90b6291c3c32c30c6187ac49aa32f9905d1cce280ec1e110b0f" dependencies = [ "js-sys", "wasm-bindgen", @@ -12526,9 +12552,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.121" +version = "0.2.122" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e68e6f4afd367a562002c05637acb8578ff2dea1943df76afb9e83d177c8578" +checksum = "916151b09da36bd82f6615cbf3a419e2f0ba23a03c6160e8e92eb6bd4aa1dec6" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -12536,9 +12562,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.121" +version = "0.2.122" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d95a9ec35c64b2a7cb35d3fead40c4238d0940c86d107136999567a4703259f2" +checksum = "299047362ccbfce148b67ab7e73349f77748e00c8296f9542adfad2ad82c5c5e" dependencies = [ "bumpalo", "proc-macro2", @@ -12549,9 +12575,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.121" +version = "0.2.122" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4e0100b01e9f0d03189a92b96772a1fb998639d981193d7dbab487302513441" +checksum = "9a929b2c61f11ba3e9bc35b50c1f25cb38e0e892c0c231ae2b8cf78d5dad4437" dependencies = [ "unicode-ident", ] @@ -12605,9 +12631,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.98" +version = "0.3.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b572dff8bcf38bad0fa19729c89bb5748b2b9b1d8be70cf90df697e3a8f32aa" +checksum = "6d621441cfc37b84979402712047321980c178f299193a3589d05b99e8763436" dependencies = [ "js-sys", "wasm-bindgen", @@ -12712,7 +12738,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -12839,7 +12865,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets 0.52.6", + "windows-targets", ] [[package]] @@ -12848,16 +12874,7 @@ version = "0.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" dependencies = [ - "windows-targets 0.52.6", -] - -[[package]] -name = "windows-sys" -version = "0.60.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" -dependencies = [ - "windows-targets 0.53.5", + "windows-targets", ] [[package]] @@ -12875,31 +12892,14 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm 0.52.6", - "windows_aarch64_msvc 0.52.6", - "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm 0.52.6", - "windows_i686_msvc 0.52.6", - "windows_x86_64_gnu 0.52.6", - "windows_x86_64_gnullvm 0.52.6", - "windows_x86_64_msvc 0.52.6", -] - -[[package]] -name = "windows-targets" -version = "0.53.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" -dependencies = [ - "windows-link", - "windows_aarch64_gnullvm 0.53.1", - "windows_aarch64_msvc 0.53.1", - "windows_i686_gnu 0.53.1", - "windows_i686_gnullvm 0.53.1", - "windows_i686_msvc 0.53.1", - "windows_x86_64_gnu 0.53.1", - "windows_x86_64_gnullvm 0.53.1", - "windows_x86_64_msvc 0.53.1", + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", ] [[package]] @@ -12917,96 +12917,48 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" - [[package]] name = "windows_aarch64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" -[[package]] -name = "windows_aarch64_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" - [[package]] name = "windows_i686_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" -[[package]] -name = "windows_i686_gnu" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" - [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" -[[package]] -name = "windows_i686_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" - [[package]] name = "windows_i686_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" -[[package]] -name = "windows_i686_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" - [[package]] name = "windows_x86_64_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" -[[package]] -name = "windows_x86_64_gnu" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" - [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" - [[package]] name = "windows_x86_64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" -[[package]] -name = "windows_x86_64_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" - [[package]] name = "winnow" version = "1.0.3" diff --git a/Cargo.toml b/Cargo.toml index ec9c62e3f..7c236bd39 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -47,6 +47,7 @@ members = [ "crates/signer", # client signer "crates/targets", # Target-specific configurations and utilities "crates/trusted-proxies", # Trusted proxies management + "crates/tls-runtime", # Project-wide TLS runtime foundation "crates/utils", # Utility functions and helpers "crates/io-metrics", # Zero-copy metrics collection for performance analysis "crates/io-core", # Zero-copy core reader and writer implementations @@ -110,6 +111,7 @@ rustfs-scanner = { path = "crates/scanner", version = "1.0.0-beta.4" } rustfs-signer = { path = "crates/signer", version = "1.0.0-beta.4" } rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.0-beta.4" } rustfs-targets = { path = "crates/targets", version = "1.0.0-beta.4" } +rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.0-beta.4" } rustfs-utils = { path = "crates/utils", version = "1.0.0-beta.4" } rustfs-zip = { path = "./crates/zip", version = "1.0.0-beta.4" } @@ -141,7 +143,7 @@ tokio-rustls = { version = "0.26.4", default-features = false, features = ["logg tokio-stream = { version = "0.1.18" } tokio-test = "0.4.5" tokio-util = { version = "0.7.18", features = ["io", "compat"] } -tonic = { version = "0.14.6", features = ["gzip"] } +tonic = { version = "0.14.6", features = ["gzip", "deflate"] } tonic-prost = { version = "0.14.6" } tonic-prost-build = { version = "0.14.6" } tower = { version = "0.5.3", features = ["timeout"] } @@ -307,7 +309,7 @@ unftp-core = "0.1.0" suppaftp = { version = "8.0.3", features = ["tokio", "tokio-rustls-aws-lc-rs"] } rcgen = "0.14.8" russh = { version = "0.60.3", git = "https://github.com/Eugeny/russh", rev = "fc6e3ab4cd4338e94ae64e17aeed2acee9335e6b" } -russh-sftp = "2.2.2" +russh-sftp = "2.3.0" # WebDAV dav-server = "0.11.0" diff --git a/crates/common/src/globals.rs b/crates/common/src/globals.rs index e561ed2c1..ed1f13537 100644 --- a/crates/common/src/globals.rs +++ b/crates/common/src/globals.rs @@ -17,6 +17,7 @@ use chrono::{DateTime, Utc}; use std::collections::HashMap; use std::sync::LazyLock; +use std::sync::atomic::{AtomicU64, Ordering}; use tokio::sync::RwLock; use tonic::transport::Channel; @@ -27,6 +28,7 @@ pub static GLOBAL_RUSTFS_ADDR: LazyLock> = LazyLock::new(|| RwLoc pub static GLOBAL_CONN_MAP: LazyLock>> = LazyLock::new(|| RwLock::new(HashMap::new())); pub static GLOBAL_ROOT_CERT: LazyLock>>> = LazyLock::new(|| RwLock::new(None)); pub static GLOBAL_MTLS_IDENTITY: LazyLock>> = LazyLock::new(|| RwLock::new(None)); +pub static GLOBAL_OUTBOUND_TLS_GENERATION: LazyLock = LazyLock::new(|| AtomicU64::new(0)); /// Global initialization time of the RustFS node. pub static GLOBAL_INIT_TIME: LazyLock>>> = LazyLock::new(|| RwLock::new(None)); @@ -88,6 +90,16 @@ pub async fn set_global_mtls_identity(identity: Option) { *GLOBAL_MTLS_IDENTITY.write().await = identity; } +/// Set the global outbound TLS generation. +pub fn set_global_outbound_tls_generation(generation: u64) { + GLOBAL_OUTBOUND_TLS_GENERATION.store(generation, Ordering::Relaxed); +} + +/// Get the global outbound TLS generation. +pub fn get_global_outbound_tls_generation() -> u64 { + GLOBAL_OUTBOUND_TLS_GENERATION.load(Ordering::Relaxed) +} + /// Evict a stale/dead connection from the global connection cache. /// This is critical for cluster recovery when a node dies unexpectedly (e.g., power-off). /// By removing the cached connection, subsequent requests will establish a fresh connection. diff --git a/crates/ecstore/Cargo.toml b/crates/ecstore/Cargo.toml index e3dfe1fd6..bf298ab3c 100644 --- a/crates/ecstore/Cargo.toml +++ b/crates/ecstore/Cargo.toml @@ -38,6 +38,7 @@ rustfs-filemeta.workspace = true rustfs-utils = { workspace = true, features = ["full"] } rustfs-rio.workspace = true rustfs-signer.workspace = true +rustfs-tls-runtime.workspace = true rustfs-checksums.workspace = true rustfs-config = { workspace = true, features = ["constants", "notify", "audit"] } rustfs-credentials = { workspace = true } @@ -91,6 +92,7 @@ hyper.workspace = true hyper-util.workspace = true hyper-rustls.workspace = true rustls.workspace = true +rustls-pki-types.workspace = true tokio = { workspace = true, features = ["io-util", "sync", "signal","io-uring"] } tonic.workspace = true xxhash-rust = { workspace = true, features = ["xxh64", "xxh3"] } diff --git a/crates/ecstore/src/client/transition_api.rs b/crates/ecstore/src/client/transition_api.rs index e2a2ac386..95687e040 100644 --- a/crates/ecstore/src/client/transition_api.rs +++ b/crates/ecstore/src/client/transition_api.rs @@ -51,6 +51,7 @@ use md5::Md5; use rand::{Rng, RngExt}; use rustfs_config::MAX_S3_CLIENT_RESPONSE_SIZE; use rustfs_rio::HashReader; +use rustfs_tls_runtime::{load_global_outbound_tls_state, record_tls_generation}; use rustfs_utils::HashAlgorithm; use rustfs_utils::{ net::get_endpoint_url, @@ -58,6 +59,8 @@ use rustfs_utils::{ DEFAULT_RETRY_CAP, DEFAULT_RETRY_UNIT, MAX_JITTER, MAX_RETRY, RetryTimer, is_http_status_retryable, is_s3code_retryable, }, }; +use rustls_pki_types::PrivateKeyDer; +use rustls_pki_types::pem::PemObject; use s3s::S3ErrorCode; use s3s::dto::Owner; use s3s::dto::ReplicationStatus; @@ -152,24 +155,11 @@ pub enum BucketLookupType { BucketLookupPath, } -fn load_root_store_from_tls_path() -> Option { - // Load the root certificate bundle from the path specified by the - // RUSTFS_TLS_PATH environment variable. - let tp = rustfs_utils::get_env_str(rustfs_config::ENV_RUSTFS_TLS_PATH, rustfs_config::DEFAULT_RUSTFS_TLS_PATH); - // If no TLS path is configured, do not fall back to a CA bundle in the current directory. - if tp.is_empty() { - return None; - } - let ca = std::path::Path::new(&tp).join(rustfs_config::RUSTFS_CA_CERT); - if !ca.exists() { - return None; - } - - let der_list = rustfs_utils::load_cert_bundle_der_bytes(ca.to_str().unwrap_or_default()).ok()?; +fn build_root_store_from_der_list(der_list: Vec>) -> Option { let mut store = rustls::RootCertStore::empty(); for der in der_list { if let Err(e) = store.add(der.into()) { - warn!("Warning: failed to add certificate from '{}' to root store: {e}", ca.display()); + warn!("Warning: failed to add certificate to root store: {e}"); } } Some(store) @@ -199,18 +189,38 @@ where }) } -fn build_tls_config() -> Result { - with_rustls_init_guard(|| { - let config = if let Some(store) = load_root_store_from_tls_path() { - rustls::ClientConfig::builder() - .with_root_certificates(store) - .with_no_client_auth() - } else { - rustls::ClientConfig::builder().with_native_roots()?.with_no_client_auth() - }; +async fn build_tls_config() -> Result { + with_rustls_init_guard(|| Ok(()))?; - Ok(config) - }) + let outbound_tls = load_global_outbound_tls_state().await; + record_tls_generation("ecstore_transition_client", outbound_tls.generation.0); + let builder = if let Some(root_ca_pem) = outbound_tls.root_ca_pem.as_ref() { + let mut reader = std::io::BufReader::new(root_ca_pem.as_slice()); + let certs_der = rustls_pki_types::CertificateDer::pem_reader_iter(&mut reader) + .collect::, _>>() + .map_err(|e| std::io::Error::other(format!("failed to parse published root CA PEM: {e}")))?; + + let root_store = build_root_store_from_der_list(certs_der.into_iter().map(|cert| cert.to_vec()).collect::>()) + .ok_or_else(|| std::io::Error::other("published outbound root CA material could not build root store"))?; + rustls::ClientConfig::builder().with_root_certificates(root_store) + } else { + rustls::ClientConfig::builder().with_native_roots()? + }; + + let config = if let Some(identity) = outbound_tls.mtls_identity.as_ref() { + let certs = rustls_pki_types::CertificateDer::pem_reader_iter(&mut std::io::BufReader::new(identity.cert_pem.as_slice())) + .collect::, _>>() + .map_err(|e| std::io::Error::other(format!("failed to parse published client cert PEM: {e}")))?; + let key = PrivateKeyDer::from_pem_reader(&mut std::io::BufReader::new(identity.key_pem.as_slice())) + .map_err(|e| std::io::Error::other(format!("failed to parse published client key PEM: {e}")))?; + builder + .with_client_auth_cert(certs, key) + .map_err(|e| std::io::Error::other(format!("failed to build client mTLS identity: {e}")))? + } else { + builder.with_no_client_auth() + }; + + Ok(config) } impl TransitionClient { @@ -234,7 +244,7 @@ impl TransitionClient { let endpoint_url = get_endpoint_url(endpoint, opts.secure)?; - let tls = build_tls_config()?; + let tls = build_tls_config().await?; let https = hyper_rustls::HttpsConnectorBuilder::new() .with_tls_config(tls) @@ -1374,9 +1384,7 @@ pub struct CreateBucketConfiguration { #[cfg(test)] mod tests { - use super::{ - build_tls_config, load_root_store_from_tls_path, signer_error_to_io_error, validate_header_values, with_rustls_init_guard, - }; + use super::{build_tls_config, signer_error_to_io_error, validate_header_values, with_rustls_init_guard}; use http::{HeaderMap, HeaderValue}; #[test] @@ -1395,22 +1403,6 @@ mod tests { assert!(outcome.is_ok(), "TLS config creation should not panic"); } - /// When RUSTFS_TLS_PATH is not set, `load_root_store_from_tls_path` must return `None` - /// (i.e. it must not silently look for a CA bundle in the current working directory). - #[test] - fn tls_path_unset_returns_none() { - let result = temp_env::with_var_unset(rustfs_config::ENV_RUSTFS_TLS_PATH, || load_root_store_from_tls_path()); - assert!(result.is_none(), "expected None when RUSTFS_TLS_PATH is unset, but got a root store"); - } - - /// When RUSTFS_TLS_PATH is set to an empty string, `load_root_store_from_tls_path` must - /// return `None` to avoid accidentally trusting a CA bundle in the current directory. - #[test] - fn tls_path_empty_returns_none() { - let result = temp_env::with_var(rustfs_config::ENV_RUSTFS_TLS_PATH, Some(""), || load_root_store_from_tls_path()); - assert!(result.is_none(), "expected None when RUSTFS_TLS_PATH is empty, but got a root store"); - } - /// Installing the rustls crypto provider when one is already set must not panic or return /// an error that surfaces to callers (the race-safe `get_default` check guards the install). #[test] diff --git a/crates/protocols/Cargo.toml b/crates/protocols/Cargo.toml index 61f52c554..bf677e056 100644 --- a/crates/protocols/Cargo.toml +++ b/crates/protocols/Cargo.toml @@ -27,7 +27,7 @@ categories = ["network-programming", "filesystem"] [features] default = [] -ftps = ["dep:libunftp", "dep:unftp-core", "dep:rustls"] +ftps = ["dep:libunftp", "dep:unftp-core", "dep:rustls", "dep:rustfs-tls-runtime"] swift = [ "dep:rustfs-keystone", "dep:rustfs-ecstore", @@ -53,7 +53,7 @@ swift = [ "dep:base64", "dep:async-compression", ] -webdav = ["dep:dav-server", "dep:hyper", "dep:hyper-util", "dep:http-body-util", "dep:tokio-rustls", "dep:base64", "dep:rustls", "dep:percent-encoding"] +webdav = ["dep:dav-server", "dep:hyper", "dep:hyper-util", "dep:http-body-util", "dep:tokio-rustls", "dep:base64", "dep:rustls", "dep:percent-encoding", "dep:rustfs-tls-runtime"] sftp = ["dep:russh", "dep:russh-sftp", "dep:uuid", "dep:subtle", "dep:tokio-util", "dep:socket2"] [dependencies] @@ -63,6 +63,7 @@ rustfs-credentials = { workspace = true } rustfs-policy = { workspace = true } rustfs-utils = { workspace = true } rustfs-config = { workspace = true } +rustfs-tls-runtime = { workspace = true, optional = true } # Async dependencies tokio = { workspace = true, features = ["fs", "io-util", "sync", "time","io-uring"] } @@ -128,7 +129,6 @@ socket2 = { workspace = true, optional = true } [dev-dependencies] tempfile = { workspace = true } proptest = "1" -rcgen = { workspace = true } tracing-subscriber = { workspace = true } [package.metadata.docs.rs] diff --git a/crates/protocols/src/ftps/server.rs b/crates/protocols/src/ftps/server.rs index 98f273d1f..09d9a7c17 100644 --- a/crates/protocols/src/ftps/server.rs +++ b/crates/protocols/src/ftps/server.rs @@ -17,12 +17,14 @@ use super::driver::FtpsDriver; use crate::common::client::s3::StorageBackend; use crate::common::session::{Protocol, ProtocolPrincipal, SessionContext}; use crate::constants::{network::DEFAULT_SOURCE_IP, paths::ROOT_PATH}; -use crate::tls_hot_reload::{ReloadableCertResolver, spawn_cert_reload_loop}; use libunftp::options::FtpsRequired; +use rustfs_config::{DEFAULT_TLS_RELOAD_ENABLE, DEFAULT_TLS_RELOAD_INTERVAL, ENV_TLS_RELOAD_ENABLE, ENV_TLS_RELOAD_INTERVAL}; +use rustfs_tls_runtime::{ReloadableServerCertResolver, TlsReloadOptions, spawn_server_cert_reload_loop}; use std::fmt::{Debug, Display, Formatter}; use std::net::IpAddr; use std::path::Path; use std::sync::Arc; +use std::time::Duration; use tokio::sync::broadcast; use tokio::sync::watch; use tracing::{debug, error, info, warn}; @@ -68,6 +70,14 @@ impl FtpsServer where S: StorageBackend + Clone + Send + Sync + 'static + Debug, { + fn tls_reload_options() -> TlsReloadOptions { + TlsReloadOptions { + enabled: rustfs_utils::get_env_bool(ENV_TLS_RELOAD_ENABLE, DEFAULT_TLS_RELOAD_ENABLE), + interval: Duration::from_secs(rustfs_utils::get_env_u64(ENV_TLS_RELOAD_INTERVAL, DEFAULT_TLS_RELOAD_INTERVAL).max(5)), + ..TlsReloadOptions::default() + } + } + /// Create a new FTPS server pub async fn new(config: FtpsConfig, storage: S) -> Result { config.validate().await?; @@ -114,9 +124,14 @@ where if let Some(cert_dir) = &self.config.cert_dir { debug!("Enabling FTPS with multi-certificate support from directory: {}", cert_dir); - let resolver = ReloadableCertResolver::load_from_directory(cert_dir) + let resolver = ReloadableServerCertResolver::load_from_directory(cert_dir) .map_err(|e| FtpsInitError::InvalidConfig(format!("Failed to create certificate resolver: {}", e)))?; - let _reload_task = spawn_cert_reload_loop("ftps", cert_dir.clone(), resolver.clone(), reload_shutdown_rx.clone()); + let _reload_task = spawn_server_cert_reload_loop( + "ftps", + resolver.clone(), + Self::tls_reload_options(), + reload_shutdown_rx.clone(), + ); // Build ServerConfig with SNI support let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); diff --git a/crates/protocols/src/lib.rs b/crates/protocols/src/lib.rs index e6ac15bc6..2133b614a 100644 --- a/crates/protocols/src/lib.rs +++ b/crates/protocols/src/lib.rs @@ -17,9 +17,6 @@ pub mod common; pub mod constants; -#[cfg(any(feature = "ftps", feature = "webdav"))] -mod tls_hot_reload; - #[cfg(feature = "ftps")] pub mod ftps; diff --git a/crates/protocols/src/sftp/errors.rs b/crates/protocols/src/sftp/errors.rs index 35f46c85f..9d57ed4f0 100644 --- a/crates/protocols/src/sftp/errors.rs +++ b/crates/protocols/src/sftp/errors.rs @@ -18,6 +18,7 @@ use super::constants::{http_error_codes, s3_error_codes}; use russh_sftp::protocol::{Status, StatusCode}; +use russh_sftp::server::StatusReply; use s3s::{S3Error, S3ErrorCode}; use std::{any::Any, fmt::Display}; @@ -31,6 +32,12 @@ impl From for StatusCode { } } +impl From for StatusReply { + fn from(err: SftpError) -> Self { + StatusReply::new(err.0) + } +} + impl SftpError { pub(super) fn code(code: StatusCode) -> Self { Self(code) diff --git a/crates/protocols/src/webdav/server.rs b/crates/protocols/src/webdav/server.rs index b788f9bba..cf0cb4d78 100644 --- a/crates/protocols/src/webdav/server.rs +++ b/crates/protocols/src/webdav/server.rs @@ -16,7 +16,6 @@ use super::config::{WebDavConfig, WebDavInitError}; use super::driver::WebDavDriver; use crate::common::client::s3::StorageBackend; use crate::common::session::{Protocol, ProtocolPrincipal, SessionContext}; -use crate::tls_hot_reload::{ReloadableCertResolver, spawn_cert_reload_loop}; use bytes::Bytes; use dav_server::DavHandler; use dav_server::fakels::FakeLs; @@ -25,10 +24,13 @@ use hyper::server::conn::http1; use hyper::service::service_fn; use hyper::{Request, Response, StatusCode}; use hyper_util::rt::TokioIo; +use rustfs_config::{DEFAULT_TLS_RELOAD_ENABLE, DEFAULT_TLS_RELOAD_INTERVAL, ENV_TLS_RELOAD_ENABLE, ENV_TLS_RELOAD_INTERVAL}; +use rustfs_tls_runtime::{ReloadableServerCertResolver, TlsReloadOptions, spawn_server_cert_reload_loop}; use rustls::ServerConfig; use std::convert::Infallible; use std::net::IpAddr; use std::sync::Arc; +use std::time::Duration; use tokio::net::TcpListener; use tokio::sync::{broadcast, watch}; use tokio_rustls::TlsAcceptor; @@ -49,6 +51,14 @@ impl WebDavServer where S: StorageBackend + Clone + Send + Sync + 'static + std::fmt::Debug, { + fn tls_reload_options() -> TlsReloadOptions { + TlsReloadOptions { + enabled: rustfs_utils::get_env_bool(ENV_TLS_RELOAD_ENABLE, DEFAULT_TLS_RELOAD_ENABLE), + interval: Duration::from_secs(rustfs_utils::get_env_u64(ENV_TLS_RELOAD_INTERVAL, DEFAULT_TLS_RELOAD_INTERVAL).max(5)), + ..TlsReloadOptions::default() + } + } + /// Create a new WebDAV server pub async fn new(config: WebDavConfig, storage: S) -> Result { config.validate().await?; @@ -68,10 +78,14 @@ where if let Some(cert_dir) = &self.config.cert_dir { debug!("Enabling WebDAV TLS with certificates from: {}", cert_dir); - let resolver = ReloadableCertResolver::load_from_directory(cert_dir) + let resolver = ReloadableServerCertResolver::load_from_directory(cert_dir) .map_err(|e| WebDavInitError::Tls(format!("Failed to create certificate resolver: {}", e)))?; - let _reload_task = - spawn_cert_reload_loop("webdav", cert_dir.clone(), resolver.clone(), reload_shutdown_rx.clone()); + let _reload_task = spawn_server_cert_reload_loop( + "webdav", + resolver.clone(), + Self::tls_reload_options(), + reload_shutdown_rx.clone(), + ); let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); diff --git a/crates/protos/Cargo.toml b/crates/protos/Cargo.toml index 0b28716d4..362721417 100644 --- a/crates/protos/Cargo.toml +++ b/crates/protos/Cargo.toml @@ -36,14 +36,15 @@ path = "src/main.rs" rustfs-common.workspace = true rustfs-io-metrics.workspace = true rustfs-config.workspace = true +rustfs-tls-runtime.workspace = true rustfs-utils.workspace = true flatbuffers = { workspace = true } prost = { workspace = true } -tonic = { workspace = true, features = ["transport"] } +tonic = { workspace = true, features = ["transport", "tls-native-roots", "tls-aws-lc"] } tonic-prost = { workspace = true } tonic-prost-build = { workspace = true } +tokio = { workspace = true, features = ["sync"] } tracing = { workspace = true } [lib] -test = false doctest = false diff --git a/crates/protos/src/lib.rs b/crates/protos/src/lib.rs index 08c94660f..a6350ffdf 100644 --- a/crates/protos/src/lib.rs +++ b/crates/protos/src/lib.rs @@ -18,12 +18,16 @@ mod generated; use proto_gen::node_service::node_service_client::NodeServiceClient; -use rustfs_common::{GLOBAL_CONN_MAP, GLOBAL_MTLS_IDENTITY, GLOBAL_ROOT_CERT, evict_connection}; +use rustfs_common::{GLOBAL_CONN_MAP, evict_connection}; use rustfs_io_metrics::internode_metrics::global_internode_metrics; +use rustfs_tls_runtime::{load_global_outbound_tls_state, record_tls_consumer_stale_generation}; use std::{ + collections::HashMap, error::Error, + sync::LazyLock, time::{Duration, Instant}, }; +use tokio::sync::Mutex; use tonic::{ Request, Status, service::interceptor::InterceptedService, @@ -46,6 +50,21 @@ pub const DEFAULT_GRPC_SERVER_MESSAGE_LEN: usize = 100 * 1024 * 1024; /// It is used to identify HTTPS URLs. /// Default value: https:// const RUSTFS_HTTPS_PREFIX: &str = "https://"; +const TLS_GENERATION_CACHE_MAX_SIZE: usize = 512; +static TLS_GENERATION_CACHE: LazyLock>> = LazyLock::new(|| Mutex::new(HashMap::new())); + +fn enforce_tls_generation_cache_bound(generation_cache: &mut HashMap, generation: u64, addr: &str) { + if generation_cache.len() < TLS_GENERATION_CACHE_MAX_SIZE || generation_cache.contains_key(addr) { + return; + } + + generation_cache.retain(|_, g| *g == generation); + if generation_cache.len() >= TLS_GENERATION_CACHE_MAX_SIZE + && let Some(victim) = generation_cache.keys().next().cloned() + { + generation_cache.remove(&victim); + } +} fn internode_connect_timeout() -> Duration { Duration::from_secs(rustfs_utils::get_env_u64( @@ -114,14 +133,19 @@ pub async fn create_new_channel(addr: &str) -> Result> { // Overall timeout for any RPC - fail fast on unresponsive peers .timeout(rpc_timeout); - let root_cert = GLOBAL_ROOT_CERT.read().await; - if addr.starts_with(RUSTFS_HTTPS_PREFIX) { - if root_cert.is_none() { - debug!("No custom root certificate configured; using system roots for TLS: {}", addr); - // If no custom root cert is configured, try to use system roots. - connector = connector.tls_config(ClientTlsConfig::new())?; + let outbound_tls = load_global_outbound_tls_state().await; + let generation = outbound_tls.generation.0; + let mut stale_generation = false; + { + let generation_cache = TLS_GENERATION_CACHE.lock().await; + if let Some(cached_generation) = generation_cache.get(addr) + && *cached_generation != generation + { + stale_generation = true; } - if let Some(cert_pem) = root_cert.as_ref() { + } + if addr.starts_with(RUSTFS_HTTPS_PREFIX) { + if let Some(cert_pem) = outbound_tls.root_ca_pem.as_ref() { let ca = Certificate::from_pem(cert_pem); // Derive the hostname from the HTTPS URL for TLS hostname verification. let domain = addr @@ -134,8 +158,7 @@ pub async fn create_new_channel(addr: &str) -> Result> { .unwrap_or(""); let tls = if !domain.is_empty() { let mut cfg = ClientTlsConfig::new().ca_certificate(ca).domain_name(domain); - let mtls_identity = GLOBAL_MTLS_IDENTITY.read().await; - if let Some(id) = mtls_identity.as_ref() { + if let Some(id) = outbound_tls.mtls_identity.as_ref() { let identity = tonic::transport::Identity::from_pem(id.cert_pem.clone(), id.key_pem.clone()); cfg = cfg.identity(identity); } @@ -147,9 +170,10 @@ pub async fn create_new_channel(addr: &str) -> Result> { connector = connector.tls_config(tls)?; debug!("Configured TLS with custom root certificate for: {}", addr); } else { - return Err(std::io::Error::other( - "HTTPS requested but no trusted roots are configured. Provide tls/ca.crt (or enable system roots via RUSTFS_TRUST_SYSTEM_CA=true)." - ).into()); + // No custom root CA published — fall back to system roots. + // This is the expected path when no TLS path is configured. + debug!("No custom root certificate configured; using system roots for TLS: {}", addr); + connector = connector.tls_config(ClientTlsConfig::new())?; } } @@ -168,6 +192,14 @@ pub async fn create_new_channel(addr: &str) -> Result> { { GLOBAL_CONN_MAP.write().await.insert(addr.to_string(), channel.clone()); } + { + let mut generation_cache = TLS_GENERATION_CACHE.lock().await; + enforce_tls_generation_cache_bound(&mut generation_cache, generation, addr); + generation_cache.insert(addr.to_string(), generation); + } + if stale_generation { + record_tls_consumer_stale_generation("protos_grpc_channel"); + } debug!("Successfully created and cached gRPC channel to: {}", addr); Ok(channel) @@ -178,4 +210,21 @@ pub async fn create_new_channel(addr: &str) -> Result> { pub async fn evict_failed_connection(addr: &str) { warn!("Evicting failed gRPC connection: {}", addr); evict_connection(addr).await; + TLS_GENERATION_CACHE.lock().await.remove(addr); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn enforce_tls_generation_cache_bound_evicts_when_retained_entries_still_full() { + let mut cache = HashMap::new(); + for i in 0..TLS_GENERATION_CACHE_MAX_SIZE { + cache.insert(format!("node-{i}"), 42); + } + + enforce_tls_generation_cache_bound(&mut cache, 42, "new-node"); + assert_eq!(cache.len(), TLS_GENERATION_CACHE_MAX_SIZE - 1); + } } diff --git a/crates/rio/Cargo.toml b/crates/rio/Cargo.toml index 7b749ce18..19a0057de 100644 --- a/crates/rio/Cargo.toml +++ b/crates/rio/Cargo.toml @@ -38,12 +38,14 @@ pin-project-lite.workspace = true serde = { workspace = true } bytes.workspace = true reqwest.workspace = true +rustls-pki-types.workspace = true tokio-util.workspace = true faster-hex.workspace = true futures.workspace = true rustfs-config = { workspace = true, features = ["constants"] } rustfs-io-metrics.workspace = true -rustfs-utils = { workspace = true, features = ["io", "hash", "compress", "tls"] } +rustfs-tls-runtime.workspace = true +rustfs-utils = { workspace = true, features = ["io", "hash", "compress"] } serde_json.workspace = true md-5 = { workspace = true } tracing.workspace = true diff --git a/crates/rio/src/http_reader.rs b/crates/rio/src/http_reader.rs index 2a62477a5..a3930330a 100644 --- a/crates/rio/src/http_reader.rs +++ b/crates/rio/src/http_reader.rs @@ -22,7 +22,12 @@ use rustfs_io_metrics::internode_metrics::{ INTERNODE_OPERATION_PUT_FILE_STREAM, INTERNODE_OPERATION_READ_FILE_STREAM, INTERNODE_OPERATION_WALK_DIR, INTERNODE_TRANSPORT_BACKEND_TCP_HTTP, global_internode_metrics, }; +use rustfs_tls_runtime::{ + load_cert_bundle_der_bytes, load_global_outbound_tls_generation, load_global_outbound_tls_state, + record_tls_consumer_stale_generation, +}; use rustfs_utils::get_env_opt_str; +use rustls_pki_types::pem::PemObject; use std::io::IoSlice; use std::io::{self, Error}; use std::net::IpAddr; @@ -32,76 +37,36 @@ use std::sync::LazyLock; use std::task::{Context, Poll}; use std::time::Duration; use tokio::io::{AsyncRead, AsyncWrite, ReadBuf}; -use tokio::sync::mpsc; +use tokio::sync::{Mutex, mpsc}; use tokio::time::{self, Sleep}; use tokio_util::io::StreamReader; use tokio_util::sync::PollSender; -use tracing::error; +use tracing::{error, warn}; const READ_FILE_STREAM_PATH: &str = "/rustfs/rpc/read_file_stream"; const PUT_FILE_STREAM_PATH: &str = "/rustfs/rpc/put_file_stream"; const WALK_DIR_PATH: &str = "/rustfs/rpc/walk_dir"; -/// Get the TLS path from the RUSTFS_TLS_PATH environment variable. -/// If the variable is not set, return None. -fn tls_path() -> Option<&'static std::path::PathBuf> { - static TLS_PATH: LazyLock> = - LazyLock::new(|| get_env_opt_str("RUSTFS_TLS_PATH").and_then(|s| if s.is_empty() { None } else { Some(s.into()) })); - TLS_PATH.as_ref() -} - -/// Load CA root certificates from the RUSTFS_TLS_PATH directory. -/// The CA certificates should be in PEM format and stored in the file -/// specified by the RUSTFS_CA_CERT constant. -/// If the file does not exist or cannot be read, return the builder unchanged. -fn load_ca_roots_from_tls_path(builder: reqwest::ClientBuilder) -> reqwest::ClientBuilder { - let Some(tp) = tls_path() else { - return builder; - }; - let ca_path = tp.join(rustfs_config::RUSTFS_CA_CERT); - if !ca_path.exists() { - return builder; - } - - let Ok(certs_der) = rustfs_utils::load_cert_bundle_der_bytes(ca_path.to_str().unwrap_or_default()) else { - return builder; - }; - +fn add_root_certificates_from_der(builder: reqwest::ClientBuilder, certs_der: &[Vec]) -> reqwest::ClientBuilder { let mut b = builder; for der in certs_der { - if let Ok(cert) = Certificate::from_der(&der) { + if let Ok(cert) = Certificate::from_der(der) { b = b.add_root_certificate(cert); } } b } -/// Load optional mTLS identity from the RUSTFS_TLS_PATH directory. -/// The client certificate and private key should be in PEM format and stored in the files -/// specified by RUSTFS_CLIENT_CERT_FILENAME and RUSTFS_CLIENT_KEY_FILENAME constants. -/// If the files do not exist or cannot be read, return None. -fn load_optional_mtls_identity_from_tls_path() -> Option { - let tp = tls_path()?; - let cert = std::fs::read(tp.join(rustfs_config::RUSTFS_CLIENT_CERT_FILENAME)).ok()?; - let key = std::fs::read(tp.join(rustfs_config::RUSTFS_CLIENT_KEY_FILENAME)).ok()?; - - let mut pem = Vec::with_capacity(cert.len() + key.len() + 1); - pem.extend_from_slice(&cert); - if !pem.ends_with(b"\n") { - pem.push(b'\n'); - } - pem.extend_from_slice(&key); - - match Identity::from_pem(&pem) { - Ok(id) => Some(id), - Err(e) => { - error!("Failed to load mTLS identity from PEM: {e}"); - None - } - } +#[derive(Clone)] +struct CachedClients { + generation: u64, + client: Client, + local_client: Client, } -fn build_http_client(disable_proxy: bool) -> Client { +static CLIENT_CACHE: LazyLock>> = LazyLock::new(|| Mutex::new(None)); + +async fn build_http_client(disable_proxy: bool, outbound_tls: &rustfs_tls_runtime::GlobalPublishedOutboundTlsState) -> Client { let mut builder = Client::builder() .connect_timeout(std::time::Duration::from_secs(5)) .tcp_keepalive(std::time::Duration::from_secs(10)) @@ -113,9 +78,51 @@ fn build_http_client(disable_proxy: bool) -> Client { builder = builder.no_proxy(); } - builder = load_ca_roots_from_tls_path(builder); - if let Some(id) = load_optional_mtls_identity_from_tls_path() { - builder = builder.identity(id); + if let Some(root_ca_pem) = outbound_tls.root_ca_pem.as_ref() { + let mut reader = std::io::BufReader::new(root_ca_pem.as_slice()); + match rustls_pki_types::CertificateDer::pem_reader_iter(&mut reader).collect::, _>>() { + Ok(certs_der) => { + let certs_der = certs_der.into_iter().map(|cert| cert.to_vec()).collect::>(); + builder = add_root_certificates_from_der(builder, &certs_der); + } + Err(err) => { + warn!("Failed to parse published outbound root CA PEM; falling back to default trust roots: {err}"); + } + } + } else if let Some(tp) = get_env_opt_str(rustfs_config::ENV_RUSTFS_TLS_PATH).and_then(|s| { + if s.is_empty() { + None + } else { + Some(std::path::PathBuf::from(s)) + } + }) { + let ca_path = tp.join(rustfs_config::RUSTFS_CA_CERT); + if ca_path.exists() + && let Some(ca_path_str) = ca_path.to_str() + { + match load_cert_bundle_der_bytes(ca_path_str) { + Ok(certs_der) => { + builder = add_root_certificates_from_der(builder, &certs_der); + } + Err(err) => { + warn!("Failed to parse fallback root CA bundle '{}': {}", ca_path.display(), err); + } + } + } + } + + if let Some(identity) = outbound_tls.mtls_identity.as_ref() { + let mut pem = Vec::with_capacity(identity.cert_pem.len() + identity.key_pem.len() + 1); + pem.extend_from_slice(&identity.cert_pem); + if !pem.ends_with(b"\n") { + pem.push(b'\n'); + } + pem.extend_from_slice(&identity.key_pem); + + match Identity::from_pem(&pem) { + Ok(id) => builder = builder.identity(id), + Err(e) => error!("Failed to load mTLS identity from PEM: {e}"), + } } builder.build().expect("Failed to create global HTTP client") @@ -133,17 +140,53 @@ fn should_bypass_proxy_for_url(url: &str) -> bool { host.eq_ignore_ascii_case("localhost") || host.parse::().is_ok_and(|addr| addr.is_loopback()) } -fn get_http_client(url: &str) -> Client { +async fn get_http_client(url: &str) -> Client { // Reuse HTTP connection pools while keeping loopback traffic away from // system proxies so local RPC/tests do not leak to proxy listeners. - static CLIENT: LazyLock = LazyLock::new(|| build_http_client(false)); - static LOCAL_CLIENT: LazyLock = LazyLock::new(|| build_http_client(true)); + let disable_proxy = should_bypass_proxy_for_url(url); - if should_bypass_proxy_for_url(url) { - return LOCAL_CLIENT.clone(); + // Fast path: check generation first (cheap atomic read) to avoid cloning + // the full PEM + identity bytes when the TLS state hasn't changed. + let generation = load_global_outbound_tls_generation().0; + + let guard = CLIENT_CACHE.lock().await; + if let Some(cached) = guard.as_ref() { + if cached.generation == generation { + return if disable_proxy { + cached.local_client.clone() + } else { + cached.client.clone() + }; + } + record_tls_consumer_stale_generation("rio_http_reader"); } + drop(guard); - CLIENT.clone() + // Cache miss or stale generation — load full outbound TLS state. + let outbound_tls = load_global_outbound_tls_state().await; + + let client = build_http_client(false, &outbound_tls).await; + let local_client = build_http_client(true, &outbound_tls).await; + let cached = CachedClients { + generation, + client, + local_client, + }; + + let return_client = if disable_proxy { + cached.local_client.clone() + } else { + cached.client.clone() + }; + + let mut guard = CLIENT_CACHE.lock().await; + // Guard against races: only overwrite the cache if it is empty or + // contains an older generation, so a slower task cannot regress the + // TLS state after a faster task already cached a newer generation. + if guard.as_ref().is_none_or(|c| c.generation <= generation) { + *guard = Some(cached); + } + return_client } pin_project! { @@ -197,7 +240,7 @@ impl HttpReader { ) -> io::Result { let track_internode_metrics = is_internode_rpc_url(&url); let internode_operation = internode_rpc_operation(&url); - let client = get_http_client(&url); + let client = get_http_client(&url).await; let mut request: RequestBuilder = client.request(method.clone(), url.clone()).headers(headers.clone()); if let Some(body) = body { request = request.body(body); @@ -379,7 +422,7 @@ impl HttpWriter { // "[HttpWriter::spawn] sending HTTP request: url={url_clone}, method={method_clone:?}, headers={headers_clone:?}" // ); - let client = get_http_client(&url_clone); + let client = get_http_client(&url_clone).await; let request = client .request(method_clone, url_clone.clone()) .headers(headers_clone.clone()) diff --git a/crates/targets/Cargo.toml b/crates/targets/Cargo.toml index 576f42885..aa906a741 100644 --- a/crates/targets/Cargo.toml +++ b/crates/targets/Cargo.toml @@ -14,20 +14,23 @@ documentation = "https://docs.rs/rustfs-targets/latest/rustfs_targets/" [dependencies] rustfs-config = { workspace = true, features = ["notify", "constants", "audit"] } rustfs-ecstore = { workspace = true } -rustfs-utils = { workspace = true, features = ["notify", "tls"] } +rustfs-tls-runtime = { workspace = true } rustfs-s3-types = { workspace = true } async-trait = { workspace = true } async-nats = { workspace = true } deadpool-postgres = { workspace = true } +hyper = { workspace = true } hyper-rustls = { workspace = true } lapin = { workspace = true } +libc = { workspace = true } pulsar = { workspace = true } +regex = { workspace = true } reqwest = { workspace = true } rumqttc = { workspace = true } redis = { workspace = true } rustls = { workspace = true } rustls-native-certs = { workspace = true } -rustls-pki-types = { workspace = true } +s3s = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } snap = { workspace = true } @@ -45,6 +48,8 @@ mysql_async = { workspace = true } chrono = { workspace = true } parking_lot = { workspace = true } hashbrown = { workspace = true } +arc-swap = { workspace = true } +metrics = { workspace = true } [dev-dependencies] criterion = { workspace = true } diff --git a/crates/targets/src/check.rs b/crates/targets/src/check.rs index 61a79f915..2054f5cc4 100644 --- a/crates/targets/src/check.rs +++ b/crates/targets/src/check.rs @@ -64,8 +64,8 @@ pub async fn check_mqtt_broker_available_with_tls( use crate::target::mqtt::build_mqtt_options; use rumqttc::{AsyncClient, QoS}; - let url = rustfs_utils::parse_url(broker_url) - .map_err(|e| crate::TargetError::Configuration(format!("Broker URL parsing failed: {e}")))?; + let url = + crate::parse_url(broker_url).map_err(|e| crate::TargetError::Configuration(format!("Broker URL parsing failed: {e}")))?; let url = url.url(); // build_mqtt_options returns TargetError directly; Configuration variants propagate as-is. diff --git a/crates/targets/src/lib.rs b/crates/targets/src/lib.rs index 0fb7db9cc..0959fecb6 100644 --- a/crates/targets/src/lib.rs +++ b/crates/targets/src/lib.rs @@ -20,6 +20,7 @@ pub mod control_plane; pub mod domain; pub mod error; pub mod manifest; +mod net; pub mod plugin; pub mod runtime; pub mod store; @@ -48,6 +49,7 @@ pub use manifest::{ TargetPluginExternalRuntimeContract, TargetPluginManifest, TargetPluginMarketplaceManifest, TargetPluginPackaging, TargetPluginRuntimeTransport, builtin_target_marketplace_manifest, installable_target_marketplace_manifest, }; +pub use net::*; pub use plugin::{ BuiltinTargetAdminDescriptor, BuiltinTargetDescriptor, TargetAdminMetadata, TargetPluginDescriptor, TargetPluginRegistry, TargetRequestValidator, boxed_target, diff --git a/crates/utils/src/notify/net.rs b/crates/targets/src/net.rs similarity index 65% rename from crates/utils/src/notify/net.rs rename to crates/targets/src/net.rs index 178ff4089..b9462e972 100644 --- a/crates/utils/src/notify/net.rs +++ b/crates/targets/src/net.rs @@ -1,18 +1,21 @@ -// Copyright 2024 RustFS Team +// Copyright 2024 RustFS Team // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +use hashbrown::HashMap; +use hyper::HeaderMap; use regex::Regex; +use s3s::{S3Request, S3Response}; use serde::{Deserialize, Serialize}; use std::net::IpAddr; use std::path::Path; @@ -20,7 +23,6 @@ use std::sync::LazyLock; use thiserror::Error; use url::Url; -// Lazy static for the host label regex. static HOST_LABEL_REGEX: LazyLock = LazyLock::new(|| Regex::new(r"^[a-zA-Z0-9]([a-zA-Z0-9-]*[a-zA-Z0-9])?$").unwrap()); /// NetError represents errors that can occur in network operations. @@ -41,21 +43,17 @@ pub enum NetError { } /// Host represents a network host with IP/name and port. -/// Similar to Go's net.Host structure. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct Host { pub name: String, - pub port: Option, // Using Option to represent if port is set, similar to IsPortSet. + pub port: Option, } -// Implementation of Host methods. impl Host { - // is_empty returns true if the host name is empty. pub fn is_empty(&self) -> bool { self.name.is_empty() } - // equal checks if two hosts are equal by comparing their string representations. pub fn equal(&self, other: &Host) -> bool { self.to_string() == other.to_string() } @@ -70,13 +68,122 @@ impl std::fmt::Display for Host { } } -// parse_host parses a string into a Host, with validation similar to Go's ParseHost. +/// Extract request parameters from S3Request, mainly header information. +pub fn extract_req_params(req: &S3Request) -> HashMap { + extract_params_header(&req.headers) +} + +/// Extract request parameters from hyper::HeaderMap, mainly header information. +/// This function is useful when you have a raw HTTP request and need to extract parameters. +#[deprecated(since = "0.1.0", note = "Use extract_params_header instead")] +pub fn extract_req_params_header(head: &HeaderMap) -> HashMap { + extract_params_header(head) +} + +/// Extract parameters from hyper::HeaderMap, mainly header information. +pub fn extract_params_header(head: &HeaderMap) -> HashMap { + let mut params = HashMap::new(); + for (key, value) in head.iter() { + if let Ok(val_str) = value.to_str() { + params.insert(key.as_str().to_string(), val_str.to_string()); + } + } + params +} + +/// Extract response elements from S3Response, mainly header information. +pub fn extract_resp_elements(resp: &S3Response) -> HashMap { + extract_params_header(&resp.headers) +} + +/// Get host from header information. +pub fn get_request_host(headers: &HeaderMap) -> String { + headers + .get("host") + .and_then(|v| v.to_str().ok()) + .unwrap_or_default() + .to_string() +} + +/// Get Port from header information. +/// Priority: +/// 1. x-forwarded-port +/// 2. host header (parse port) +/// 3. x-forwarded-proto inferred default (http=80, https=443) when host has no explicit port +/// 4. port header +pub fn get_request_port(headers: &HeaderMap) -> u16 { + if let Some(port) = headers + .get("x-forwarded-port") + .and_then(|v| v.to_str().ok()) + .and_then(|s| s.parse::().ok()) + { + return port; + } + + if let Some(host) = headers.get("host").and_then(|v| v.to_str().ok()) { + if let Some(idx) = host.rfind(':') { + let valid_colon = match host.rfind(']') { + Some(close_bracket_idx) => idx > close_bracket_idx, + None => true, + }; + + if valid_colon + && let Ok(port) = host[idx + 1..].parse::() + && port > 0 + { + return port; + } + } + + if let Some(proto) = headers.get("x-forwarded-proto").and_then(|v| v.to_str().ok()) { + match proto { + "http" => return 80, + "https" => return 443, + _ => {} + } + } + } + + headers + .get("port") + .and_then(|v| v.to_str().ok()) + .and_then(|s| s.parse::().ok()) + .unwrap_or(0) +} + +/// Get content-length from header information. +pub fn get_request_content_length(headers: &HeaderMap) -> u64 { + headers + .get("content-length") + .and_then(|v| v.to_str().ok()) + .and_then(|s| s.parse::().ok()) + .unwrap_or(0) +} + +/// Get referer from header information. +pub fn get_request_referer(headers: &HeaderMap) -> String { + headers + .get("referer") + .and_then(|v| v.to_str().ok()) + .unwrap_or_default() + .to_string() +} + +/// Get user-agent from header information. +pub fn get_request_user_agent(headers: &HeaderMap) -> String { + headers + .get("user-agent") + .and_then(|v| v.to_str().ok()) + .unwrap_or_default() + .to_string() +} + +/// parse_host parses a string into a Host, with validation similar to Go's ParseHost. pub fn parse_host(s: &str) -> Result { if s.is_empty() { return Err(NetError::InvalidArgument); } - // is_valid_host validates the host string, checking for IP or hostname validity. let is_valid_host = |host: &str| -> bool { if host.is_empty() { return true; @@ -122,9 +229,6 @@ pub fn parse_host(s: &str) -> Result { return Err(NetError::MissingBracket); } - // A host with multiple colons is an IPv6 literal, optionally with a - // zone identifier. Unbracketed IPv6 with port is ambiguous, so callers - // must use the standard bracketed form when they need a port. let (host_str, port_str) = if s.matches(':').count() > 1 { (s, "") } else { @@ -139,7 +243,6 @@ pub fn parse_host(s: &str) -> Result { (trim_ipv6(host_str)?, port) }; - // Handle IPv6 zone identifier. let trimmed_host = host.split('%').next().unwrap_or(&host); if !is_valid_host(trimmed_host) { @@ -149,7 +252,6 @@ pub fn parse_host(s: &str) -> Result { Ok(Host { name: host, port }) } -// trim_ipv6 removes square brackets from IPv6 addresses, similar to Go's trimIPv6. fn trim_ipv6(host: &str) -> Result { if host.ends_with(']') { if !host.starts_with('[') { @@ -162,37 +264,18 @@ fn trim_ipv6(host: &str) -> Result { } /// URL is a wrapper around url::Url for custom handling. -/// Provides methods similar to Go's URL struct. #[derive(Debug, Clone)] pub struct ParsedURL(pub Url); impl ParsedURL { - /// is_empty returns true if the URL is empty or "about:blank". - /// - /// # Arguments - /// * `&self` - Reference to the ParsedURL instance. - /// - /// # Returns - /// * `bool` - True if the URL is empty or "about:blank", false otherwise. - /// pub fn is_empty(&self) -> bool { self.0.as_str() == "" || (self.0.scheme() == "about" && self.0.path() == "blank") } - /// hostname returns the hostname of the URL. - /// - /// # Returns - /// * `String` - The hostname of the URL, or an empty string if not set. - /// pub fn hostname(&self) -> String { self.0.host_str().unwrap_or("").to_string() } - /// port returns the port of the URL as a string, defaulting to "80" for http and "443" for https if not set. - /// - /// # Returns - /// * `String` - The port of the URL as a string. - /// pub fn port(&self) -> String { match self.0.port() { Some(p) => p.to_string(), @@ -204,20 +287,10 @@ impl ParsedURL { } } - /// scheme returns the scheme of the URL. - /// - /// # Returns - /// * `&str` - The scheme of the URL. - /// pub fn scheme(&self) -> &str { self.0.scheme() } - /// url returns a reference to the underlying Url. - /// - /// # Returns - /// * `&Url` - Reference to the underlying Url. - /// pub fn url(&self) -> &Url { &self.0 } @@ -235,7 +308,6 @@ impl std::fmt::Display for ParsedURL { } let mut s = url.to_string(); - // If the URL ends with a slash and the path is just "/", remove the trailing slash. if s.ends_with('/') && url.path() == "/" { s.pop(); } @@ -268,17 +340,6 @@ impl<'de> serde::Deserialize<'de> for ParsedURL { } /// parse_url parses a string into a ParsedURL, with host validation and path cleaning. -/// -/// # Arguments -/// * `s` - The URL string to parse. -/// -/// # Returns -/// * `Ok(ParsedURL)` - If parsing is successful. -/// * `Err(NetError)` - If parsing fails or host is invalid. -/// -/// # Errors -/// Returns NetError if parsing fails or host is invalid. -/// pub fn parse_url(s: &str) -> Result { if let Some(scheme_end) = s.find("://") && s[scheme_end + 3..].starts_with('/') @@ -303,13 +364,11 @@ pub fn parse_url(s: &str) -> Result { if !port_str.is_empty() { let host_port = format!("{}:{}", uu.host_str().unwrap(), port_str); - parse_host(&host_port)?; // Validate host. + parse_host(&host_port)?; } } - // Clean path: Use Url's path_segments to normalize. if !uu.path().is_empty() { - // Url automatically cleans paths, but we ensure trailing slash if original had it. let mut cleaned_path = String::new(); for comp in Path::new(uu.path()).components() { use std::path::Component; @@ -337,15 +396,6 @@ pub fn parse_url(s: &str) -> Result { } #[allow(dead_code)] -/// parse_http_url parses a string into a ParsedURL, ensuring the scheme is http or https. -/// -/// # Arguments -/// * `s` - The URL string to parse. -/// -/// # Returns -/// * `Ok(ParsedURL)` - If parsing is successful and scheme is http/https. -/// * `Err(NetError)` - If parsing fails or scheme is not http/https. -/// pub fn parse_http_url(s: &str) -> Result { let u = parse_url(s)?; match u.0.scheme() { @@ -355,20 +405,10 @@ pub fn parse_http_url(s: &str) -> Result { } #[allow(dead_code)] -/// is_network_or_host_down checks if an error indicates network or host down, considering timeouts. -/// -/// # Arguments -/// * `err` - The std::io::Error to check. -/// * `expect_timeouts` - Whether timeouts are expected. -/// -/// # Returns -/// * `bool` - True if the error indicates network or host down, false otherwise. -/// pub fn is_network_or_host_down(err: &std::io::Error, expect_timeouts: bool) -> bool { if err.kind() == std::io::ErrorKind::TimedOut { return !expect_timeouts; } - // Simplified checks based on Go logic; adapt for Rust as needed let err_str = err.to_string().to_lowercase(); err_str.contains("connection reset by peer") || err_str.contains("connection timed out") @@ -377,27 +417,11 @@ pub fn is_network_or_host_down(err: &std::io::Error, expect_timeouts: bool) -> b } #[allow(dead_code)] -/// is_conn_reset_err checks if an error indicates a connection reset by peer. -/// -/// # Arguments -/// * `err` - The std::io::Error to check. -/// -/// # Returns -/// * `bool` - True if the error indicates connection reset, false otherwise. -/// pub fn is_conn_reset_err(err: &std::io::Error) -> bool { err.to_string().contains("connection reset by peer") || matches!(err.raw_os_error(), Some(libc::ECONNRESET)) } #[allow(dead_code)] -/// is_conn_refused_err checks if an error indicates a connection refused. -/// -/// # Arguments -/// * `err` - The std::io::Error to check. -/// -/// # Returns -/// * `bool` - True if the error indicates connection refused, false otherwise. -/// pub fn is_conn_refused_err(err: &std::io::Error) -> bool { err.to_string().contains("connection refused") || matches!(err.raw_os_error(), Some(libc::ECONNREFUSED)) } @@ -405,6 +429,51 @@ pub fn is_conn_refused_err(err: &std::io::Error) -> bool { #[cfg(test)] mod tests { use super::*; + use hyper::header::HeaderValue; + + #[test] + fn test_get_request_port() { + let mut headers = HeaderMap::new(); + + assert_eq!(get_request_port(&headers), 0); + + headers.insert("port", HeaderValue::from_static("8080")); + assert_eq!(get_request_port(&headers), 8080); + + headers.remove("port"); + headers.insert("host", HeaderValue::from_static("example.com:9000")); + assert_eq!(get_request_port(&headers), 9000); + + headers.insert("host", HeaderValue::from_static("example.com")); + assert_eq!(get_request_port(&headers), 0); + + headers.insert("host", HeaderValue::from_static("[::1]:9001")); + assert_eq!(get_request_port(&headers), 9001); + + headers.insert("host", HeaderValue::from_static("[::1]")); + assert_eq!(get_request_port(&headers), 0); + + headers.insert("x-forwarded-port", HeaderValue::from_static("7000")); + assert_eq!(get_request_port(&headers), 7000); + + headers.remove("x-forwarded-port"); + headers.insert("host", HeaderValue::from_static("example.com")); + headers.insert("x-forwarded-proto", HeaderValue::from_static("http")); + assert_eq!(get_request_port(&headers), 80); + + headers.insert("x-forwarded-proto", HeaderValue::from_static("https")); + assert_eq!(get_request_port(&headers), 443); + + headers.insert("x-forwarded-proto", HeaderValue::from_static("ftp")); + assert_eq!(get_request_port(&headers), 0); + + headers.insert("host", HeaderValue::from_static("example.com:0")); + headers.insert("x-forwarded-proto", HeaderValue::from_static("https")); + assert_eq!(get_request_port(&headers), 443); + + headers.remove("x-forwarded-proto"); + assert_eq!(get_request_port(&headers), 0); + } #[test] fn parse_host_with_empty_string_returns_error() { @@ -532,32 +601,6 @@ mod tests { assert_eq!(host.to_string(), "example.com"); } - #[test] - fn host_equal_when_same() { - let host1 = Host { - name: "example.com".to_string(), - port: Some(80), - }; - let host2 = Host { - name: "example.com".to_string(), - port: Some(80), - }; - assert!(host1.equal(&host2)); - } - - #[test] - fn host_not_equal_when_different() { - let host1 = Host { - name: "example.com".to_string(), - port: Some(80), - }; - let host2 = Host { - name: "example.com".to_string(), - port: Some(443), - }; - assert!(!host1.equal(&host2)); - } - #[test] fn parse_url_with_valid_http_url() { let result = parse_url("http://example.com/path"); @@ -565,100 +608,35 @@ mod tests { let parsed = result.unwrap(); assert_eq!(parsed.hostname(), "example.com"); assert_eq!(parsed.port(), "80"); + assert_eq!(parsed.scheme(), "http"); + assert_eq!(parsed.to_string(), "http://example.com/path"); } #[test] - fn parse_url_with_valid_https_url() { + fn parse_url_with_explicit_default_https_port() { let result = parse_url("https://example.com:443/path"); assert!(result.is_ok()); let parsed = result.unwrap(); - assert_eq!(parsed.hostname(), "example.com"); - assert_eq!(parsed.port(), "443"); + assert_eq!(parsed.to_string(), "https://example.com/path"); } #[test] - fn parse_url_with_scheme_but_empty_host() { + fn parse_url_with_empty_host_returns_error() { let result = parse_url("http:///path"); assert!(matches!(result, Err(NetError::SchemeWithEmptyHost))); } #[test] - fn parse_url_with_invalid_host() { + fn parse_url_with_invalid_host_returns_error() { let result = parse_url("http://invalid..host/path"); assert!(matches!(result, Err(NetError::InvalidHost))); } #[test] - fn parse_url_with_path_cleaning() { + fn parse_url_normalizes_path() { let result = parse_url("http://example.com//path/../path/"); assert!(result.is_ok()); let parsed = result.unwrap(); - assert_eq!(parsed.0.path(), "/path/"); - } - - #[test] - fn parse_http_url_with_http_scheme() { - let result = parse_http_url("http://example.com"); - assert!(result.is_ok()); - } - - #[test] - fn parse_http_url_with_https_scheme() { - let result = parse_http_url("https://example.com"); - assert!(result.is_ok()); - } - - #[test] - fn parse_http_url_with_invalid_scheme() { - let result = parse_http_url("ftp://example.com"); - assert!(matches!(result, Err(NetError::UnexpectedScheme(_)))); - } - - #[test] - fn parsed_url_is_empty_when_url_is_empty() { - let url = ParsedURL(Url::parse("about:blank").unwrap()); - assert!(url.is_empty()); - } - - #[test] - fn parsed_url_hostname() { - let url = ParsedURL(Url::parse("http://example.com:8080").unwrap()); - assert_eq!(url.hostname(), "example.com"); - } - - #[test] - fn parsed_url_port() { - let url = ParsedURL(Url::parse("http://example.com:8080").unwrap()); - assert_eq!(url.port(), "8080"); - } - - #[test] - fn parsed_url_to_string_removes_default_ports() { - let url = ParsedURL(Url::parse("http://example.com:80").unwrap()); - assert_eq!(url.to_string(), "http://example.com"); - } - - #[test] - fn is_network_or_host_down_with_timeout() { - let err = std::io::Error::new(std::io::ErrorKind::TimedOut, "timeout"); - assert!(is_network_or_host_down(&err, false)); - } - - #[test] - fn is_network_or_host_down_with_expected_timeout() { - let err = std::io::Error::new(std::io::ErrorKind::TimedOut, "timeout"); - assert!(!is_network_or_host_down(&err, true)); - } - - #[test] - fn is_conn_reset_err_with_reset_message() { - let err = std::io::Error::other("connection reset by peer"); - assert!(is_conn_reset_err(&err)); - } - - #[test] - fn is_conn_refused_err_with_refused_message() { - let err = std::io::Error::other("connection refused"); - assert!(is_conn_refused_err(&err)); + assert_eq!(parsed.to_string(), "http://example.com/path/"); } } diff --git a/crates/targets/src/runtime/mod.rs b/crates/targets/src/runtime/mod.rs index cee4e1136..6ac58fb1c 100644 --- a/crates/targets/src/runtime/mod.rs +++ b/crates/targets/src/runtime/mod.rs @@ -15,6 +15,7 @@ pub mod adapter; pub mod sidecar; pub mod sidecar_protocol; +pub mod tls; use crate::Target; use crate::arn::TargetID; diff --git a/crates/targets/src/runtime/tls/adapter.rs b/crates/targets/src/runtime/tls/adapter.rs new file mode 100644 index 000000000..b022464f7 --- /dev/null +++ b/crates/targets/src/runtime/tls/adapter.rs @@ -0,0 +1,217 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! `TlsReloadAdapter` — the single entry-point that connects a target to +//! the TLS reload coordinator. Each target holds an `Option>` +//! and calls `current_material()` on the hot path. When `None`, the target +//! falls back to its legacy inline fingerprint logic. + +use super::config::TlsReloadOptions; +use super::coordinator::TargetTlsReloadCoordinator; +use super::state::{TargetTlsRuntimeState, TargetTlsStatusSnapshot}; +use super::r#trait::ReloadableTargetTls; +use std::sync::Arc; +use tracing::warn; + +/// Bridges a `ReloadableTargetTls` implementor and the reload coordinator. +/// +/// Created via [`TlsReloadAdapter::try_register`]. Holds the coordinator- +/// managed runtime state and exposes a zero-cost `current_material()` accessor +/// for the send hot-path. +pub struct TlsReloadAdapter { + runtime_state: Arc>, + options: TlsReloadOptions, +} + +impl std::fmt::Debug for TlsReloadAdapter { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("TlsReloadAdapter") + .field("target_label", &self.runtime_state.inputs.target_label) + .finish_non_exhaustive() + } +} + +impl Clone for TlsReloadAdapter { + fn clone(&self) -> Self { + Self { + runtime_state: Arc::clone(&self.runtime_state), + options: self.options.clone(), + } + } +} + +impl TlsReloadAdapter { + /// Registers `target` with the coordinator and returns an adapter. + /// + /// On success the coordinator has: + /// - built initial TLS material + /// - spawned a background poll loop + /// + /// On failure returns `None` (the caller should keep its inline fallback + /// path intact — the target continues to work, just without coordinator + /// support). + pub async fn try_register>( + target: Arc, + options: TlsReloadOptions, + coordinator: &TargetTlsReloadCoordinator, + ) -> Option { + let label = target.tls_input_set().target_label.clone(); + match coordinator.register(target, options.clone()).await { + Ok(runtime_state) => { + tracing::info!(target = %label, "TLS reload adapter registered"); + Some(Self { runtime_state, options }) + } + Err(err) => { + warn!(target = %label, error = %err, "TLS reload adapter registration failed; target will use inline fallback"); + None + } + } + } + + /// Hot-path accessor: returns the current TLS material managed by the + /// coordinator. The returned `Arc` is cheap to clone. + #[inline] + pub fn current_material(&self) -> Arc { + Arc::clone(&self.runtime_state.current.load().material) + } + + /// Returns the active generation counter. + #[inline] + pub fn generation(&self) -> u64 { + self.runtime_state.current.load().generation.0 + } + + /// Returns a read-only status snapshot for admin/observability. + pub fn status_snapshot(&self) -> TargetTlsStatusSnapshot { + TargetTlsReloadCoordinator::build_status_snapshot(&self.runtime_state, &self.options) + } + + /// Returns the underlying runtime state (for `close()` cleanup etc.). + pub fn runtime_state(&self) -> &Arc> { + &self.runtime_state + } + + /// Unregisters from the coordinator (stops the poll loop). + pub async fn unregister(&self, coordinator: &TargetTlsReloadCoordinator) { + let label = &self.runtime_state.inputs.target_label; + if let Err(err) = coordinator.unregister(label).await { + warn!(target = %label, error = %err, "Failed to unregister TLS reload adapter"); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::error::TargetError; + use async_trait::async_trait; + use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + + struct FakeTarget { + label: String, + build_calls: AtomicUsize, + should_fail: AtomicBool, + } + + impl FakeTarget { + fn new(label: &str) -> Self { + Self { + label: label.to_string(), + build_calls: AtomicUsize::new(0), + should_fail: AtomicBool::new(false), + } + } + } + + #[async_trait] + impl ReloadableTargetTls for FakeTarget { + type Material = String; + + fn tls_input_set(&self) -> super::super::state::TargetTlsInputSet { + super::super::state::TargetTlsInputSet { + ca_path: String::new(), + client_cert_path: String::new(), + client_key_path: String::new(), + target_label: self.label.clone(), + } + } + + async fn build_tls_material(&self) -> Result { + self.build_calls.fetch_add(1, Ordering::SeqCst); + if self.should_fail.load(Ordering::SeqCst) { + return Err(TargetError::Configuration("fail".to_string())); + } + Ok("material".to_string()) + } + + async fn apply_tls_material( + &self, + _generation: super::super::fingerprint::TargetTlsGeneration, + _material: Arc, + _mode: super::super::config::ReloadApplyMode, + ) -> Result<(), TargetError> { + Ok(()) + } + } + + #[tokio::test] + async fn try_register_returns_adapter_on_success() { + let coordinator = TargetTlsReloadCoordinator::new(); + let target = Arc::new(FakeTarget::new("test:fake")); + let options = TlsReloadOptions::default(); + + let adapter = TlsReloadAdapter::try_register(target.clone(), options, &coordinator).await; + assert!(adapter.is_some()); + assert_eq!(target.build_calls.load(Ordering::SeqCst), 1); + + let a = adapter.unwrap(); + assert_eq!(*a.current_material(), "material"); + } + + #[tokio::test] + async fn try_register_returns_none_on_failure() { + let coordinator = TargetTlsReloadCoordinator::new(); + let target = Arc::new(FakeTarget::new("test:fail")); + target.should_fail.store(true, Ordering::SeqCst); + let options = TlsReloadOptions::default(); + + let adapter = TlsReloadAdapter::try_register(target, options, &coordinator).await; + assert!(adapter.is_none()); + } + + #[tokio::test] + async fn adapter_is_clone_and_shares_state() { + let coordinator = TargetTlsReloadCoordinator::new(); + let target = Arc::new(FakeTarget::new("test:clone")); + let options = TlsReloadOptions::default(); + + let a = TlsReloadAdapter::try_register(target, options, &coordinator).await.unwrap(); + let b = a.clone(); + + assert_eq!(*a.current_material(), *b.current_material()); + assert_eq!(a.generation(), b.generation()); + } + + #[tokio::test] + async fn status_snapshot_contains_label() { + let coordinator = TargetTlsReloadCoordinator::new(); + let target = Arc::new(FakeTarget::new("test:snap")); + let options = TlsReloadOptions::default(); + + let adapter = TlsReloadAdapter::try_register(target, options, &coordinator).await.unwrap(); + let snap = adapter.status_snapshot(); + assert_eq!(snap.target_label, "test:snap"); + assert!(snap.reload_enabled); + } +} diff --git a/crates/targets/src/runtime/tls/config.rs b/crates/targets/src/runtime/tls/config.rs new file mode 100644 index 000000000..3ed7c023a --- /dev/null +++ b/crates/targets/src/runtime/tls/config.rs @@ -0,0 +1,20 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Target-level TLS reload configuration. +//! +//! Re-exports the shared types from `rustfs_tls_runtime::config` so that +//! targets and their callers use a single source of truth. + +pub use rustfs_tls_runtime::config::{ReloadApplyHint as ReloadApplyMode, ReloadDetectMode, TlsReloadOptions}; diff --git a/crates/targets/src/runtime/tls/coordinator.rs b/crates/targets/src/runtime/tls/coordinator.rs new file mode 100644 index 000000000..24324cc25 --- /dev/null +++ b/crates/targets/src/runtime/tls/coordinator.rs @@ -0,0 +1,656 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Target TLS reload coordinator. Manages per-target background poll loops +//! that periodically check TLS material fingerprints and drive safe reload. + +use super::config::{ReloadApplyMode, ReloadDetectMode, TlsReloadOptions}; +#[cfg(test)] +use super::fingerprint::TargetTlsFingerprint; +use super::fingerprint::{TargetTlsGeneration, build_target_tls_fingerprint}; +use super::metrics::{record_target_tls_publication_fail, record_target_tls_reload_result, record_target_tls_reload_skipped}; +#[cfg(test)] +use super::state::TargetTlsInputSet; +use super::state::{TargetTlsPublishedState, TargetTlsRuntimeState, TargetTlsStatusSnapshot}; +use super::r#trait::ReloadableTargetTls; +use super::validate::validate_tls_material; +use crate::error::TargetError; +use std::collections::HashMap; +use std::sync::Arc; +use std::time::{SystemTime, UNIX_EPOCH}; +use tokio::sync::RwLock; +use tokio::task::JoinHandle; +use tracing::{debug, info, warn}; + +struct TargetReloadEntry { + #[expect(dead_code)] + target_label: String, + cancel_tx: tokio::sync::mpsc::Sender<()>, + poll_handle: JoinHandle<()>, +} + +/// The top-level coordinator that manages TLS reload for all registered targets. +/// +/// Typically one instance per process, held alongside `TargetRuntimeManager`. +/// Each registered target gets its own background poll loop that periodically +/// checks TLS fingerprints and drives the build/apply cycle. +pub struct TargetTlsReloadCoordinator { + entries: RwLock>, +} + +impl Default for TargetTlsReloadCoordinator { + fn default() -> Self { + Self::new() + } +} + +impl TargetTlsReloadCoordinator { + pub fn new() -> Self { + Self { + entries: RwLock::new(HashMap::new()), + } + } + + /// Register a target for coordinated TLS reload. Spawns a background poll loop. + /// + /// Returns the initial runtime state that the target should hold for + /// accessing the current TLS material via `ArcSwap`. + pub async fn register( + &self, + target: Arc, + options: TlsReloadOptions, + ) -> Result>, TargetError> { + if !options.enabled { + return Err(TargetError::Configuration("TLS reload is disabled".to_string())); + } + + let inputs = target.tls_input_set(); + let target_label = inputs.target_label.clone(); + + // Build initial material + let initial_material = Arc::new(target.build_tls_material().await?); + let initial_fingerprint = + build_target_tls_fingerprint(&inputs.ca_path, &inputs.client_cert_path, &inputs.client_key_path).await?; + + let initial_state = Arc::new(TargetTlsPublishedState { + generation: TargetTlsGeneration(1), + fingerprint: initial_fingerprint, + material: initial_material, + loaded_at_unix_ms: unix_time_ms(), + }); + + let runtime_state = Arc::new(TargetTlsRuntimeState::new(initial_state.clone(), inputs)); + + if options.detect_mode == ReloadDetectMode::Poll || options.detect_mode == ReloadDetectMode::Hybrid { + let (cancel_tx, cancel_rx) = tokio::sync::mpsc::channel(1); + let poll_handle = tokio::spawn(spawn_target_poll_loop(target, Arc::clone(&runtime_state), options, cancel_rx)); + + let mut entries = self.entries.write().await; + entries.insert( + target_label.clone(), + TargetReloadEntry { + target_label: target_label.clone(), + cancel_tx, + poll_handle, + }, + ); + + info!(target = %target_label, "Registered target for TLS reload coordinator"); + } + + Ok(runtime_state) + } + + /// Unregister a target and stop its poll loop. + pub async fn unregister(&self, target_label: &str) -> Result<(), TargetError> { + let mut entries = self.entries.write().await; + if let Some(entry) = entries.remove(target_label) { + let _ = entry.cancel_tx.send(()).await; + entry.poll_handle.abort(); + info!(target = %target_label, "Unregistered target from TLS reload coordinator"); + } + Ok(()) + } + + /// Force an immediate reload check for a specific target. + /// Used by admin endpoints and test harnesses. + pub async fn force_reload( + &self, + target: &T, + runtime_state: &TargetTlsRuntimeState, + options: &TlsReloadOptions, + ) -> Result { + reload_target_once(target, runtime_state, options).await + } + + /// Stop all poll loops. + pub async fn shutdown(&self) { + let mut entries = self.entries.write().await; + for (label, entry) in entries.drain() { + let _ = entry.cancel_tx.send(()).await; + entry.poll_handle.abort(); + debug!(target = %label, "Stopped TLS reload poll loop"); + } + } + + /// Collect status snapshots from all registered targets. + /// The caller must provide the runtime states separately since the + /// coordinator does not hold type-erased references to them. + pub fn build_status_snapshot( + runtime_state: &TargetTlsRuntimeState, + options: &TlsReloadOptions, + ) -> TargetTlsStatusSnapshot { + let current = runtime_state.current.load(); + let last_attempt = runtime_state.last_attempt_unix_ms(); + let last_success = runtime_state.last_success_unix_ms(); + let last_error = runtime_state.last_error.read().clone(); + + TargetTlsStatusSnapshot { + target_label: runtime_state.inputs.target_label.clone(), + generation: current.generation.0, + reload_enabled: options.enabled, + detect_mode: match options.detect_mode { + ReloadDetectMode::Poll => "poll", + ReloadDetectMode::Watch => "watch", + ReloadDetectMode::Hybrid => "hybrid", + }, + apply_mode: match options.apply_hint { + ReloadApplyMode::Lazy => "lazy", + ReloadApplyMode::SoftReconnect => "soft_reconnect", + }, + last_attempt_time: if last_attempt > 0 { Some(last_attempt) } else { None }, + last_success_time: if last_success > 0 { Some(last_success) } else { None }, + last_error, + ca_path: runtime_state.inputs.ca_path.clone(), + client_cert_path: runtime_state.inputs.client_cert_path.clone(), + client_key_path: runtime_state.inputs.client_key_path.clone(), + } + } +} + +/// Background poll loop for a single target. +async fn spawn_target_poll_loop( + target: Arc, + runtime_state: Arc>, + options: TlsReloadOptions, + mut cancel_rx: tokio::sync::mpsc::Receiver<()>, +) { + let mut interval = tokio::time::interval(options.interval); + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + interval.tick().await; // skip the immediate first tick + + let label = &runtime_state.inputs.target_label; + let debounce = options.debounce; + debug!(target = %label, interval_secs = options.interval.as_secs(), "TLS reload poll loop started"); + + loop { + tokio::select! { + biased; + _ = cancel_rx.recv() => { + info!(target = %label, "TLS reload poll loop stopped"); + return; + } + _ = interval.tick() => { + // Enforce minimum stable age: if the last attempt was too recent + // (e.g. a rapid succession of ticks), wait one debounce period + // before reading files again to avoid picking up half-written certs. + let last_attempt = runtime_state.last_attempt_unix_ms(); + if last_attempt > 0 { + let elapsed_since_last = unix_time_ms().saturating_sub(last_attempt); + if elapsed_since_last < debounce.as_millis() as u64 { + continue; + } + } + + if let Err(err) = reload_target_once(target.as_ref(), runtime_state.as_ref(), &options).await { + warn!(target = %label, error = %err, "TLS reload poll check failed (will retry)"); + } + } + } + } +} + +/// Single reload cycle: read → compare → validate → build → apply → publish. +/// +/// Returns the new generation on success, or an error on failure. +/// On failure the current generation and material are untouched. +async fn reload_target_once( + target: &T, + runtime_state: &TargetTlsRuntimeState, + options: &TlsReloadOptions, +) -> Result { + let now = unix_time_ms(); + runtime_state.mark_attempt(now); + let started_at = std::time::Instant::now(); + let label = &runtime_state.inputs.target_label; + + // 1. Read TLS files and compute fingerprint + let inputs = &runtime_state.inputs; + let next_fingerprint = + build_target_tls_fingerprint(&inputs.ca_path, &inputs.client_cert_path, &inputs.client_key_path).await?; + + // 2. Compare with current — skip if unchanged + let current = runtime_state.current.load(); + if current.fingerprint == next_fingerprint { + record_target_tls_reload_skipped(label, "unchanged"); + return Ok(current.generation); + } + + // 3. Validate TLS files (cert/key pairing, CA parseable) + if let Err(err) = validate_tls_material(&inputs.ca_path, &inputs.client_cert_path, &inputs.client_key_path) { + *runtime_state.last_error.write() = Some(err.to_string()); + record_target_tls_publication_fail(label); + return Err(err); + } + + // Also call target-specific validation + if let Err(err) = target.validate_tls_files().await { + *runtime_state.last_error.write() = Some(err.to_string()); + record_target_tls_publication_fail(label); + return Err(err); + } + + // 4. Build new material (does not touch current state yet) + let new_material = match target.build_tls_material().await { + Ok(m) => Arc::new(m), + Err(err) => { + *runtime_state.last_error.write() = Some(err.to_string()); + record_target_tls_publication_fail(label); + return Err(err); + } + }; + + // 5. Bump generation and apply + let new_generation = runtime_state.bump_generation(); + if let Err(err) = target + .apply_tls_material(new_generation, Arc::clone(&new_material), options.apply_hint) + .await + { + *runtime_state.last_error.write() = Some(err.to_string()); + record_target_tls_publication_fail(label); + return Err(err); + } + + // 6. Publish new state + let published = Arc::new(TargetTlsPublishedState { + generation: new_generation, + fingerprint: next_fingerprint, + material: new_material, + loaded_at_unix_ms: now, + }); + runtime_state.current.store(published.clone()); + runtime_state.last_good.store(published); + runtime_state.mark_success(now); + *runtime_state.last_error.write() = None; + + record_target_tls_reload_result(label, "ok", started_at.elapsed().as_secs_f64(), new_generation.0); + + debug!(target = %label, generation = new_generation.0, "TLS reload successful"); + Ok(new_generation) +} + +fn unix_time_ms() -> u64 { + SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_millis() as u64 +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + + struct MockTarget { + inputs: TargetTlsInputSet, + build_calls: AtomicUsize, + apply_calls: AtomicUsize, + validate_calls: AtomicUsize, + should_fail_build: AtomicBool, + should_fail_apply: AtomicBool, + should_fail_validate: AtomicBool, + } + + impl MockTarget { + fn new(label: &str) -> Self { + Self { + inputs: TargetTlsInputSet { + ca_path: String::new(), + client_cert_path: String::new(), + client_key_path: String::new(), + target_label: label.to_string(), + }, + build_calls: AtomicUsize::new(0), + apply_calls: AtomicUsize::new(0), + validate_calls: AtomicUsize::new(0), + should_fail_build: AtomicBool::new(false), + should_fail_apply: AtomicBool::new(false), + should_fail_validate: AtomicBool::new(false), + } + } + } + + #[async_trait::async_trait] + impl ReloadableTargetTls for MockTarget { + type Material = String; + + fn tls_input_set(&self) -> TargetTlsInputSet { + self.inputs.clone() + } + + async fn build_tls_material(&self) -> Result { + self.build_calls.fetch_add(1, Ordering::SeqCst); + if self.should_fail_build.load(Ordering::SeqCst) { + return Err(TargetError::Configuration("build failed".to_string())); + } + Ok("mock-material".to_string()) + } + + async fn apply_tls_material( + &self, + _generation: TargetTlsGeneration, + _material: Arc, + _mode: ReloadApplyMode, + ) -> Result<(), TargetError> { + self.apply_calls.fetch_add(1, Ordering::SeqCst); + if self.should_fail_apply.load(Ordering::SeqCst) { + return Err(TargetError::Configuration("apply failed".to_string())); + } + Ok(()) + } + + async fn validate_tls_files(&self) -> Result<(), TargetError> { + self.validate_calls.fetch_add(1, Ordering::SeqCst); + if self.should_fail_validate.load(Ordering::SeqCst) { + return Err(TargetError::Configuration("validate failed".to_string())); + } + Ok(()) + } + } + + fn default_options() -> TlsReloadOptions { + TlsReloadOptions { + enabled: true, + detect_mode: ReloadDetectMode::Poll, + interval: std::time::Duration::from_secs(1), + debounce: std::time::Duration::from_secs(1), + min_stable_age: std::time::Duration::from_millis(100), + apply_hint: ReloadApplyMode::Lazy, + } + } + + #[tokio::test] + async fn register_builds_initial_material() { + let coordinator = TargetTlsReloadCoordinator::new(); + let target = Arc::new(MockTarget::new("test:webhook")); + + let options = TlsReloadOptions { + detect_mode: ReloadDetectMode::Watch, // no poll loop for this test + ..default_options() + }; + let state = coordinator.register(target.clone(), options).await.unwrap(); + + assert_eq!(state.current.load().generation, TargetTlsGeneration(1)); + assert_eq!(target.build_calls.load(Ordering::SeqCst), 1); + } + + #[tokio::test] + async fn register_disabled_returns_error() { + let coordinator = TargetTlsReloadCoordinator::new(); + let target = Arc::new(MockTarget::new("test:webhook")); + + let options = TlsReloadOptions { + enabled: false, + ..default_options() + }; + let result = coordinator.register(target, options).await; + assert!(result.is_err()); + } + + #[tokio::test] + async fn shutdown_stops_all_loops() { + let coordinator = TargetTlsReloadCoordinator::new(); + let target = Arc::new(MockTarget::new("test:webhook")); + + let _state = coordinator.register(target.clone(), default_options()).await.unwrap(); + assert_eq!(coordinator.entries.read().await.len(), 1); + + coordinator.shutdown().await; + assert!(coordinator.entries.read().await.is_empty()); + } + + #[tokio::test] + async fn force_reload_calls_build_and_apply() { + let target = MockTarget::new("test:webhook"); + let initial_material = Arc::new("initial".to_string()); + let initial_state = Arc::new(TargetTlsPublishedState { + generation: TargetTlsGeneration(1), + fingerprint: TargetTlsFingerprint::default(), + material: initial_material, + loaded_at_unix_ms: 0, + }); + let inputs = target.tls_input_set(); + let runtime_state = Arc::new(TargetTlsRuntimeState::new(initial_state, inputs)); + let options = default_options(); + + // Force reload should succeed since MockTarget uses empty paths + // and the fingerprint won't change from default + let result = reload_target_once(&target, &runtime_state, &options).await.unwrap(); + // Since fingerprint is unchanged (empty paths), generation stays at 1 + assert_eq!(result, TargetTlsGeneration(1)); + // Build should NOT be called because fingerprint unchanged + assert_eq!(target.build_calls.load(Ordering::SeqCst), 0); + } + + #[tokio::test] + async fn build_failure_preserves_old_generation() { + let target = MockTarget::new("test:webhook"); + target.should_fail_build.store(true, Ordering::SeqCst); + + // Use a non-default fingerprint so the reload will detect a change + // (empty paths → default fingerprint ≠ initial fingerprint) + let initial_material = Arc::new("initial".to_string()); + let initial_fingerprint = TargetTlsFingerprint { + ca_sha256: Some([1; 32]), + client_cert_sha256: None, + client_key_sha256: None, + }; + let initial_state = Arc::new(TargetTlsPublishedState { + generation: TargetTlsGeneration(1), + fingerprint: initial_fingerprint, + material: initial_material, + loaded_at_unix_ms: 0, + }); + let runtime_state = Arc::new(TargetTlsRuntimeState::new(initial_state, target.tls_input_set())); + let options = default_options(); + + // Empty paths produce default fingerprint which differs from initial → + // validate passes (empty paths), then build is called and fails. + let result = reload_target_once(&target, &runtime_state, &options).await; + assert!(result.is_err()); + // Generation should remain at 1 + assert_eq!(runtime_state.current_generation(), TargetTlsGeneration(1)); + assert_eq!(target.build_calls.load(Ordering::SeqCst), 1); + } + + #[tokio::test] + async fn status_snapshot_reflects_state() { + let target = MockTarget::new("test:webhook"); + let initial_material = Arc::new("initial".to_string()); + let initial_state = Arc::new(TargetTlsPublishedState { + generation: TargetTlsGeneration(1), + fingerprint: TargetTlsFingerprint::default(), + material: initial_material, + loaded_at_unix_ms: 0, + }); + let runtime_state = Arc::new(TargetTlsRuntimeState::new(initial_state, target.tls_input_set())); + let options = default_options(); + + let snapshot = TargetTlsReloadCoordinator::build_status_snapshot(runtime_state.as_ref(), &options); + assert_eq!(snapshot.target_label, "test:webhook"); + assert_eq!(snapshot.generation, 1); + assert!(snapshot.reload_enabled); + assert_eq!(snapshot.detect_mode, "poll"); + assert_eq!(snapshot.apply_mode, "lazy"); + assert!(snapshot.last_attempt_time.is_none()); + assert!(snapshot.last_error.is_none()); + } + + #[tokio::test] + async fn apply_failure_preserves_old_generation() { + let target = MockTarget::new("test:webhook"); + target.should_fail_apply.store(true, Ordering::SeqCst); + + // Use a non-default fingerprint so reload detects a change + let initial_material = Arc::new("initial".to_string()); + let initial_fingerprint = TargetTlsFingerprint { + ca_sha256: Some([42; 32]), + client_cert_sha256: None, + client_key_sha256: None, + }; + let initial_state = Arc::new(TargetTlsPublishedState { + generation: TargetTlsGeneration(3), + fingerprint: initial_fingerprint, + material: initial_material, + loaded_at_unix_ms: 0, + }); + let runtime_state = Arc::new(TargetTlsRuntimeState::new(initial_state, target.tls_input_set())); + let options = default_options(); + + let result = reload_target_once(&target, &runtime_state, &options).await; + assert!(result.is_err()); + // Generation should remain at 3 + assert_eq!(runtime_state.current_generation(), TargetTlsGeneration(3)); + assert!(runtime_state.last_error.read().is_some()); + } + + #[tokio::test] + async fn validate_failure_prevents_build() { + let target = MockTarget::new("test:kafka"); + target.should_fail_validate.store(true, Ordering::SeqCst); + + // Non-default fingerprint to trigger reload + let initial_material = Arc::new("initial".to_string()); + let initial_fingerprint = TargetTlsFingerprint { + ca_sha256: Some([99; 32]), + client_cert_sha256: None, + client_key_sha256: None, + }; + let initial_state = Arc::new(TargetTlsPublishedState { + generation: TargetTlsGeneration(1), + fingerprint: initial_fingerprint, + material: initial_material, + loaded_at_unix_ms: 0, + }); + let runtime_state = Arc::new(TargetTlsRuntimeState::new(initial_state, target.tls_input_set())); + let options = default_options(); + + let result = reload_target_once(&target, &runtime_state, &options).await; + assert!(result.is_err()); + // Build should NOT have been called + assert_eq!(target.build_calls.load(Ordering::SeqCst), 0); + // Validate was called + assert!(target.validate_calls.load(Ordering::SeqCst) > 0); + assert_eq!(runtime_state.current_generation(), TargetTlsGeneration(1)); + } + + #[tokio::test] + async fn error_is_cleared_on_successful_reload() { + let target = MockTarget::new("test:nats"); + + let initial_material = Arc::new("initial".to_string()); + let initial_fingerprint = TargetTlsFingerprint { + ca_sha256: Some([1; 32]), + client_cert_sha256: None, + client_key_sha256: None, + }; + let initial_state = Arc::new(TargetTlsPublishedState { + generation: TargetTlsGeneration(1), + fingerprint: initial_fingerprint, + material: initial_material, + loaded_at_unix_ms: 0, + }); + let runtime_state = Arc::new(TargetTlsRuntimeState::new(initial_state, target.tls_input_set())); + let options = default_options(); + + // First: fail the reload + target.should_fail_build.store(true, Ordering::SeqCst); + let _ = reload_target_once(&target, &runtime_state, &options).await; + assert!(runtime_state.last_error.read().is_some()); + + // Now succeed (fingerprint still different from default) + target.should_fail_build.store(false, Ordering::SeqCst); + let result = reload_target_once(&target, &runtime_state, &options).await; + assert!(result.is_ok()); + assert!(runtime_state.last_error.read().is_none()); + assert!(runtime_state.last_success_unix_ms() > 0); + } + + #[tokio::test] + async fn last_good_is_never_overwritten_by_failed_reload() { + let target = MockTarget::new("test:amqp"); + + let initial_material = Arc::new("good".to_string()); + let initial_fingerprint = TargetTlsFingerprint { + ca_sha256: Some([5; 32]), + client_cert_sha256: None, + client_key_sha256: None, + }; + let initial_state = Arc::new(TargetTlsPublishedState { + generation: TargetTlsGeneration(2), + fingerprint: initial_fingerprint.clone(), + material: initial_material.clone(), + loaded_at_unix_ms: 100, + }); + let runtime_state = Arc::new(TargetTlsRuntimeState::new(initial_state, target.tls_input_set())); + + // Verify last_good matches initial + let good = runtime_state.last_good.load(); + assert_eq!(good.generation, TargetTlsGeneration(2)); + + // Fail a reload + target.should_fail_build.store(true, Ordering::SeqCst); + let _ = reload_target_once(&target, &runtime_state, &default_options()).await; + + // last_good should still be the initial state + let good_after = runtime_state.last_good.load(); + assert_eq!(good_after.generation, TargetTlsGeneration(2)); + assert_eq!(good_after.fingerprint, initial_fingerprint); + } + + #[tokio::test] + async fn unregister_stops_target_poll_loop() { + let coordinator = TargetTlsReloadCoordinator::new(); + let target = Arc::new(MockTarget::new("test:pulsar")); + + let _state = coordinator.register(target.clone(), default_options()).await.unwrap(); + assert_eq!(coordinator.entries.read().await.len(), 1); + + coordinator.unregister("test:pulsar").await.unwrap(); + assert!(coordinator.entries.read().await.is_empty()); + } + + #[tokio::test] + async fn bump_generation_saturates_at_max() { + let target = MockTarget::new("test:saturation"); + let initial_material = Arc::new("initial".to_string()); + let max_gen_state = Arc::new(TargetTlsPublishedState { + generation: TargetTlsGeneration(u64::MAX), + fingerprint: TargetTlsFingerprint::default(), + material: initial_material, + loaded_at_unix_ms: 0, + }); + let runtime_state = Arc::new(TargetTlsRuntimeState::new(max_gen_state, target.tls_input_set())); + + let bumped = runtime_state.bump_generation(); + assert_eq!(bumped, TargetTlsGeneration(u64::MAX)); // saturating add + } +} diff --git a/crates/targets/src/runtime/tls/fingerprint.rs b/crates/targets/src/runtime/tls/fingerprint.rs new file mode 100644 index 000000000..753de10f6 --- /dev/null +++ b/crates/targets/src/runtime/tls/fingerprint.rs @@ -0,0 +1,160 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! TLS fingerprint types for per-target certificate hot-reload detection. + +use crate::error::TargetError; + +/// SHA256 digest per TLS file component used to detect certificate changes. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct TargetTlsFingerprint { + pub ca_sha256: Option<[u8; 32]>, + pub client_cert_sha256: Option<[u8; 32]>, + pub client_key_sha256: Option<[u8; 32]>, +} + +/// Monotonically increasing generation counter bumped on each successful reload. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct TargetTlsGeneration(pub u64); + +/// Combined TLS state held per-target for tracking reload progress. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct TargetTlsState { + pub generation: TargetTlsGeneration, + pub fingerprint: Option, +} + +impl TargetTlsState { + /// Compares `next_fingerprint` with the current one. If different, bumps + /// generation and stores the new fingerprint. Returns `true` when changed. + pub fn refresh(&mut self, next_fingerprint: TargetTlsFingerprint) -> bool { + if self.fingerprint.as_ref() == Some(&next_fingerprint) { + return false; + } + + self.generation = TargetTlsGeneration(self.generation.0.saturating_add(1)); + self.fingerprint = Some(next_fingerprint); + true + } + + /// Checks whether `candidate` differs from the stored fingerprint without + /// mutating state. Use this to gate a rebuild, then call `refresh` only + /// after the rebuild succeeds. + pub fn needs_update(&self, candidate: &TargetTlsFingerprint) -> bool { + self.fingerprint.as_ref() != Some(candidate) + } + + /// Resets state to default (generation 0, no fingerprint). + pub fn reset(&mut self) { + *self = Self::default(); + } +} + +/// Reads the three TLS material files from disk and returns a fingerprint +/// computed from their SHA256 digests. Empty paths produce `None` digests. +pub async fn build_target_tls_fingerprint( + ca_path: &str, + client_cert_path: &str, + client_key_path: &str, +) -> Result { + async fn load_optional_digest(path: &str) -> Result, TargetError> { + if path.is_empty() { + return Ok(None); + } + + let bytes = tokio::fs::read(path) + .await + .map_err(|e| TargetError::Configuration(format!("Failed to read TLS material '{path}': {e}")))?; + let digest = rustfs_tls_runtime::TlsFingerprint::from_optional_bytes(Some(&bytes), None, None, None, None).server_sha256; + Ok(digest) + } + + Ok(TargetTlsFingerprint { + ca_sha256: load_optional_digest(ca_path).await?, + client_cert_sha256: load_optional_digest(client_cert_path).await?, + client_key_sha256: load_optional_digest(client_key_path).await?, + }) +} + +#[cfg(test)] +mod tests { + use super::{TargetTlsFingerprint, TargetTlsGeneration, TargetTlsState}; + + #[test] + fn refresh_increments_generation_only_when_fingerprint_changes() { + let mut state = TargetTlsState::default(); + let first = TargetTlsFingerprint { + ca_sha256: Some([1; 32]), + client_cert_sha256: None, + client_key_sha256: None, + }; + let second = TargetTlsFingerprint { + ca_sha256: Some([2; 32]), + client_cert_sha256: None, + client_key_sha256: None, + }; + + assert!(state.refresh(first.clone())); + assert_eq!(state.generation, TargetTlsGeneration(1)); + assert!(!state.refresh(first)); + assert_eq!(state.generation, TargetTlsGeneration(1)); + assert!(state.refresh(second)); + assert_eq!(state.generation, TargetTlsGeneration(2)); + } + + #[test] + fn reset_clears_generation_and_fingerprint() { + let mut state = TargetTlsState { + generation: TargetTlsGeneration(5), + fingerprint: Some(TargetTlsFingerprint { + ca_sha256: Some([9; 32]), + client_cert_sha256: None, + client_key_sha256: None, + }), + }; + + state.reset(); + assert_eq!(state, TargetTlsState::default()); + } + + #[test] + fn fingerprint_eq_when_all_fields_match() { + let a = TargetTlsFingerprint { + ca_sha256: Some([42; 32]), + client_cert_sha256: Some([1; 32]), + client_key_sha256: None, + }; + let b = TargetTlsFingerprint { + ca_sha256: Some([42; 32]), + client_cert_sha256: Some([1; 32]), + client_key_sha256: None, + }; + assert_eq!(a, b); + } + + #[test] + fn fingerprint_ne_when_ca_differs() { + let a = TargetTlsFingerprint { + ca_sha256: Some([1; 32]), + client_cert_sha256: None, + client_key_sha256: None, + }; + let b = TargetTlsFingerprint { + ca_sha256: Some([2; 32]), + client_cert_sha256: None, + client_key_sha256: None, + }; + assert_ne!(a, b); + } +} diff --git a/crates/targets/src/runtime/tls/metrics.rs b/crates/targets/src/runtime/tls/metrics.rs new file mode 100644 index 000000000..af4d69a72 --- /dev/null +++ b/crates/targets/src/runtime/tls/metrics.rs @@ -0,0 +1,63 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Target-level TLS reload metrics. + +use ::metrics::{counter, describe_counter, describe_gauge, describe_histogram, gauge, histogram}; + +const TARGET_TLS_RELOAD_TOTAL: &str = "rustfs_target_tls_reload_total"; +const TARGET_TLS_RELOAD_SKIPPED_TOTAL: &str = "rustfs_target_tls_reload_skipped_total"; +const TARGET_TLS_GENERATION: &str = "rustfs_target_tls_generation"; +const TARGET_TLS_RELOAD_DURATION_SECONDS: &str = "rustfs_target_tls_reload_duration_seconds"; +const TARGET_TLS_PUBLICATION_FAIL_TOTAL: &str = "rustfs_target_tls_publication_fail_total"; +const TARGET_TLS_ACTIVE_GENERATION_MISMATCH_TOTAL: &str = "rustfs_target_tls_active_generation_mismatch_total"; + +/// Describes all target TLS metrics. Call once during initialization. +pub fn init_target_tls_metrics() { + describe_counter!(TARGET_TLS_RELOAD_TOTAL, "Total number of TLS reload attempts per target"); + describe_counter!( + TARGET_TLS_RELOAD_SKIPPED_TOTAL, + "Number of TLS reloads skipped per target (unchanged, etc.)" + ); + describe_gauge!(TARGET_TLS_GENERATION, "Current TLS generation per target"); + describe_histogram!(TARGET_TLS_RELOAD_DURATION_SECONDS, "Duration of TLS reload attempts per target"); + describe_counter!(TARGET_TLS_PUBLICATION_FAIL_TOTAL, "Number of TLS reload publication failures per target"); + describe_counter!( + TARGET_TLS_ACTIVE_GENERATION_MISMATCH_TOTAL, + "Number of times a target's active connection used a stale TLS generation" + ); +} + +/// Records a TLS reload result (success or failure). +pub fn record_target_tls_reload_result(target: &str, result: &str, duration_secs: f64, generation: u64) { + counter!(TARGET_TLS_RELOAD_TOTAL, "target_id" => target.to_string(), "result" => result.to_string()).increment(1); + histogram!(TARGET_TLS_RELOAD_DURATION_SECONDS, "target_id" => target.to_string(), "result" => result.to_string()) + .record(duration_secs); + gauge!(TARGET_TLS_GENERATION, "target_id" => target.to_string()).set(generation as f64); +} + +/// Records a skipped reload (typically because fingerprint was unchanged). +pub fn record_target_tls_reload_skipped(target: &str, reason: &str) { + counter!(TARGET_TLS_RELOAD_SKIPPED_TOTAL, "target_id" => target.to_string(), "reason" => reason.to_string()).increment(1); +} + +/// Records a TLS reload publication failure. +pub fn record_target_tls_publication_fail(target: &str) { + counter!(TARGET_TLS_PUBLICATION_FAIL_TOTAL, "target_id" => target.to_string()).increment(1); +} + +/// Records that a target used a stale TLS generation (active ≠ latest published). +pub fn record_target_tls_stale_generation(target: &str) { + counter!(TARGET_TLS_ACTIVE_GENERATION_MISMATCH_TOTAL, "target_id" => target.to_string()).increment(1); +} diff --git a/crates/targets/src/runtime/tls/mod.rs b/crates/targets/src/runtime/tls/mod.rs new file mode 100644 index 000000000..be4e0dcf2 --- /dev/null +++ b/crates/targets/src/runtime/tls/mod.rs @@ -0,0 +1,41 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Unified TLS hot-reload infrastructure for notification targets. +//! +//! This module provides: +//! - Fingerprint-based change detection (`fingerprint`) +//! - Per-target reload configuration (`config`) +//! - Runtime state tracking with atomic timestamps (`state`) +//! - The `ReloadableTargetTls` trait protocol (`trait`) +//! - TLS material validation helpers (`validate`) +//! - The reload coordinator with background poll loops (`coordinator`) +//! - Target-level reload metrics (`metrics`) + +pub mod adapter; +pub mod config; +pub mod coordinator; +pub mod fingerprint; +pub mod metrics; +pub mod state; +pub mod r#trait; +pub mod validate; + +pub use adapter::TlsReloadAdapter; +pub use coordinator::TargetTlsReloadCoordinator; +pub use fingerprint::{TargetTlsFingerprint, TargetTlsGeneration, TargetTlsState, build_target_tls_fingerprint}; +pub use metrics::init_target_tls_metrics; +pub use state::{TargetTlsInputSet, TargetTlsPublishedState, TargetTlsRuntimeState, TargetTlsStatusSnapshot}; +pub use r#trait::ReloadableTargetTls; +pub use validate::validate_tls_material; diff --git a/crates/targets/src/runtime/tls/state.rs b/crates/targets/src/runtime/tls/state.rs new file mode 100644 index 000000000..e0906d029 --- /dev/null +++ b/crates/targets/src/runtime/tls/state.rs @@ -0,0 +1,127 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Per-target TLS reload runtime state with atomic timestamps and error tracking. + +use super::fingerprint::{TargetTlsFingerprint, TargetTlsGeneration}; +use ::arc_swap::ArcSwap; +use serde::Serialize; +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; + +/// Describes which TLS files a target reads. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TargetTlsInputSet { + pub ca_path: String, + pub client_cert_path: String, + pub client_key_path: String, + /// Human-readable label for logging and metrics (e.g. "webhook:primary"). + pub target_label: String, +} + +impl TargetTlsInputSet { + /// Returns `true` when no TLS paths are configured (no CA, cert, or key). + pub fn is_empty(&self) -> bool { + self.ca_path.is_empty() && self.client_cert_path.is_empty() && self.client_key_path.is_empty() + } +} + +/// Immutable snapshot of a successfully published TLS material generation. +pub struct TargetTlsPublishedState { + pub generation: TargetTlsGeneration, + pub fingerprint: TargetTlsFingerprint, + pub material: Arc, + pub loaded_at_unix_ms: u64, +} + +/// Per-target TLS reload runtime state. Owns the current published material +/// and tracks timestamps and the last error for observability. +pub struct TargetTlsRuntimeState { + /// The currently active TLS material generation. + pub current: ArcSwap>, + /// The last known-good generation (never overwritten by a failed reload). + pub last_good: ArcSwap>, + /// Unix-millis timestamp of the last reload *attempt* (success or failure). + pub last_attempt_unix_ms: AtomicU64, + /// Unix-millis timestamp of the last *successful* reload. + pub last_success_unix_ms: AtomicU64, + /// Last reload error message, if any. + pub last_error: parking_lot::RwLock>, + /// The TLS file paths this state watches. + pub inputs: TargetTlsInputSet, +} + +impl TargetTlsRuntimeState { + /// Creates a new runtime state with the given initial published state. + pub fn new(initial: Arc>, inputs: TargetTlsInputSet) -> Self { + Self { + current: ArcSwap::from(initial.clone()), + last_good: ArcSwap::from(initial), + last_attempt_unix_ms: AtomicU64::new(0), + last_success_unix_ms: AtomicU64::new(0), + last_error: parking_lot::RwLock::new(None), + inputs, + } + } + + /// Returns the generation of the currently active material. + pub fn current_generation(&self) -> TargetTlsGeneration { + self.current.load().generation + } + + /// Atomically bumps and returns the next generation. + pub fn bump_generation(&self) -> TargetTlsGeneration { + // Load the current generation from the arc-swap, compute next, + // and return it. The caller is responsible for publishing the new state. + let current = self.current.load(); + TargetTlsGeneration(current.generation.0.saturating_add(1)) + } + + /// Records the timestamp of a reload attempt. + pub fn mark_attempt(&self, unix_ms: u64) { + self.last_attempt_unix_ms.store(unix_ms, Ordering::Release); + } + + /// Records the timestamp of a successful reload. + pub fn mark_success(&self, unix_ms: u64) { + self.last_success_unix_ms.store(unix_ms, Ordering::Release); + } + + /// Returns the last attempt timestamp. + pub fn last_attempt_unix_ms(&self) -> u64 { + self.last_attempt_unix_ms.load(Ordering::Acquire) + } + + /// Returns the last success timestamp. + pub fn last_success_unix_ms(&self) -> u64 { + self.last_success_unix_ms.load(Ordering::Acquire) + } +} + +/// Read-only status snapshot for admin/debug visibility. +#[derive(Debug, Clone, Serialize)] +pub struct TargetTlsStatusSnapshot { + pub target_label: String, + pub generation: u64, + pub reload_enabled: bool, + pub detect_mode: &'static str, + pub apply_mode: &'static str, + pub last_attempt_time: Option, + pub last_success_time: Option, + pub last_error: Option, + /// TLS file paths this target watches (for admin diagnostics). + pub ca_path: String, + pub client_cert_path: String, + pub client_key_path: String, +} diff --git a/crates/targets/src/runtime/tls/trait.rs b/crates/targets/src/runtime/tls/trait.rs new file mode 100644 index 000000000..c67e21b62 --- /dev/null +++ b/crates/targets/src/runtime/tls/trait.rs @@ -0,0 +1,68 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! The `ReloadableTargetTls` trait — the public protocol each TLS-capable +//! target implements to participate in coordinated hot-reload. + +use crate::error::TargetError; +use async_trait::async_trait; +use std::sync::Arc; + +use super::config::ReloadApplyMode; +use super::fingerprint::TargetTlsGeneration; +use super::state::TargetTlsInputSet; + +/// Protocol that each TLS-capable target implements so the reload coordinator +/// can drive certificate hot-reload without knowing the target's internals. +/// +/// The target is responsible for: +/// - Declaring which TLS files it reads (`tls_input_set`) +/// - Building a new client/pool/connector from current files (`build_tls_material`) +/// - Atomically swapping the active connection state (`apply_tls_material`) +/// +/// The coordinator is responsible for: +/// - Deciding *when* to check +/// - Detecting *whether* material changed +/// - Ensuring *safety* (validate, build-then-apply, fallback on failure) +#[async_trait] +pub trait ReloadableTargetTls: Send + Sync + 'static { + /// The rebuilt connection/client/pool object this target uses. + type Material: Send + Sync + 'static; + + /// Returns the TLS file paths this target reads. + fn tls_input_set(&self) -> TargetTlsInputSet; + + /// Build a fresh TLS material object from current files on disk. + /// + /// Called by the coordinator on the reload path only — never on the send hot path. + async fn build_tls_material(&self) -> Result; + + /// Atomically apply new TLS material, replacing the current active connection state. + /// + /// On success, the target's internal state must point to the new material. + /// On failure, the target must keep its current state unchanged. + async fn apply_tls_material( + &self, + generation: TargetTlsGeneration, + material: Arc, + mode: ReloadApplyMode, + ) -> Result<(), TargetError>; + + /// Optional pre-check: validate that TLS files on disk are self-consistent + /// (cert/key pair parseable, CA loadable) before attempting `build_tls_material`. + /// Default implementation returns `Ok(())`. + async fn validate_tls_files(&self) -> Result<(), TargetError> { + Ok(()) + } +} diff --git a/crates/targets/src/runtime/tls/validate.rs b/crates/targets/src/runtime/tls/validate.rs new file mode 100644 index 000000000..bb7148aac --- /dev/null +++ b/crates/targets/src/runtime/tls/validate.rs @@ -0,0 +1,58 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! TLS material validation helpers used by the reload coordinator before +//! attempting to build new client/pool objects. + +use crate::error::TargetError; +use rustfs_tls_runtime::{load_certs, load_private_key}; + +/// Validates that a client certificate and private key file can be loaded +/// and paired together. Returns `Ok(())` if both files parse successfully, +/// or `Ok(())` if both paths are empty (no mTLS configured). +pub fn validate_cert_key_pairing(cert_path: &str, key_path: &str) -> Result<(), TargetError> { + if cert_path.is_empty() && key_path.is_empty() { + return Ok(()); + } + + if cert_path.is_empty() || key_path.is_empty() { + return Err(TargetError::Configuration( + "Client certificate and key must both be specified or both be empty".to_string(), + )); + } + + load_certs(cert_path).map_err(|e| TargetError::Configuration(format!("Invalid client certificate '{cert_path}': {e}")))?; + + load_private_key(key_path).map_err(|e| TargetError::Configuration(format!("Invalid client key '{key_path}': {e}")))?; + + Ok(()) +} + +/// Validates that a CA certificate file can be loaded. Returns `Ok(())` +/// if the path is empty (no custom CA) or if the file parses successfully. +pub fn validate_ca_file(ca_path: &str) -> Result<(), TargetError> { + if ca_path.is_empty() { + return Ok(()); + } + + load_certs(ca_path).map_err(|e| TargetError::Configuration(format!("Invalid CA certificate '{ca_path}': {e}")))?; + + Ok(()) +} + +/// Validates all three TLS material files in one call. +pub fn validate_tls_material(ca_path: &str, cert_path: &str, key_path: &str) -> Result<(), TargetError> { + validate_ca_file(ca_path)?; + validate_cert_key_pairing(cert_path, key_path) +} diff --git a/crates/targets/src/target/amqp.rs b/crates/targets/src/target/amqp.rs index 0c9c01627..0fd56fa9a 100644 --- a/crates/targets/src/target/amqp.rs +++ b/crates/targets/src/target/amqp.rs @@ -22,11 +22,15 @@ use crate::{ StoreError, Target, arn::TargetID, error::TargetError, + runtime::tls::{ + ReloadableTargetTls, TargetTlsGeneration, TargetTlsInputSet, TlsReloadAdapter, config::ReloadApplyMode, + validate_tls_material, + }, store::{Key, Store}, target::{ ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot, - TargetType, build_queued_payload_with_records, is_connectivity_error, open_target_queue_store, - persist_queued_payload_to_store, + TargetTlsState, TargetType, build_queued_payload_with_records, build_target_tls_fingerprint, is_connectivity_error, + open_target_queue_store, persist_queued_payload_to_store, }, }; use async_trait::async_trait; @@ -37,6 +41,7 @@ use lapin::{ }; use parking_lot::Mutex; use rustfs_config::{AMQP_TLS_CA, AMQP_TLS_CLIENT_CERT, AMQP_TLS_CLIENT_KEY}; +use rustfs_tls_runtime::load_cert_bundle_der_bytes; use serde::Serialize; use serde::de::DeserializeOwned; use std::fmt; @@ -196,16 +201,24 @@ async fn build_tls_config(args: &AMQPArgs) -> Result>>>, + tls_state: Arc>, + /// When set, the coordinator drives TLS reload; inline fingerprint check is skipped. + tls_adapter: Option>, connect_lock: Arc>, store: Option + Send + Sync>>, delivery_counters: Arc, @@ -305,6 +321,8 @@ where id: self.id.clone(), args: self.args.clone(), connection: Arc::clone(&self.connection), + tls_state: Arc::clone(&self.tls_state), + tls_adapter: self.tls_adapter.clone(), connect_lock: Arc::clone(&self.connect_lock), store: self.store.as_ref().map(|s| s.boxed_clone()), delivery_counters: Arc::clone(&self.delivery_counters), @@ -329,6 +347,8 @@ where id: target_id, args, connection: Arc::new(Mutex::new(None)), + tls_state: Arc::new(Mutex::new(TargetTlsState::default())), + tls_adapter: None, connect_lock: Arc::new(AsyncMutex::new(())), store: queue_store, delivery_counters: Arc::new(TargetDeliveryCounters::default()), @@ -341,6 +361,27 @@ where } async fn get_or_connect(&self) -> Result, TargetError> { + // When a TLS reload adapter is attached, it drives connection rebuilds + // in the background. The inline per-send fingerprint check is skipped. + if let Some(adapter) = &self.tls_adapter { + let material = adapter.current_material(); + if material.connection.status().connected() && material.channel.status().connected() { + return Ok(material); + } + self.clear_connection_handle(); + } else { + let next_fingerprint = + build_target_tls_fingerprint(&self.args.tls_ca, &self.args.tls_client_cert, &self.args.tls_client_key).await?; + let tls_changed = { + let tls_state_guard = self.tls_state.lock(); + tls_state_guard.needs_update(&next_fingerprint) + }; + if tls_changed { + self.clear_connection_handle(); + self.tls_state.lock().refresh(next_fingerprint); + } + } + if let Some(connection) = self.connection.lock().clone() && connection.connection.status().connected() && connection.channel.status().connected() @@ -362,10 +403,19 @@ where Ok(connection) } - fn clear_connection(&self) { + fn clear_connection_handle(&self) { *self.connection.lock() = None; } + fn clear_connection_cache(&self) { + self.clear_connection_handle(); + self.tls_state.lock().reset(); + } + + fn clear_connection(&self) { + self.clear_connection_cache(); + } + async fn send_body(&self, body: &[u8]) -> Result<(), TargetError> { let connection = self.get_or_connect().await?; let publish = connection @@ -407,6 +457,46 @@ where } } +/// Coordinated TLS hot-reload implementation for AMQP targets. +/// +/// The coordinator calls these methods on a background poll loop to detect +/// TLS file changes and rebuild the connection without restarting. +#[async_trait] +impl ReloadableTargetTls for AMQPTarget +where + E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned, +{ + type Material = AMQPConnection; + + fn tls_input_set(&self) -> TargetTlsInputSet { + TargetTlsInputSet { + ca_path: self.args.tls_ca.clone(), + client_cert_path: self.args.tls_client_cert.clone(), + client_key_path: self.args.tls_client_key.clone(), + target_label: format!("amqp:{}", self.id.id), + } + } + + async fn build_tls_material(&self) -> Result { + connect_amqp(&self.args).await + } + + async fn apply_tls_material( + &self, + _generation: TargetTlsGeneration, + material: Arc, + _mode: ReloadApplyMode, + ) -> Result<(), TargetError> { + let mut guard = self.connection.lock(); + *guard = Some(material); + Ok(()) + } + + async fn validate_tls_files(&self) -> Result<(), TargetError> { + validate_tls_material(&self.args.tls_ca, &self.args.tls_client_cert, &self.args.tls_client_key) + } +} + #[async_trait] impl Target for AMQPTarget where @@ -458,6 +548,12 @@ where .await .map_err(|e| map_lapin_error(e, "Failed to close AMQP connection"))?; } + self.tls_state.lock().reset(); + // If a TLS reload adapter is attached, reset its error tracking + // so that a future re-init does not inherit stale failure state. + if let Some(adapter) = &self.tls_adapter { + *adapter.runtime_state().last_error.write() = None; + } info!(target_id = %self.id, "AMQP target closed"); Ok(()) } diff --git a/crates/targets/src/target/kafka.rs b/crates/targets/src/target/kafka.rs index 6a6afba0a..dcd3521c4 100644 --- a/crates/targets/src/target/kafka.rs +++ b/crates/targets/src/target/kafka.rs @@ -16,16 +16,21 @@ use crate::{ StoreError, Target, arn::TargetID, error::TargetError, + runtime::tls::{ + ReloadableTargetTls, TargetTlsInputSet, TlsReloadAdapter, config::ReloadApplyMode, fingerprint::TargetTlsGeneration, + validate::validate_tls_material, + }, store::{Key, Store}, target::{ ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot, - TargetType, build_queued_payload, invalidate_cache_on_connectivity_error, open_target_queue_store, - persist_queued_payload_to_store, + TargetTlsState, TargetType, build_queued_payload, build_target_tls_fingerprint, invalidate_cache_on_connectivity_error, + open_target_queue_store, persist_queued_payload_to_store, }, }; use async_trait::async_trait; use rustfs_kafka_async::error::{ConnectionError, Error as KafkaError}; use rustfs_kafka_async::{AsyncProducer, AsyncProducerConfig, Record, RequiredAcks, SecurityConfig}; +use rustfs_tls_runtime::{load_cert_bundle_der_bytes, load_private_key}; use serde::Serialize; use serde::de::DeserializeOwned; use std::{marker::PhantomData, sync::Arc, time::Duration}; @@ -104,6 +109,11 @@ where args: KafkaArgs, store: Option + Send + Sync>>, producer: Arc>>>, + tls_state: Arc>, + /// Adapter that bridges this target to the TLS reload coordinator. + /// When `Some`, the target uses coordinator-managed material; when `None`, + /// it falls back to inline fingerprint-based change detection. + tls_adapter: Option>>, delivery_counters: Arc, _phantom: PhantomData, } @@ -144,6 +154,8 @@ where args, store: queue_store, producer: Arc::new(Mutex::new(None)), + tls_state: Arc::new(Mutex::new(TargetTlsState::default())), + tls_adapter: None, delivery_counters: Arc::new(TargetDeliveryCounters::default()), _phantom: PhantomData, }) @@ -164,9 +176,27 @@ where if self.args.tls_enable { let mut security = SecurityConfig::new(); if !self.args.tls_ca.is_empty() { + let certs = load_cert_bundle_der_bytes(&self.args.tls_ca) + .map_err(|e| Self::map_kafka_error(KafkaError::Config(e.to_string()), "Failed to parse Kafka tls_ca"))?; + if certs.is_empty() { + return Err(TargetError::Configuration( + "Kafka tls_ca did not contain any parsable certificates".to_string(), + )); + } security = security.with_ca_cert(self.args.tls_ca.clone()); } if !self.args.tls_client_cert.is_empty() && !self.args.tls_client_key.is_empty() { + let certs = load_cert_bundle_der_bytes(&self.args.tls_client_cert).map_err(|e| { + Self::map_kafka_error(KafkaError::Config(e.to_string()), "Failed to parse Kafka tls_client_cert") + })?; + if certs.is_empty() { + return Err(TargetError::Configuration( + "Kafka tls_client_cert did not contain any parsable certificates".to_string(), + )); + } + let _ = load_private_key(&self.args.tls_client_key).map_err(|e| { + Self::map_kafka_error(KafkaError::Config(e.to_string()), "Failed to parse Kafka tls_client_key") + })?; security = security.with_client_cert(self.args.tls_client_cert.clone(), self.args.tls_client_key.clone()); } config = config.with_security(security); @@ -178,6 +208,31 @@ where } async fn get_or_build_producer(&self) -> Result, TargetError> { + // Adapter-managed path: use the material directly from the TLS reload adapter. + if let Some(adapter) = &self.tls_adapter { + let producer: Arc = (*adapter.current_material()).clone(); + + // Ensure the producer is also stored locally so that close() can drain it. + { + let mut guard = self.producer.lock().await; + *guard = Some(Arc::clone(&producer)); + } + return Ok(producer); + } + + // Inline fingerprint fallback path (no coordinator). + let next_fingerprint = + build_target_tls_fingerprint(&self.args.tls_ca, &self.args.tls_client_cert, &self.args.tls_client_key).await?; + let tls_changed = { + let tls_state_guard = self.tls_state.lock().await; + tls_state_guard.needs_update(&next_fingerprint) + }; + if tls_changed { + let mut cached = self.producer.lock().await; + *cached = None; + self.tls_state.lock().await.refresh(next_fingerprint); + } + let mut cached = self.producer.lock().await; if let Some(producer) = cached.as_ref() { return Ok(Arc::clone(producer)); @@ -191,6 +246,7 @@ where async fn invalidate_cached_producer(&self) { let mut cached = self.producer.lock().await; *cached = None; + self.tls_state.lock().await.reset(); } /// Serializes the event and builds a QueuedPayload @@ -230,6 +286,8 @@ where args: self.args.clone(), store: self.store.as_ref().map(|s| s.boxed_clone()), producer: Arc::clone(&self.producer), + tls_state: Arc::clone(&self.tls_state), + tls_adapter: self.tls_adapter.clone(), delivery_counters: Arc::clone(&self.delivery_counters), _phantom: PhantomData, }) @@ -292,6 +350,13 @@ where } async fn close(&self) -> Result<(), TargetError> { + { + let mut guard = self.producer.lock().await; + *guard = None; + } + + self.tls_state.lock().await.reset(); + info!("Kafka target closed: {}", self.id); Ok(()) } @@ -318,6 +383,47 @@ where } } +/// Coordinated TLS hot-reload implementation for Kafka targets. +/// +/// The coordinator calls these methods on a background poll loop to detect +/// TLS file changes and rebuild the producer without restarting. +#[async_trait] +impl ReloadableTargetTls for KafkaTarget +where + E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned, +{ + type Material = Arc; + + fn tls_input_set(&self) -> TargetTlsInputSet { + TargetTlsInputSet { + ca_path: self.args.tls_ca.clone(), + client_cert_path: self.args.tls_client_cert.clone(), + client_key_path: self.args.tls_client_key.clone(), + target_label: format!("kafka:{}", self.id.id), + } + } + + async fn build_tls_material(&self) -> Result { + let producer = self.build_producer().await?; + Ok(Arc::new(producer)) + } + + async fn apply_tls_material( + &self, + _generation: TargetTlsGeneration, + material: Arc, + _mode: ReloadApplyMode, + ) -> Result<(), TargetError> { + let mut guard = self.producer.lock().await; + *guard = Some((*material).clone()); + Ok(()) + } + + async fn validate_tls_files(&self) -> Result<(), TargetError> { + validate_tls_material(&self.args.tls_ca, &self.args.tls_client_cert, &self.args.tls_client_key) + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/targets/src/target/mod.rs b/crates/targets/src/target/mod.rs index f4a51ac46..a29e5673e 100644 --- a/crates/targets/src/target/mod.rs +++ b/crates/targets/src/target/mod.rs @@ -37,6 +37,13 @@ pub mod pulsar; pub mod redis; pub mod webhook; +#[cfg(test)] +pub(crate) use crate::runtime::tls::fingerprint::TargetTlsFingerprint as TargetTlsFingerprintState; +#[cfg(test)] +pub(crate) use crate::runtime::tls::fingerprint::TargetTlsGeneration; +pub(crate) use crate::runtime::tls::fingerprint::TargetTlsState; +pub(crate) use crate::runtime::tls::fingerprint::build_target_tls_fingerprint; + /// A read-only snapshot of delivery counters for a target. #[derive(Debug, Clone, Default, PartialEq, Eq)] pub struct TargetDeliverySnapshot { @@ -531,6 +538,48 @@ pub(crate) fn ensure_rustls_provider_installed() { } } +#[cfg(test)] +mod tls_state_tests { + use super::{TargetTlsFingerprintState, TargetTlsGeneration, TargetTlsState}; + + #[test] + fn refresh_increments_generation_only_when_fingerprint_changes() { + let mut state = TargetTlsState::default(); + let first = TargetTlsFingerprintState { + ca_sha256: Some([1; 32]), + client_cert_sha256: None, + client_key_sha256: None, + }; + let second = TargetTlsFingerprintState { + ca_sha256: Some([2; 32]), + client_cert_sha256: None, + client_key_sha256: None, + }; + + assert!(state.refresh(first.clone())); + assert_eq!(state.generation, TargetTlsGeneration(1)); + assert!(!state.refresh(first)); + assert_eq!(state.generation, TargetTlsGeneration(1)); + assert!(state.refresh(second)); + assert_eq!(state.generation, TargetTlsGeneration(2)); + } + + #[test] + fn reset_clears_generation_and_fingerprint() { + let mut state = TargetTlsState { + generation: TargetTlsGeneration(5), + fingerprint: Some(TargetTlsFingerprintState { + ca_sha256: Some([9; 32]), + client_cert_sha256: None, + client_key_sha256: None, + }), + }; + + state.reset(); + assert_eq!(state, TargetTlsState::default()); + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/targets/src/target/mqtt.rs b/crates/targets/src/target/mqtt.rs index 38cf5ea62..9c366c351 100644 --- a/crates/targets/src/target/mqtt.rs +++ b/crates/targets/src/target/mqtt.rs @@ -16,6 +16,10 @@ use crate::{ StoreError, Target, arn::TargetID, error::TargetError, + runtime::tls::{ + ReloadableTargetTls, TargetTlsGeneration, TargetTlsInputSet, TargetTlsState, TlsReloadAdapter, config::ReloadApplyMode, + validate_tls_material, + }, store::{Key, Store}, target::{ ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot, @@ -23,6 +27,7 @@ use crate::{ persist_queued_payload_to_store, }, }; +use arc_swap::ArcSwap; use async_trait::async_trait; use hyper_rustls::ConfigBuilderExt; use rumqttc::{ @@ -32,6 +37,7 @@ use rumqttc::{ use rustfs_config::{ EnableState, MQTT_TLS_CA, MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY, MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_WS_PATH_ALLOWLIST, }; +use rustfs_tls_runtime::{load_certs, load_private_key}; use rustls::ClientConfig; use serde::Serialize; use serde::de::DeserializeOwned; @@ -185,8 +191,7 @@ fn validate_path_is_absolute(path: &str, field: &str) -> Result<(), TargetError> } fn build_root_store(ca_path: &str, trust_leaf_as_ca: bool) -> Result { - let certs = - rustfs_utils::load_certs(ca_path).map_err(|e| TargetError::Configuration(format!("Failed to load MQTT tls_ca: {e}")))?; + let certs = load_certs(ca_path).map_err(|e| TargetError::Configuration(format!("Failed to load MQTT tls_ca: {e}")))?; let mut store = rustls::RootCertStore::empty(); if trust_leaf_as_ca { @@ -222,9 +227,9 @@ fn build_mqtt_tls_transport(broker: &Url, tls: &MQTTTlsConfig) -> Result Result + Send + Sync>>, connected: Arc, bg_task_manager: Arc, + /// TLS fingerprint tracking for inline fallback path. + tls_state: Arc>, + /// When set, the coordinator drives TLS reload; inline fingerprint check is skipped. + tls_adapter: Option>, + /// Updated MqttOptions from coordinator for use on next reconnection. + pending_mqtt_options: Arc>, delivery_counters: Arc, _phantom: PhantomData, } @@ -519,6 +530,17 @@ where initial_cancel_rx: Mutex::new(Some(cancel_rx)), }); + // Build the initial MqttOptions for TLS reload support. + let initial_mqtt_options = build_mqtt_options( + format!("rustfs_notify_{}", uuid::Uuid::new_v4()), + &args.broker, + Some(args.username.as_str()), + Some(args.password.as_str()), + &args.tls, + args.keep_alive, + Some(MAX_MQTT_PACKET_SIZE_BYTES), + )?; + info!(target_id = %target_id, "MQTT target created"); Ok(MQTTTarget:: { id: target_id, @@ -527,6 +549,9 @@ where store: queue_store, connected: Arc::new(AtomicBool::new(false)), bg_task_manager, + tls_state: Arc::new(parking_lot::Mutex::new(TargetTlsState::default())), + tls_adapter: None, + pending_mqtt_options: Arc::new(ArcSwap::from(Arc::new(initial_mqtt_options))), delivery_counters: Arc::new(TargetDeliveryCounters::default()), _phantom: PhantomData, }) @@ -544,20 +569,15 @@ where let connected_arc = Arc::clone(&self.connected); let target_id_clone = self.id.clone(); let args_clone = self.args.clone(); + let pending_mqtt_options = Arc::clone(&self.pending_mqtt_options); let _ = bg_task_manager .init_cell .get_or_try_init(|| async { debug!(target_id = %target_id_clone, "Initializing MQTT background task."); - let mqtt_options = build_mqtt_options( - format!("rustfs_notify_{}", uuid::Uuid::new_v4()), - &args_clone.broker, - Some(args_clone.username.as_str()), - Some(args_clone.password.as_str()), - &args_clone.tls, - args_clone.keep_alive, - Some(MAX_MQTT_PACKET_SIZE_BYTES), - )?; + + // Use the latest MqttOptions (may have been updated by TLS reload coordinator). + let mqtt_options: MqttOptions = (**pending_mqtt_options.load()).clone(); let (new_client, eventloop) = AsyncClient::builder(mqtt_options).capacity(10).build(); @@ -662,12 +682,66 @@ where store: self.store.as_ref().map(|s| s.boxed_clone()), connected: self.connected.clone(), bg_task_manager: self.bg_task_manager.clone(), + tls_state: Arc::clone(&self.tls_state), + tls_adapter: self.tls_adapter.clone(), + pending_mqtt_options: Arc::clone(&self.pending_mqtt_options), delivery_counters: self.delivery_counters.clone(), _phantom: PhantomData, }) } } +/// Coordinated TLS hot-reload implementation for MQTT targets. +/// +/// MQTT uses `MqttOptions` as the material type. The coordinator rebuilds +/// `MqttOptions` on TLS file changes, and `apply_tls_material` stores it in +/// an `ArcSwap` for use on the next reconnection. The running event loop is +/// not interrupted; rumqttc handles reconnection internally. +#[async_trait] +impl ReloadableTargetTls for MQTTTarget +where + E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned, +{ + type Material = MqttOptions; + + fn tls_input_set(&self) -> TargetTlsInputSet { + TargetTlsInputSet { + ca_path: self.args.tls.ca_path.clone(), + client_cert_path: self.args.tls.client_cert_path.clone(), + client_key_path: self.args.tls.client_key_path.clone(), + target_label: format!("mqtt:{}", self.id.id), + } + } + + async fn build_tls_material(&self) -> Result { + build_mqtt_options( + format!("rustfs_notify_{}", uuid::Uuid::new_v4()), + &self.args.broker, + Some(self.args.username.as_str()), + Some(self.args.password.as_str()), + &self.args.tls, + self.args.keep_alive, + Some(MAX_MQTT_PACKET_SIZE_BYTES), + ) + } + + async fn apply_tls_material( + &self, + _generation: TargetTlsGeneration, + material: Arc, + _mode: ReloadApplyMode, + ) -> Result<(), TargetError> { + // Store the new MqttOptions for use on next reconnection. + // The running event loop is not interrupted; rumqttc handles reconnection. + self.pending_mqtt_options.store(material); + Ok(()) + } + + async fn validate_tls_files(&self) -> Result<(), TargetError> { + validate_tls_material(&self.args.tls.ca_path, &self.args.tls.client_cert_path, &self.args.tls.client_key_path) + } +} + async fn run_mqtt_event_loop( mut eventloop: EventLoop, connected_status: Arc, @@ -968,6 +1042,13 @@ where } } + self.tls_state.lock().reset(); + // If a TLS reload adapter is attached, reset its error tracking + // so that a future re-init does not inherit stale failure state. + if let Some(adapter) = &self.tls_adapter { + *adapter.runtime_state().last_error.write() = None; + } + self.connected.store(false, Ordering::SeqCst); info!(target_id = %self.id, "MQTT target close method finished."); Ok(()) diff --git a/crates/targets/src/target/mysql.rs b/crates/targets/src/target/mysql.rs index ed2116988..9afcd22d1 100644 --- a/crates/targets/src/target/mysql.rs +++ b/crates/targets/src/target/mysql.rs @@ -16,6 +16,10 @@ use crate::{ StoreError, Target, arn::TargetID, error::TargetError, + runtime::tls::{ + ReloadableTargetTls, TargetTlsInputSet, TlsReloadAdapter, config::ReloadApplyMode, fingerprint::TargetTlsGeneration, + validate::validate_tls_material, + }, store::{Key, Store}, target::{ ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot, @@ -26,6 +30,7 @@ use crate::{ use async_trait::async_trait; use mysql_async::{Conn, Opts, OptsBuilder, Pool, PoolConstraints, PoolOpts, SslOpts, prelude::Queryable}; use rustfs_config::{MYSQL_TLS_CA, MYSQL_TLS_CLIENT_CERT, MYSQL_TLS_CLIENT_KEY}; +use rustfs_tls_runtime::{load_certs, load_private_key}; use serde::Serialize; use serde::de::DeserializeOwned; use std::marker::PhantomData; @@ -478,6 +483,11 @@ where store: Option + Send + Sync>>, /// Lazily-initialized MySQL connection pool pool: Arc>>, + /// TLS fingerprint tracking for hot reload (inline fallback path) + tls_state: Arc>, + /// When present, the adapter provides coordinator-managed TLS material; + /// otherwise the inline fingerprint path is used as a fallback. + tls_adapter: Option>, /// Success/failure counters exposed via `delivery_snapshot` delivery_counters: Arc, /// Zero-sized marker for the event type `E` @@ -489,6 +499,9 @@ where E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned, { /// Creates a new MySqlTarget. + /// + /// The target starts without a TLS reload coordinator. Use + /// `TlsReloadAdapter::try_register` to opt into coordinated TLS hot-reload. pub fn new(id: String, args: MySqlArgs) -> Result { args.validate()?; @@ -511,6 +524,8 @@ where store: queue_store, // Pool is lazily initialized on first use to avoid unnecessary connections at startup and allow for better error handling pool: Arc::new(Mutex::new(None)), + tls_state: Arc::new(parking_lot::Mutex::new(super::TargetTlsState::default())), + tls_adapter: None, delivery_counters: Arc::new(TargetDeliveryCounters::default()), _phantom: PhantomData, }) @@ -518,6 +533,10 @@ where /// Returns or lazily initializes the MySQL connection pool. /// + /// When `tls_adapter` is present (coordinator-managed), the pool + /// is sourced from the coordinator's published material. + /// Otherwise, the inline fingerprint-based path is used as a fallback. + /// /// # Errors /// /// | Scenario | Error variant | @@ -528,6 +547,31 @@ where /// | Existing table has incompatible schema | `Initialization` | /// | DSN parse failure / invalid config | `Configuration` | async fn get_or_init_pool(&self) -> Result { + // Adapter-managed path: use the material directly from the coordinator. + if let Some(adapter) = &self.tls_adapter { + let pool: Pool = (*adapter.current_material()).clone(); + + // Ensure the pool is also stored locally so that close() can drain it. + { + let mut guard = self.pool.lock().await; + *guard = Some(pool.clone()); + } + return Ok(pool); + } + + // Inline fingerprint fallback path (no coordinator). + let next_fingerprint = + super::build_target_tls_fingerprint(&self.args.tls_ca, &self.args.tls_client_cert, &self.args.tls_client_key).await?; + let tls_changed = { + let tls_state_guard = self.tls_state.lock(); + tls_state_guard.needs_update(&next_fingerprint) + }; + if tls_changed { + let mut guard = self.pool.lock().await; + *guard = None; + self.tls_state.lock().refresh(next_fingerprint); + } + { let guard = self.pool.lock().await; if let Some(pool) = guard.as_ref() { @@ -535,68 +579,7 @@ where } } - let dsn = MySqlDsn::parse(&self.args.dsn_string)?; - - let mut builder = OptsBuilder::default() - .user(Some(dsn.user.clone())) - .pass(Some(dsn.password.clone())) - .ip_or_hostname(dsn.host.clone()) - .tcp_port(dsn.port) - .db_name(Some(dsn.database.clone())); - - if dsn.tls { - super::ensure_rustls_provider_installed(); - let mut ssl_opts = SslOpts::default(); - if !self.args.tls_ca.is_empty() { - ssl_opts = ssl_opts.with_root_certs(vec![PathBuf::from(self.args.tls_ca.clone()).into()]); - } - if !self.args.tls_client_cert.is_empty() && !self.args.tls_client_key.is_empty() { - let identity = mysql_async::ClientIdentity::new( - PathBuf::from(self.args.tls_client_cert.clone()).into(), - PathBuf::from(self.args.tls_client_key.clone()).into(), - ); - ssl_opts = ssl_opts.with_client_identity(Some(identity)); - } - builder = builder.ssl_opts(Some(ssl_opts)); - } else { - warn!( - "MySQL target '{}' is configured without TLS. This is insecure and should not be used in production.", - self.id - ); - } - - // When max_open_connections is 0, no explicit upper bound is set — - // mysql_async uses its default pool constraints (10–100). - if self.args.max_open_connections > 0 { - let constraints = PoolConstraints::new(1, self.args.max_open_connections).ok_or_else(|| { - TargetError::Configuration(format!( - "MySQL max_open_connections must be >= 1, got {}", - self.args.max_open_connections - )) - })?; - builder = builder.pool_opts(PoolOpts::default().with_constraints(constraints)); - } - - let opts = Opts::from(builder); - let pool = Pool::new(opts); - - // Uses a double-check pattern: the mutex guard is only held for - // short reads/writes to the pool cache. All I/O (connecting, - // DDL, schema validation) happens outside the lock so that - // concurrent callers are not blocked by a slow MySQL server. - let mut conn = pool.get_conn().await.map_err(|_| TargetError::NotConnected)?; - - conn.query_drop("SELECT 1").await.map_err(|_| TargetError::NotConnected)?; - - let ddl = format!( - "CREATE TABLE IF NOT EXISTS {} (event_time DATETIME(6) NOT NULL, event_data JSON NOT NULL)", - quote_table_name(&self.args.table)? - ); - conn.query_drop(ddl) - .await - .map_err(|e| TargetError::Initialization(format!("Failed to create MySQL table: {e}")))?; - - validate_existing_schema(&mut conn, &self.args.table).await?; + let pool = build_mysql_pool_from_args(&self.args).await?; // Double-check: another caller may have initialized the pool // while we were doing I/O. @@ -653,12 +636,87 @@ where args: self.args.clone(), store: self.store.as_ref().map(|s| s.boxed_clone()), pool: Arc::clone(&self.pool), + tls_state: Arc::clone(&self.tls_state), + tls_adapter: self.tls_adapter.clone(), delivery_counters: Arc::clone(&self.delivery_counters), _phantom: PhantomData, }) } } +/// Builds a MySQL connection pool from the given args, including TLS setup, +/// DDL table creation, and schema validation. +/// +/// This is a standalone function so it can be called both from +/// `get_or_init_pool` (inline fallback) and from `build_tls_material` +/// (coordinator path). +async fn build_mysql_pool_from_args(args: &MySqlArgs) -> Result { + let dsn = MySqlDsn::parse(&args.dsn_string)?; + + let mut builder = OptsBuilder::default() + .user(Some(dsn.user.clone())) + .pass(Some(dsn.password.clone())) + .ip_or_hostname(dsn.host.clone()) + .tcp_port(dsn.port) + .db_name(Some(dsn.database.clone())); + + if dsn.tls { + super::ensure_rustls_provider_installed(); + let mut ssl_opts = SslOpts::default(); + if !args.tls_ca.is_empty() { + let _ = + load_certs(&args.tls_ca).map_err(|e| TargetError::Configuration(format!("Failed to load MySQL tls_ca: {e}")))?; + ssl_opts = ssl_opts.with_root_certs(vec![PathBuf::from(args.tls_ca.clone()).into()]); + } + if !args.tls_client_cert.is_empty() && !args.tls_client_key.is_empty() { + let _ = load_certs(&args.tls_client_cert) + .map_err(|e| TargetError::Configuration(format!("Failed to load MySQL tls_client_cert: {e}")))?; + let _ = load_private_key(&args.tls_client_key) + .map_err(|e| TargetError::Configuration(format!("Failed to load MySQL tls_client_key: {e}")))?; + let identity = mysql_async::ClientIdentity::new( + PathBuf::from(args.tls_client_cert.clone()).into(), + PathBuf::from(args.tls_client_key.clone()).into(), + ); + ssl_opts = ssl_opts.with_client_identity(Some(identity)); + } + builder = builder.ssl_opts(Some(ssl_opts)); + } else { + warn!("MySQL target is configured without TLS. This is insecure and should not be used in production."); + } + + // When max_open_connections is 0, no explicit upper bound is set — + // mysql_async uses its default pool constraints (10–100). + if args.max_open_connections > 0 { + let constraints = PoolConstraints::new(1, args.max_open_connections).ok_or_else(|| { + TargetError::Configuration(format!("MySQL max_open_connections must be >= 1, got {}", args.max_open_connections)) + })?; + builder = builder.pool_opts(PoolOpts::default().with_constraints(constraints)); + } + + let opts = Opts::from(builder); + let pool = Pool::new(opts); + + // Uses a double-check pattern: the mutex guard is only held for + // short reads/writes to the pool cache. All I/O (connecting, + // DDL, schema validation) happens outside the lock so that + // concurrent callers are not blocked by a slow MySQL server. + let mut conn = pool.get_conn().await.map_err(|_| TargetError::NotConnected)?; + + conn.query_drop("SELECT 1").await.map_err(|_| TargetError::NotConnected)?; + + let ddl = format!( + "CREATE TABLE IF NOT EXISTS {} (event_time DATETIME(6) NOT NULL, event_data JSON NOT NULL)", + quote_table_name(&args.table)? + ); + conn.query_drop(ddl) + .await + .map_err(|e| TargetError::Initialization(format!("Failed to create MySQL table: {e}")))?; + + validate_existing_schema(&mut conn, &args.table).await?; + + Ok(pool) +} + /// Maps a mysql_async error to `TargetError`: /// - `Io`/`Driver` → `NotConnected` (connection lost, fixed-delay retry) /// - `Server(1213|1205|1040)` → `Timeout` (deadlock/lock timeout/too @@ -793,6 +851,8 @@ where .map_err(|err| TargetError::Network(format!("Failed to disconnect MySQL pool: {err}")))?; } + // Adapter cleanup is done by the coordinator; no local state to reset. + info!("MySQL target closed: {}", self.id); Ok(()) } @@ -828,6 +888,46 @@ where } } +/// Coordinated TLS hot-reload implementation for MySQL targets. +/// +/// The coordinator calls these methods on a background poll loop to detect +/// TLS file changes and rebuild the connection pool without restarting. +#[async_trait] +impl ReloadableTargetTls for MySqlTarget +where + E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned, +{ + type Material = Pool; + + fn tls_input_set(&self) -> TargetTlsInputSet { + TargetTlsInputSet { + ca_path: self.args.tls_ca.clone(), + client_cert_path: self.args.tls_client_cert.clone(), + client_key_path: self.args.tls_client_key.clone(), + target_label: format!("mysql:{}", self.id.id), + } + } + + async fn build_tls_material(&self) -> Result { + build_mysql_pool_from_args(&self.args).await + } + + async fn apply_tls_material( + &self, + _generation: TargetTlsGeneration, + material: Arc, + _mode: ReloadApplyMode, + ) -> Result<(), TargetError> { + let mut guard = self.pool.lock().await; + *guard = Some((*material).clone()); + Ok(()) + } + + async fn validate_tls_files(&self) -> Result<(), TargetError> { + validate_tls_material(&self.args.tls_ca, &self.args.tls_client_cert, &self.args.tls_client_key) + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/targets/src/target/nats.rs b/crates/targets/src/target/nats.rs index 943fb1dff..17bfbf4e7 100644 --- a/crates/targets/src/target/nats.rs +++ b/crates/targets/src/target/nats.rs @@ -16,10 +16,15 @@ use crate::{ StoreError, Target, arn::TargetID, error::TargetError, + runtime::tls::{ + ReloadableTargetTls, TargetTlsGeneration, TargetTlsInputSet, TlsReloadAdapter, config::ReloadApplyMode, + validate_tls_material, + }, store::{Key, Store}, target::{ ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot, - TargetType, build_queued_payload_with_records, open_target_queue_store, persist_queued_payload_to_store, + TargetTlsState, TargetType, build_queued_payload_with_records, build_target_tls_fingerprint, open_target_queue_store, + persist_queued_payload_to_store, }, }; use async_trait::async_trait; @@ -28,9 +33,10 @@ use serde::Serialize; use serde::de::DeserializeOwned; use std::path::{Path, PathBuf}; use std::str::FromStr; +use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::{Arc, Mutex}; -use tracing::{info, instrument}; +use tokio::sync::Mutex; +use tracing::{info, instrument, warn}; #[derive(Debug, Clone)] pub struct NATSArgs { @@ -168,7 +174,12 @@ where { id: TargetID, args: NATSArgs, - client: Mutex>, + client: Arc>>, + tls_state: Arc>, + /// Adapter that bridges this target to the TLS reload coordinator. + /// When `Some`, the target uses coordinator-managed material; when `None`, + /// it falls back to inline fingerprint-based change detection. + tls_adapter: Option>, store: Option + Send + Sync>>, connected: AtomicBool, delivery_counters: Arc, @@ -183,7 +194,9 @@ where Box::new(NATSTarget:: { id: self.id.clone(), args: self.args.clone(), - client: Mutex::new(self.client.lock().unwrap().clone()), + client: Arc::clone(&self.client), + tls_state: Arc::clone(&self.tls_state), + tls_adapter: self.tls_adapter.clone(), store: self.store.as_ref().map(|s| s.boxed_clone()), connected: AtomicBool::new(self.connected.load(Ordering::SeqCst)), delivery_counters: Arc::clone(&self.delivery_counters), @@ -207,7 +220,9 @@ where Ok(Self { id: target_id, args, - client: Mutex::new(None), + client: Arc::new(Mutex::new(None)), + tls_state: Arc::new(parking_lot::Mutex::new(TargetTlsState::default())), + tls_adapter: None, store: queue_store, connected: AtomicBool::new(false), delivery_counters: Arc::new(TargetDeliveryCounters::default()), @@ -215,11 +230,42 @@ where }) } + async fn invalidate_cached_client_connection(&self) { + *self.client.lock().await = None; + } + async fn get_or_connect(&self) -> Result { - if let Some(client) = self.client.lock().unwrap().clone() { + // Adapter-managed path: use the material directly from the TLS reload adapter. + if let Some(adapter) = &self.tls_adapter { + let client: async_nats::Client = (*adapter.current_material()).clone(); + + // Ensure the client is also stored locally so that close() can drain it. + { + let mut guard = self.client.lock().await; + *guard = Some(client.clone()); + } return Ok(client); } + // Inline fingerprint fallback path (no coordinator). + let next_fingerprint = + build_target_tls_fingerprint(&self.args.tls_ca, &self.args.tls_client_cert, &self.args.tls_client_key).await?; + let tls_changed = { + let tls_state_guard = self.tls_state.lock(); + tls_state_guard.needs_update(&next_fingerprint) + }; + if tls_changed { + self.invalidate_cached_client_connection().await; + self.tls_state.lock().refresh(next_fingerprint); + } + + { + let guard = self.client.lock().await; + if let Some(client) = guard.as_ref() { + return Ok(client.clone()); + } + } + let client = connect_nats(&self.args).await?; client .flush() @@ -227,7 +273,7 @@ where .map_err(|e| TargetError::Network(format!("Failed to flush NATS connection: {e}")))?; self.connected.store(true, Ordering::SeqCst); - let mut guard = self.client.lock().unwrap(); + let mut guard = self.client.lock().await; let shared = guard.get_or_insert_with(|| client.clone()).clone(); Ok(shared) } @@ -294,7 +340,11 @@ where } async fn close(&self) -> Result<(), TargetError> { - let client = self.client.lock().unwrap().take(); + let client = { + let mut guard = self.client.lock().await; + guard.take() + }; + self.tls_state.lock().reset(); self.connected.store(false, Ordering::SeqCst); if let Some(client) = client { client @@ -335,3 +385,87 @@ where self.delivery_counters.record_final_failure(); } } + +/// Coordinated TLS hot-reload implementation for NATS targets. +/// +/// The coordinator calls these methods on a background poll loop to detect +/// TLS file changes and rebuild the NATS client without restarting. +#[async_trait] +impl ReloadableTargetTls for NATSTarget +where + E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned, +{ + type Material = async_nats::Client; + + fn tls_input_set(&self) -> TargetTlsInputSet { + TargetTlsInputSet { + ca_path: self.args.tls_ca.clone(), + client_cert_path: self.args.tls_client_cert.clone(), + client_key_path: self.args.tls_client_key.clone(), + target_label: format!("nats:{}", self.id.id), + } + } + + async fn build_tls_material(&self) -> Result { + connect_nats(&self.args).await + } + + async fn apply_tls_material( + &self, + _generation: TargetTlsGeneration, + material: Arc, + _mode: ReloadApplyMode, + ) -> Result<(), TargetError> { + let mut guard = self.client.lock().await; + *guard = Some((*material).clone()); + Ok(()) + } + + async fn validate_tls_files(&self) -> Result<(), TargetError> { + validate_tls_material(&self.args.tls_ca, &self.args.tls_client_cert, &self.args.tls_client_key) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn base_args() -> NATSArgs { + NATSArgs { + enable: true, + address: "nats://127.0.0.1:4222".to_string(), + subject: "rustfs.events".to_string(), + username: String::new(), + password: String::new(), + token: String::new(), + credentials_file: String::new(), + tls_ca: String::new(), + tls_client_cert: String::new(), + tls_client_key: String::new(), + tls_required: false, + queue_dir: String::new(), + queue_limit: 0, + target_type: TargetType::NotifyEvent, + } + } + + #[test] + fn validate_nats_rejects_multiple_auth_methods() { + let args = NATSArgs { + token: "abc".to_string(), + username: "user".to_string(), + password: "pass".to_string(), + ..base_args() + }; + assert!(args.validate().is_err()); + } + + #[test] + fn validate_nats_rejects_relative_queue_dir() { + let args = NATSArgs { + queue_dir: "relative/path".to_string(), + ..base_args() + }; + assert!(args.validate().is_err()); + } +} diff --git a/crates/targets/src/target/postgres.rs b/crates/targets/src/target/postgres.rs index 344a34ba8..42d3a4bc3 100644 --- a/crates/targets/src/target/postgres.rs +++ b/crates/targets/src/target/postgres.rs @@ -29,6 +29,10 @@ use crate::{ StoreError, Target, arn::TargetID, error::TargetError, + runtime::tls::{ + ReloadableTargetTls, TargetTlsGeneration, TargetTlsInputSet, TlsReloadAdapter, config::ReloadApplyMode, + validate_tls_material, + }, store::{Key, Store}, target::{ ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot, @@ -38,12 +42,10 @@ use crate::{ use async_trait::async_trait; use deadpool_postgres::{Manager, ManagerConfig, Pool, RecyclingMethod}; use rustfs_config::{POSTGRES_DSN_STRING, POSTGRES_TLS_CA, POSTGRES_TLS_CLIENT_CERT, POSTGRES_TLS_CLIENT_KEY}; -use rustls_pki_types::pem::PemObject; -use rustls_pki_types::{CertificateDer, PrivateKeyDer}; +use rustfs_tls_runtime::{load_certs, load_private_key}; use serde::Serialize; use serde::de::DeserializeOwned; use std::fmt; -use std::io::BufReader; use std::path::Path; use std::sync::Arc; use tokio_postgres::Config; @@ -425,11 +427,9 @@ pub fn build_tls_config(args: &PostgresArgs) -> Result Result = CertificateDer::pem_reader_iter(&mut BufReader::new(cert_pem.as_slice())) - .collect::>() + let certs = load_certs(&args.tls_client_cert) .map_err(|e| TargetError::Configuration(format!("invalid {POSTGRES_TLS_CLIENT_CERT}: {e}")))?; - - let key = PrivateKeyDer::from_pem_reader(&mut BufReader::new(key_pem.as_slice())) + let key = load_private_key(&args.tls_client_key) .map_err(|e| TargetError::Configuration(format!("invalid {POSTGRES_TLS_CLIENT_KEY}: {e}")))?; builder @@ -548,13 +541,22 @@ fn resolve_payload_key(payload: &serde_json::Value, meta: &QueuedPayloadMeta) -> /// so that `clone_box` does not duplicate connection state. The optional /// `QueueStore` provides at-least-once delivery semantics consistent with the /// other built-in targets. +/// +/// When `tls_adapter` is `Some`, the target participates in the +/// coordinated TLS hot-reload system driven by `TlsReloadAdapter`, +/// and the inline fingerprint check in `send_body` is skipped. When `None`, +/// the legacy inline fingerprint check is used as a fallback. pub struct PostgresTarget where E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned, { id: TargetID, args: PostgresArgs, - pool: Pool, + pool: Arc>, + tls_state: Arc>, + /// When present, the adapter provides coordinator-managed TLS material; + /// otherwise the inline fingerprint path is used as a fallback. + tls_adapter: Option>, namespace_sql: String, access_sql: String, store: Option + Send + Sync>>, @@ -570,7 +572,9 @@ where Box::new(PostgresTarget:: { id: self.id.clone(), args: self.args.clone(), - pool: self.pool.clone(), + pool: Arc::clone(&self.pool), + tls_state: Arc::clone(&self.tls_state), + tls_adapter: self.tls_adapter.clone(), namespace_sql: self.namespace_sql.clone(), access_sql: self.access_sql.clone(), store: self.store.as_ref().map(|s| s.boxed_clone()), @@ -599,7 +603,9 @@ where namespace_sql: namespace_upsert_sql(&args.schema, &args.table), access_sql: access_insert_sql(&args.schema, &args.table), args, - pool, + pool: Arc::new(parking_lot::Mutex::new(pool)), + tls_state: Arc::new(parking_lot::Mutex::new(super::TargetTlsState::default())), + tls_adapter: None, store: queue_store, delivery_counters: Arc::new(TargetDeliveryCounters::default()), _phantom: std::marker::PhantomData, @@ -611,8 +617,25 @@ where /// Identifier validation has already happened in `PostgresArgs::validate()`, /// so `qualified_table` cannot produce a malformed SQL string here. async fn send_body(&self, body: &[u8], event_id: &str, meta: &QueuedPayloadMeta) -> Result<(), TargetError> { - let client = self - .pool + // When a TLS reload adapter is attached, it drives pool rebuilds in + // the background. The inline per-send fingerprint check is skipped. + if self.tls_adapter.is_none() { + let next_fingerprint = + super::build_target_tls_fingerprint(&self.args.tls_ca, &self.args.tls_client_cert, &self.args.tls_client_key) + .await?; + let tls_changed = { + let tls_state_guard = self.tls_state.lock(); + tls_state_guard.fingerprint.as_ref() != Some(&next_fingerprint) + }; + if tls_changed { + let new_pool = build_pool(&self.args)?; + *self.pool.lock() = new_pool; + self.tls_state.lock().refresh(next_fingerprint); + } + } + + let pool = self.pool.lock().clone(); + let client = pool .get() .await .map_err(|e| map_pool_error(e, "PostgreSQL pool checkout failed"))?; @@ -645,8 +668,8 @@ where /// Probes the table from `init()`. Failure is non-fatal when a queue is /// configured: events buffer in the store until the schema is fixed. async fn probe_table(&self) -> Result<(), TargetError> { - let client = self - .pool + let pool = self.pool.lock().clone(); + let client = pool .get() .await .map_err(|e| map_pool_error(e, "PostgreSQL pool checkout failed during init probe"))?; @@ -659,6 +682,41 @@ where } } +#[async_trait] +impl ReloadableTargetTls for PostgresTarget +where + E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned, +{ + type Material = Pool; + + fn tls_input_set(&self) -> TargetTlsInputSet { + TargetTlsInputSet { + ca_path: self.args.tls_ca.clone(), + client_cert_path: self.args.tls_client_cert.clone(), + client_key_path: self.args.tls_client_key.clone(), + target_label: format!("postgres:{}", self.id.id), + } + } + + async fn build_tls_material(&self) -> Result { + build_pool(&self.args) + } + + async fn apply_tls_material( + &self, + _generation: TargetTlsGeneration, + material: Arc, + _mode: ReloadApplyMode, + ) -> Result<(), TargetError> { + *self.pool.lock() = (*material).clone(); + Ok(()) + } + + async fn validate_tls_files(&self) -> Result<(), TargetError> { + validate_tls_material(&self.args.tls_ca, &self.args.tls_client_cert, &self.args.tls_client_key) + } +} + #[async_trait] impl Target for PostgresTarget where @@ -674,8 +732,8 @@ where } match tokio::time::timeout(std::time::Duration::from_secs(10), async { - let client = self - .pool + let pool = self.pool.lock().clone(); + let client = pool .get() .await .map_err(|e| map_pool_error(e, "PostgreSQL pool checkout failed"))?; @@ -728,7 +786,8 @@ where } async fn close(&self) -> Result<(), TargetError> { - self.pool.close(); + self.pool.lock().close(); + // Adapter cleanup is done by the coordinator; no local state to reset. info!(target_id = %self.id, "PostgreSQL target closed"); Ok(()) } diff --git a/crates/targets/src/target/pulsar.rs b/crates/targets/src/target/pulsar.rs index d3d8a9403..6866086ea 100644 --- a/crates/targets/src/target/pulsar.rs +++ b/crates/targets/src/target/pulsar.rs @@ -16,14 +16,20 @@ use crate::{ StoreError, Target, arn::TargetID, error::TargetError, + runtime::tls::{ + ReloadableTargetTls, TargetTlsGeneration, TargetTlsInputSet, TlsReloadAdapter, config::ReloadApplyMode, + validate_tls_material, + }, store::{Key, Store}, target::{ ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot, - TargetType, build_queued_payload_with_records, open_target_queue_store, persist_queued_payload_to_store, + TargetTlsState, TargetType, build_queued_payload_with_records, build_target_tls_fingerprint, open_target_queue_store, + persist_queued_payload_to_store, }, }; use async_trait::async_trait; use pulsar::{Authentication, Producer, Pulsar, TokioExecutor}; +use rustfs_tls_runtime::load_cert_bundle_der_bytes; use serde::Serialize; use serde::de::DeserializeOwned; use std::path::Path; @@ -136,6 +142,13 @@ pub async fn connect_pulsar(args: &PulsarArgs) -> Result, } if !args.tls_ca.is_empty() { + let certs = load_cert_bundle_der_bytes(&args.tls_ca) + .map_err(|e| TargetError::Configuration(format!("Failed to parse Pulsar tls_ca: {e}")))?; + if certs.is_empty() { + return Err(TargetError::Configuration( + "Pulsar tls_ca did not contain any parsable certificates".to_string(), + )); + } builder = builder .with_certificate_chain_file(&args.tls_ca) .map_err(|e| TargetError::Configuration(format!("Failed to load Pulsar tls_ca: {e}")))?; @@ -158,6 +171,9 @@ where id: TargetID, args: PulsarArgs, client: Mutex>>, + tls_state: Mutex, + /// When set, the coordinator drives TLS reload; inline fingerprint check is skipped. + tls_adapter: Option>>, producer: AsyncMutex>>, store: Option + Send + Sync>>, connected: AtomicBool, @@ -174,6 +190,8 @@ where id: self.id.clone(), args: self.args.clone(), client: Mutex::new(self.client.lock().unwrap().clone()), + tls_state: Mutex::new(self.tls_state.lock().unwrap().clone()), + tls_adapter: self.tls_adapter.clone(), producer: AsyncMutex::new(None), store: self.store.as_ref().map(|s| s.boxed_clone()), connected: AtomicBool::new(self.connected.load(Ordering::SeqCst)), @@ -199,6 +217,8 @@ where id: target_id, args, client: Mutex::new(None), + tls_state: Mutex::new(TargetTlsState::default()), + tls_adapter: None, producer: AsyncMutex::new(None), store: queue_store, connected: AtomicBool::new(false), @@ -207,7 +227,36 @@ where }) } + fn clear_cached_client_connection(&self) { + self.client.lock().unwrap().take(); + } + + fn clear_cached_client(&self) { + self.clear_cached_client_connection(); + self.tls_state.lock().unwrap().reset(); + } + async fn get_or_connect_client(&self) -> Result, TargetError> { + // When a TLS reload adapter is attached, it drives client rebuilds + // in the background. The inline per-send fingerprint check is skipped. + if let Some(adapter) = &self.tls_adapter { + let material = adapter.current_material(); + { + let mut guard = self.client.lock().unwrap(); + *guard = Some((*material).clone()); + } + } else { + let next_fingerprint = build_target_tls_fingerprint(&self.args.tls_ca, "", "").await?; + let tls_changed = { + let tls_state_guard = self.tls_state.lock().unwrap(); + tls_state_guard.needs_update(&next_fingerprint) + }; + if tls_changed { + self.clear_cached_client_connection(); + self.tls_state.lock().unwrap().refresh(next_fingerprint); + } + } + if let Some(client) = self.client.lock().unwrap().clone() { return Ok(client); } @@ -262,6 +311,56 @@ where } } +/// Coordinated TLS hot-reload implementation for Pulsar targets. +/// +/// Pulsar only uses a CA certificate (no client cert/key). +/// The coordinator calls these methods on a background poll loop to detect +/// TLS file changes and rebuild the client without restarting. +#[async_trait] +impl ReloadableTargetTls for PulsarTarget +where + E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned, +{ + type Material = Pulsar; + + fn tls_input_set(&self) -> TargetTlsInputSet { + TargetTlsInputSet { + ca_path: self.args.tls_ca.clone(), + client_cert_path: String::new(), + client_key_path: String::new(), + target_label: format!("pulsar:{}", self.id.id), + } + } + + async fn build_tls_material(&self) -> Result { + connect_pulsar(&self.args).await + } + + async fn apply_tls_material( + &self, + _generation: TargetTlsGeneration, + material: Arc, + _mode: ReloadApplyMode, + ) -> Result<(), TargetError> { + // Pulsar client is Clone, so we clone from the Arc and store it. + { + let mut guard = self.client.lock().unwrap(); + *guard = Some((*material).clone()); + } + // Producer is bound to the old client; clear it so next send rebuilds. + { + let mut producer = self.producer.lock().await; + *producer = None; + } + Ok(()) + } + + async fn validate_tls_files(&self) -> Result<(), TargetError> { + // Pulsar only uses CA, no client cert/key. + validate_tls_material(&self.args.tls_ca, "", "") + } +} + #[async_trait] impl Target for PulsarTarget where @@ -321,8 +420,13 @@ where .map_err(|e| TargetError::Network(format!("Failed to close Pulsar producer: {e}")))?; } *producer = None; - self.client.lock().unwrap().take(); + self.clear_cached_client(); self.connected.store(false, Ordering::SeqCst); + // If a TLS reload adapter is attached, reset its error tracking + // so that a future re-init does not inherit stale failure state. + if let Some(adapter) = &self.tls_adapter { + *adapter.runtime_state().last_error.write() = None; + } info!(target_id = %self.id, "Pulsar target closed"); Ok(()) } @@ -355,3 +459,45 @@ where self.delivery_counters.record_final_failure(); } } + +#[cfg(test)] +mod tests { + use super::*; + + fn base_args() -> PulsarArgs { + PulsarArgs { + enable: true, + broker: "pulsar://127.0.0.1:6650".to_string(), + topic: "persistent://public/default/rustfs-events".to_string(), + auth_token: String::new(), + username: String::new(), + password: String::new(), + tls_ca: String::new(), + tls_allow_insecure: false, + tls_hostname_verification: true, + queue_dir: String::new(), + queue_limit: 0, + target_type: TargetType::NotifyEvent, + } + } + + #[test] + fn validate_pulsar_rejects_mixed_auth_methods() { + let args = PulsarArgs { + auth_token: "token".to_string(), + username: "user".to_string(), + password: "pass".to_string(), + ..base_args() + }; + assert!(args.validate().is_err()); + } + + #[test] + fn validate_pulsar_rejects_relative_queue_dir() { + let args = PulsarArgs { + queue_dir: "relative/path".to_string(), + ..base_args() + }; + assert!(args.validate().is_err()); + } +} diff --git a/crates/targets/src/target/redis.rs b/crates/targets/src/target/redis.rs index e749934ca..cc107fdad 100644 --- a/crates/targets/src/target/redis.rs +++ b/crates/targets/src/target/redis.rs @@ -16,6 +16,10 @@ use crate::{ StoreError, Target, arn::TargetID, error::TargetError, + runtime::tls::{ + ReloadableTargetTls, TargetTlsGeneration, TargetTlsInputSet, TlsReloadAdapter, config::ReloadApplyMode, + validate_tls_material, + }, store::{Key, Store}, target::{ ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot, @@ -31,9 +35,12 @@ use redis::{ io::tcp::{TcpSettings, socket2}, }; use rustfs_config::{REDIS_TLS_CA, REDIS_TLS_CLIENT_CERT, REDIS_TLS_CLIENT_KEY, REDIS_TLS_POLICY}; +use rustls::pki_types::CertificateDer; +use rustls::pki_types::pem::PemObject; use serde::Serialize; use serde::de::DeserializeOwned; use std::fmt; +use std::io::BufReader; use std::path::Path; use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; @@ -297,7 +304,8 @@ where { id: TargetID, args: RedisArgs, - publisher_client: Client, + /// Redis client, wrapped in a lock so TLS hot-reload can atomically replace it. + publisher_client: Arc>, publisher: Arc>>, store: Option + Send + Sync>>, /// Business-level liveness flag. @@ -306,6 +314,12 @@ where /// publish exhausted retries, or the target was explicitly closed). Temporary reconnectable /// errors only invalidate the cached publisher so that a later request can lazily rebuild it. connected: Arc, + /// TLS fingerprint tracking for hot reload (inline fallback path). + tls_state: Arc>, + /// Adapter that bridges this target to the TLS reload coordinator. + /// When `Some`, the target uses coordinator-managed material; when `None`, + /// it falls back to inline fingerprint-based change detection. + tls_adapter: Option>, delivery_counters: Arc, _phantom: std::marker::PhantomData, } @@ -334,10 +348,12 @@ where Ok(Self { id: target_id, args, - publisher_client, + publisher_client: Arc::new(parking_lot::Mutex::new(publisher_client)), publisher: Arc::new(Mutex::new(None)), store: queue_store, connected: Arc::new(AtomicBool::new(false)), + tls_state: Arc::new(parking_lot::Mutex::new(super::TargetTlsState::default())), + tls_adapter: None, delivery_counters: Arc::new(TargetDeliveryCounters::default()), _phantom: std::marker::PhantomData, }) @@ -347,23 +363,61 @@ where Box::new(Self { id: self.id.clone(), args: self.args.clone(), - publisher_client: self.publisher_client.clone(), + publisher_client: Arc::clone(&self.publisher_client), publisher: Arc::clone(&self.publisher), store: self.store.as_ref().map(|s| s.boxed_clone()), connected: Arc::clone(&self.connected), + tls_state: Arc::clone(&self.tls_state), + tls_adapter: self.tls_adapter.clone(), delivery_counters: Arc::clone(&self.delivery_counters), _phantom: std::marker::PhantomData, }) } async fn get_or_create_publisher(&self) -> Result { + // Adapter-managed path: use the material directly from the TLS reload adapter. + if let Some(adapter) = &self.tls_adapter { + let client: Client = (*adapter.current_material()).clone(); + + // Ensure the client is also stored locally so close() can drain it. + *self.publisher_client.lock() = client.clone(); + + let manager = client + .get_connection_manager_lazy(build_redis_connection_manager_config(&self.args)) + .map_err(map_redis_error)?; + + *self.publisher.lock().await = Some(manager.clone()); + return Ok(manager); + } + + // Inline fingerprint fallback path (no coordinator). + let secure_scheme = matches!(self.args.url.scheme(), "rediss" | "valkeys"); + if secure_scheme { + let next_fingerprint = super::build_target_tls_fingerprint( + &self.args.tls.ca_path, + &self.args.tls.client_cert_path, + &self.args.tls.client_key_path, + ) + .await?; + let tls_changed = { + let tls_state_guard = self.tls_state.lock(); + tls_state_guard.needs_update(&next_fingerprint) + }; + if tls_changed { + let new_client = build_redis_client(&self.args)?; + *self.publisher_client.lock() = new_client; + self.invalidate_cached_publisher().await; + self.tls_state.lock().refresh(next_fingerprint); + } + } + let mut guard = self.publisher.lock().await; if let Some(manager) = guard.clone() { return Ok(manager); } - let manager = self - .publisher_client + let client = self.publisher_client.lock().clone(); + let manager = client .get_connection_manager_lazy(build_redis_connection_manager_config(&self.args)) .map_err(map_redis_error)?; @@ -475,7 +529,8 @@ where return Ok(false); } - match tokio::time::timeout(Duration::from_secs(5), ping_redis_server(&self.publisher_client, &self.args)).await { + let client = self.publisher_client.lock().clone(); + match tokio::time::timeout(Duration::from_secs(5), ping_redis_server(&client, &self.args)).await { Ok(Ok(())) => { self.connected.store(true, Ordering::SeqCst); Ok(true) @@ -557,6 +612,7 @@ where async fn close(&self) -> Result<(), TargetError> { self.invalidate_cached_publisher().await; + self.tls_state.lock().reset(); self.connected.store(false, Ordering::SeqCst); info!(target_id = %self.id, "Redis target closed"); Ok(()) @@ -696,9 +752,20 @@ fn read_root_cert(tls: &RedisTlsConfig) -> Result>, TargetError> return Ok(None); } - std::fs::read(&tls.ca_path) - .map(Some) - .map_err(|e| TargetError::Configuration(format!("Failed to read Redis root CA cert: {e}"))) + let pem = + std::fs::read(&tls.ca_path).map_err(|e| TargetError::Configuration(format!("Failed to read Redis root CA cert: {e}")))?; + let mut reader = BufReader::new(pem.as_slice()); + let certs_der = CertificateDer::pem_reader_iter(&mut reader) + .collect::, _>>() + .map_err(|e| TargetError::Configuration(format!("Failed to parse Redis root CA cert: {e}")))?; + + if certs_der.is_empty() { + return Err(TargetError::Configuration( + "Redis root CA cert did not contain any parsable certificates".to_string(), + )); + } + + Ok(Some(pem)) } fn map_redis_error(err: RedisError) -> TargetError { @@ -722,6 +789,46 @@ fn compute_retry_delay(attempt: usize, min_delay: Duration, max_delay: Duration) min_delay.saturating_mul(factor).min(max_delay) } +/// Coordinated TLS hot-reload implementation for Redis targets. +/// +/// The coordinator calls these methods on a background poll loop to detect +/// TLS file changes and rebuild the Redis client without restarting. +#[async_trait] +impl ReloadableTargetTls for RedisTarget +where + E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned, +{ + type Material = Client; + + fn tls_input_set(&self) -> TargetTlsInputSet { + TargetTlsInputSet { + ca_path: self.args.tls.ca_path.clone(), + client_cert_path: self.args.tls.client_cert_path.clone(), + client_key_path: self.args.tls.client_key_path.clone(), + target_label: format!("redis:{}", self.id.id), + } + } + + async fn build_tls_material(&self) -> Result { + build_redis_client(&self.args) + } + + async fn apply_tls_material( + &self, + _generation: TargetTlsGeneration, + material: Arc, + _mode: ReloadApplyMode, + ) -> Result<(), TargetError> { + *self.publisher_client.lock() = (*material).clone(); + self.invalidate_cached_publisher().await; + Ok(()) + } + + async fn validate_tls_files(&self) -> Result<(), TargetError> { + validate_tls_material(&self.args.tls.ca_path, &self.args.tls.client_cert_path, &self.args.tls.client_key_path) + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/targets/src/target/webhook.rs b/crates/targets/src/target/webhook.rs index 03cd4e615..5c65cdccd 100644 --- a/crates/targets/src/target/webhook.rs +++ b/crates/targets/src/target/webhook.rs @@ -16,14 +16,21 @@ use crate::{ StoreError, Target, arn::TargetID, error::TargetError, + runtime::tls::{ + ReloadableTargetTls, TargetTlsInputSet, TlsReloadAdapter, config::ReloadApplyMode, fingerprint::TargetTlsGeneration, + validate::validate_tls_material, + }, store::{Key, Store}, target::{ ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot, - TargetType, build_queued_payload, open_target_queue_store, persist_queued_payload_to_store, + TargetTlsState, TargetType, build_queued_payload, build_target_tls_fingerprint, open_target_queue_store, + persist_queued_payload_to_store, }, }; use async_trait::async_trait; +use parking_lot::Mutex; use reqwest::{Client, StatusCode, Url}; +use rustfs_tls_runtime::load_cert_bundle_der_bytes; use serde::Serialize; use serde::de::DeserializeOwned; use std::{ @@ -104,7 +111,11 @@ where id: TargetID, args: WebhookArgs, health_check_url: Option, - http_client: Arc, + http_client: Arc>, + tls_state: Arc>, + /// When present, the adapter provides coordinator-managed TLS material; + /// otherwise the inline fingerprint path is used as a fallback. + tls_adapter: Option>, // Add Send + Sync constraints to ensure thread safety store: Option + Send + Sync>>, initialized: AtomicBool, @@ -124,6 +135,8 @@ where args: self.args.clone(), health_check_url: self.health_check_url.clone(), http_client: Arc::clone(&self.http_client), + tls_state: Arc::clone(&self.tls_state), + tls_adapter: self.tls_adapter.clone(), store: self.store.as_ref().map(|s| s.boxed_clone()), initialized: AtomicBool::new(self.initialized.load(Ordering::SeqCst)), cancel_sender: self.cancel_sender.clone(), @@ -146,7 +159,7 @@ where }; // Build HTTP client using the helper function - let http_client = Arc::new(Self::build_http_client(&args)?); + let http_client = Arc::new(Mutex::new(Self::build_http_client(&args)?)); let queue_store = open_target_queue_store( &args.queue_dir, @@ -165,6 +178,8 @@ where args, health_check_url, http_client, + tls_state: Arc::new(Mutex::new(TargetTlsState::default())), + tls_adapter: None, store: queue_store, initialized: AtomicBool::new(false), cancel_sender, @@ -188,11 +203,18 @@ where ); } else if !args.client_ca.is_empty() { // Use user-provided custom CA certificate - let ca_cert_pem = std::fs::read(&args.client_ca) - .map_err(|e| TargetError::Configuration(format!("Failed to read root CA cert: {e}")))?; - let ca_cert = reqwest::Certificate::from_pem(&ca_cert_pem) + let certs_der = load_cert_bundle_der_bytes(&args.client_ca) .map_err(|e| TargetError::Configuration(format!("Failed to parse root CA cert: {e}")))?; - client_builder = client_builder.add_root_certificate(ca_cert); + if certs_der.is_empty() { + return Err(TargetError::Configuration( + "Webhook client_ca did not contain any parsable certificates".to_string(), + )); + } + for cert_der in certs_der { + let ca_cert = reqwest::Certificate::from_der(&cert_der) + .map_err(|e| TargetError::Configuration(format!("Failed to load root CA cert: {e}")))?; + client_builder = client_builder.add_root_certificate(ca_cert); + } } // If neither is set, use the system's default trust store @@ -213,6 +235,29 @@ where .map_err(|e| TargetError::Configuration(format!("Failed to build HTTP client: {e}"))) } + async fn refresh_tls(&self) -> Result<(), TargetError> { + let next_fingerprint = + build_target_tls_fingerprint(&self.args.client_ca, &self.args.client_cert, &self.args.client_key).await?; + let tls_changed = { + let tls_state_guard = self.tls_state.lock(); + tls_state_guard.fingerprint.as_ref() != Some(&next_fingerprint) + }; + if !tls_changed { + return Ok(()); + } + + let new_client = Self::build_http_client(&self.args)?; + { + let mut tls_state_guard = self.tls_state.lock(); + if tls_state_guard.fingerprint.as_ref() == Some(&next_fingerprint) { + return Ok(()); + } + *self.http_client.lock() = new_client; + tls_state_guard.refresh(next_fingerprint); + } + Ok(()) + } + fn health_check_url(endpoint: &Url) -> Result { endpoint .host() @@ -230,7 +275,8 @@ where return Ok(false); }; - match tokio::time::timeout(Duration::from_secs(5), self.http_client.head(health_check_url.as_str()).send()).await { + let client = self.http_client.lock().clone(); + match tokio::time::timeout(Duration::from_secs(5), client.head(health_check_url.as_str()).send()).await { Ok(Ok(resp)) => { debug!( target = %self.id, @@ -299,8 +345,14 @@ where "Sending webhook payload" ); - let mut req_builder = self - .http_client + // When a TLS reload adapter is attached, it drives client rebuilds in + // the background. The inline per-send fingerprint check is skipped. + if self.tls_adapter.is_none() { + self.refresh_tls().await?; + } + + let client = self.http_client.lock().clone(); + let mut req_builder = client .post(self.args.endpoint.as_str()) .header("Content-Type", meta.content_type.as_str()); @@ -425,6 +477,7 @@ where async fn close(&self) -> Result<(), TargetError> { // Send cancel signal to background tasks let _ = self.cancel_sender.try_send(()); + // Adapter cleanup is done by the coordinator; no local state to reset. info!("Webhook target closed: {}", self.id); Ok(()) } @@ -460,6 +513,48 @@ where } } +/// Coordinated TLS hot-reload implementation for Webhook targets. +/// +/// The coordinator calls these methods on a background poll loop to detect +/// TLS file changes and rebuild the HTTP client without restarting. +#[async_trait] +impl ReloadableTargetTls for WebhookTarget +where + E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned, +{ + type Material = Client; + + fn tls_input_set(&self) -> TargetTlsInputSet { + TargetTlsInputSet { + ca_path: self.args.client_ca.clone(), + client_cert_path: self.args.client_cert.clone(), + client_key_path: self.args.client_key.clone(), + target_label: format!("webhook:{}", self.id.id), + } + } + + async fn build_tls_material(&self) -> Result { + // build_http_client is synchronous (reads files + configures reqwest). + // The coordinator already runs this in a background task, so the + // synchronous file I/O does not block the send path. + Self::build_http_client(&self.args) + } + + async fn apply_tls_material( + &self, + _generation: TargetTlsGeneration, + material: Arc, + _mode: ReloadApplyMode, + ) -> Result<(), TargetError> { + *self.http_client.lock() = (*material).clone(); + Ok(()) + } + + async fn validate_tls_files(&self) -> Result<(), TargetError> { + validate_tls_material(&self.args.client_ca, &self.args.client_cert, &self.args.client_key) + } +} + #[cfg(test)] mod tests { use super::{WebhookArgs, WebhookTarget}; diff --git a/crates/tls-runtime/Cargo.toml b/crates/tls-runtime/Cargo.toml new file mode 100644 index 000000000..d4f2a2a0e --- /dev/null +++ b/crates/tls-runtime/Cargo.toml @@ -0,0 +1,50 @@ +# Copyright 2024 RustFS Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[package] +name = "rustfs-tls-runtime" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +homepage.workspace = true +description = "Project-wide TLS runtime foundation for RustFS." +keywords = ["tls", "runtime", "rustls", "hot-reload", "rustfs"] +categories = ["network-programming", "web-programming", "development-tools"] + +[lints] +workspace = true + +[dependencies] +rustfs-common.workspace = true +rustfs-config.workspace = true +arc-swap.workspace = true +metrics.workspace = true +rustls.workspace = true +rustls-pki-types.workspace = true +serde = { workspace = true, features = ["derive"] } +sha2.workspace = true +thiserror.workspace = true +tokio = { workspace = true, features = ["fs", "rt-multi-thread", "sync", "time"] } +tracing.workspace = true + +[dev-dependencies] +rcgen.workspace = true +serde_json.workspace = true +tempfile.workspace = true +tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } + +[lib] +doctest = false diff --git a/crates/utils/src/certs.rs b/crates/tls-runtime/src/certs.rs similarity index 51% rename from crates/utils/src/certs.rs rename to crates/tls-runtime/src/certs.rs index 50e4d2a52..994cde161 100644 --- a/crates/utils/src/certs.rs +++ b/crates/tls-runtime/src/certs.rs @@ -25,7 +25,6 @@ use std::sync::Arc; use std::{fs, io}; use tracing::{debug, warn}; -/// Options for loading certificate/key pairs from a directory tree. #[derive(Debug, Clone)] pub struct CertDirectoryLoadOptions { dir_path: PathBuf, @@ -34,7 +33,6 @@ pub struct CertDirectoryLoadOptions { } impl CertDirectoryLoadOptions { - /// Create a builder with explicit certificate and private key filenames. pub fn builder( dir_path: impl Into, cert_filename: impl Into, @@ -58,7 +56,6 @@ impl CertDirectoryLoadOptions { } } -/// Builder for [`CertDirectoryLoadOptions`]. #[derive(Debug, Clone)] pub struct CertDirectoryLoadOptionsBuilder { dir_path: PathBuf, @@ -67,19 +64,16 @@ pub struct CertDirectoryLoadOptionsBuilder { } impl CertDirectoryLoadOptionsBuilder { - /// Override the certificate filename searched in the directory. pub fn cert_filename(mut self, cert_filename: impl Into) -> Self { self.cert_filename = cert_filename.into(); self } - /// Override the private key filename searched in the directory. pub fn key_filename(mut self, key_filename: impl Into) -> Self { self.key_filename = key_filename.into(); self } - /// Build the load options value. pub fn build(self) -> CertDirectoryLoadOptions { CertDirectoryLoadOptions { dir_path: self.dir_path, @@ -89,7 +83,6 @@ impl CertDirectoryLoadOptionsBuilder { } } -/// Options for building an mTLS WebPki client verifier. #[derive(Debug, Clone)] pub struct WebPkiClientVerifierOptions { tls_path: PathBuf, @@ -99,7 +92,6 @@ pub struct WebPkiClientVerifierOptions { } impl WebPkiClientVerifierOptions { - /// Create a builder with explicit CA bundle filenames. pub fn builder( tls_path: impl Into, client_ca_cert_filename: impl Into, @@ -114,7 +106,6 @@ impl WebPkiClientVerifierOptions { } } -/// Builder for [`WebPkiClientVerifierOptions`]. #[derive(Debug, Clone)] pub struct WebPkiClientVerifierOptionsBuilder { tls_path: PathBuf, @@ -124,25 +115,21 @@ pub struct WebPkiClientVerifierOptionsBuilder { } impl WebPkiClientVerifierOptionsBuilder { - /// Set whether mTLS verification should be enabled. pub fn enabled(mut self, enabled: bool) -> Self { self.enabled = enabled; self } - /// Override the preferred client CA bundle filename. pub fn client_ca_cert_filename(mut self, client_ca_cert_filename: impl Into) -> Self { self.client_ca_cert_filename = client_ca_cert_filename.into(); self } - /// Override the fallback CA bundle filename. pub fn fallback_ca_cert_filename(mut self, fallback_ca_cert_filename: impl Into) -> Self { self.fallback_ca_cert_filename = fallback_ca_cert_filename.into(); self } - /// Build the verifier options value. pub fn build(self) -> WebPkiClientVerifierOptions { WebPkiClientVerifierOptions { tls_path: self.tls_path, @@ -153,21 +140,10 @@ impl WebPkiClientVerifierOptionsBuilder { } } -/// Load public certificate from file. -/// This function loads a public certificate from the specified file. -/// -/// # Arguments -/// * `filename` - A string slice that holds the name of the file containing the public certificate. -/// -/// # Returns -/// * An io::Result containing a vector of CertificateDer if successful, or an io::Error if an error occurs during loading. -/// pub fn load_certs(filename: &str) -> io::Result>> { - // Open certificate file. let cert_file = fs::File::open(filename).map_err(|e| certs_error(format!("failed to open {filename}: {e}")))?; let mut reader = io::BufReader::new(cert_file); - // Load and return certificate. let certs = CertificateDer::pem_reader_iter(&mut reader) .collect::, _>>() .map_err(|e| certs_error(format!("certificate file {filename} format error:{e:?}")))?; @@ -177,16 +153,6 @@ pub fn load_certs(filename: &str) -> io::Result>> { Ok(certs) } -/// Load a PEM certificate bundle and return each certificate as DER bytes. -/// -/// This is a low-level helper intended for TLS clients (reqwest/hyper-rustls) that -/// need to add root certificates one-by-one. -/// -/// - Input: a PEM file that may contain multiple cert blocks. -/// - Output: Vec of DER-encoded cert bytes, one per cert. -/// -/// NOTE: This intentionally returns raw bytes to avoid forcing downstream crates -/// to depend on rustls types. pub fn load_cert_bundle_der_bytes(path: &str) -> io::Result>> { let pem = fs::read(path)?; let mut reader = io::BufReader::new(&pem[..]); @@ -198,15 +164,6 @@ pub fn load_cert_bundle_der_bytes(path: &str) -> io::Result>> { Ok(certs.into_iter().map(|c| c.to_vec()).collect()) } -/// Builds a WebPkiClientVerifier for mTLS when enabled by the caller. -/// -/// # Arguments -/// * `options` - mTLS verifier options, including the TLS directory and CA bundle filenames -/// -/// # Returns -/// * `Ok(Some(verifier))` if mTLS is enabled and CA certs are found -/// * `Ok(None)` if mTLS is disabled -/// * `Err` if mTLS is enabled but configuration is invalid pub fn build_webpki_client_verifier(options: WebPkiClientVerifierOptions) -> io::Result>> { if !options.enabled { return Ok(None); @@ -243,7 +200,6 @@ pub fn build_webpki_client_verifier(options: WebPkiClientVerifierOptions) -> io: Ok(Some(verifier)) } -/// Locate the mTLS client CA bundle in the specified TLS path fn mtls_ca_bundle_path(options: &WebPkiClientVerifierOptions) -> Option { let p1 = options.tls_path.join(&options.client_ca_cert_filename); if p1.exists() { @@ -256,34 +212,14 @@ fn mtls_ca_bundle_path(options: &WebPkiClientVerifierOptions) -> Option None } -/// Load private key from file. -/// This function loads a private key from the specified file. -/// -/// # Arguments -/// * `filename` - A string slice that holds the name of the file containing the private key. -/// -/// # Returns -/// * An io::Result containing the PrivateKeyDer if successful, or an io::Error if an error occurs during loading. -/// pub fn load_private_key(filename: &str) -> io::Result> { - // Open keyfile. let keyfile = fs::File::open(filename).map_err(|e| certs_error(format!("failed to open {filename}: {e}")))?; let mut reader = io::BufReader::new(keyfile); - // Load and return a single private key. PrivateKeyDer::from_pem_reader(&mut reader) .map_err(|e| certs_error(format!("failed to parse private key in {filename}: {e}"))) } -/// error function -/// This function creates a new io::Error with the provided error message. -/// -/// # Arguments -/// * `err` - A string containing the error message. -/// -/// # Returns -/// * An io::Error instance with the specified error message. -/// pub fn certs_error(err: String) -> Error { Error::other(err) } @@ -292,17 +228,6 @@ fn is_discoverable_cert_domain_dir(domain_name: &str) -> bool { !domain_name.starts_with('.') } -/// Load all certificates and private keys in the directory -/// This function loads all certificate and private key pairs from the specified directory. -/// It looks for files named `options.cert_filename` and `options.key_filename` in each subdirectory. -/// The root directory can also contain a default certificate/private key pair. -/// -/// # Arguments -/// * `options` - Directory and filename options for discovering certificates and private keys. -/// -/// # Returns -/// * An io::Result containing a HashMap where the keys are domain names (or "default" for the root certificate) and the values are tuples of (Vec, PrivateKeyDer). If no valid certificate/private key pairs are found, an io::Error is returned. -/// pub fn load_all_certs_from_directory( options: CertDirectoryLoadOptions, ) -> io::Result>, PrivateKeyDer<'static>)>> { @@ -318,7 +243,6 @@ pub fn load_all_certs_from_directory( ))); } - // 1. First check whether there is a certificate/private key pair in the root directory let root_cert_path = dir.join(&options.cert_filename); let root_key_path = dir.join(&options.key_filename); @@ -332,7 +256,6 @@ pub fn load_all_certs_from_directory( .ok_or_else(|| certs_error(format!("Invalid UTF-8 in root key path: {root_key_path:?}")))?; match load_cert_key_pair(root_cert_str, root_key_str) { Ok((certs, key)) => { - // The root directory certificate is used as the default certificate and is stored using special keys. cert_key_pairs.insert("default".to_string(), (certs, key)); } Err(e) => { @@ -341,7 +264,6 @@ pub fn load_all_certs_from_directory( } } - // 2.iterate through all folders in the directory for entry in fs::read_dir(dir)? { let entry = entry?; let path = entry.path(); @@ -356,9 +278,8 @@ pub fn load_all_certs_from_directory( continue; } - // find certificate and private key files - let cert_path = path.join(&options.cert_filename); // e.g., rustfs_cert.pem - let key_path = path.join(&options.key_filename); // e.g., rustfs_key.pem + let cert_path = path.join(&options.cert_filename); + let key_path = path.join(&options.key_filename); if cert_path.exists() && key_path.exists() { debug!("find the domain name certificate: {} in {:?}", domain_name, cert_path); @@ -391,43 +312,21 @@ pub fn load_all_certs_from_directory( } if cert_key_pairs.is_empty() { - return Err(certs_error(format!( - "No valid certificate/private key pair found in directory {}", - dir.display() - ))); + return Err(io::Error::new( + io::ErrorKind::NotFound, + format!("No valid certificate/private key pair found in directory {}", dir.display()), + )); } Ok(cert_key_pairs) } -/// loading a single certificate private key pair -/// This function loads a certificate and private key from the specified paths. -/// It returns a tuple containing the certificate and private key. -/// -/// # Arguments -/// * `cert_path` - A string slice that holds the path to the certificate file. -/// * `key_path` - A string slice that holds the path to the private key file -/// -/// # Returns -/// * An io::Result containing a tuple of (Vec, PrivateKeyDer) if successful, or an io::Error if an error occurs during loading. -/// fn load_cert_key_pair(cert_path: &str, key_path: &str) -> io::Result<(Vec>, PrivateKeyDer<'static>)> { let certs = load_certs(cert_path)?; let key = load_private_key(key_path)?; Ok((certs, key)) } -/// Create a multi-cert resolver -/// This function loads all certificates and private keys from the specified directory. -/// It uses the first certificate/private key pair found in the root directory as the default certificate. -/// The rest of the certificates/private keys are used for SNI resolution. -/// -/// # Arguments -/// * `cert_key_pairs` - A HashMap where the keys are domain names (or "default" for the root certificate) and the values are tuples of (Vec, PrivateKeyDer). -/// -/// # Returns -/// * An io::Result containing an implementation of ResolvesServerCert if successful, or an io::Error if an error occurs during loading. -/// pub fn create_multi_cert_resolver( cert_key_pairs: HashMap>, PrivateKeyDer<'static>)>, ) -> io::Result { @@ -436,14 +335,13 @@ pub fn create_multi_cert_resolver( cert_resolver: ResolvesServerCertUsingSni, default_cert: Option>, } + impl ResolvesServerCert for MultiCertResolver { - fn resolve(&self, client_hello: ClientHello) -> Option> { - // try matching certificates with sni + fn resolve(&self, client_hello: ClientHello<'_>) -> Option> { if let Some(cert) = self.cert_resolver.resolve(client_hello) { return Some(cert); } - // If there is no matching SNI certificate, use the default certificate self.default_cert.clone() } } @@ -452,16 +350,13 @@ pub fn create_multi_cert_resolver( let mut default_cert = None; for (domain, (certs, key)) in cert_key_pairs { - // create a signature let signing_key = rustls::crypto::aws_lc_rs::sign::any_supported_type(&key) .map_err(|e| certs_error(format!("unsupported private key types:{domain}, err:{e:?}")))?; - // create a CertifiedKey let certified_key = CertifiedKey::new(certs, signing_key); if domain == "default" { default_cert = Some(Arc::new(certified_key.clone())); } else { - // add certificate to resolver resolver .add(&domain, certified_key) .map_err(|e| certs_error(format!("failed to add a domain name certificate:{domain},err: {e:?}")))?; @@ -479,6 +374,7 @@ mod tests { use super::*; use std::fs; use std::io::ErrorKind; + use std::path::PathBuf; use tempfile::TempDir; fn default_load_options(path: impl Into) -> CertDirectoryLoadOptions { @@ -487,9 +383,9 @@ mod tests { fn write_test_cert_pair(dir: &std::path::Path) { let rcgen::CertifiedKey { cert, signing_key } = - rcgen::generate_simple_self_signed(vec!["example.com".to_string()]).unwrap(); - fs::write(dir.join("rustfs_cert.pem"), cert.pem()).unwrap(); - fs::write(dir.join("rustfs_key.pem"), signing_key.serialize_pem()).unwrap(); + rcgen::generate_simple_self_signed(vec!["example.com".to_string()]).expect("cert should generate"); + fs::write(dir.join("rustfs_cert.pem"), cert.pem()).expect("cert should write"); + fs::write(dir.join("rustfs_key.pem"), signing_key.serialize_pem()).expect("key should write"); } #[test] @@ -506,7 +402,7 @@ mod tests { let result = load_certs("non_existent_file.pem"); assert!(result.is_err()); - let error = result.unwrap_err(); + let error = result.expect_err("missing cert should error"); assert_eq!(error.kind(), ErrorKind::Other); assert!(error.to_string().contains("failed to open")); } @@ -516,252 +412,40 @@ mod tests { let result = load_private_key("non_existent_key.pem"); assert!(result.is_err()); - let error = result.unwrap_err(); + let error = result.expect_err("missing key should error"); assert_eq!(error.kind(), ErrorKind::Other); assert!(error.to_string().contains("failed to open")); } - #[test] - fn test_load_certs_empty_file() { - let temp_dir = TempDir::new().unwrap(); - let cert_path = temp_dir.path().join("empty.pem"); - fs::write(&cert_path, "").unwrap(); - - let result = load_certs(cert_path.to_str().unwrap()); - assert!(result.is_err()); - - let error = result.unwrap_err(); - assert!(error.to_string().contains("No valid certificate was found")); - } - - #[test] - fn test_load_certs_invalid_format() { - let temp_dir = TempDir::new().unwrap(); - let cert_path = temp_dir.path().join("invalid.pem"); - fs::write(&cert_path, "invalid certificate content").unwrap(); - - let result = load_certs(cert_path.to_str().unwrap()); - assert!(result.is_err()); - - let error = result.unwrap_err(); - assert!(error.to_string().contains("No valid certificate was found")); - } - - #[test] - fn test_load_private_key_empty_file() { - let temp_dir = TempDir::new().unwrap(); - let key_path = temp_dir.path().join("empty_key.pem"); - fs::write(&key_path, "").unwrap(); - - let result = load_private_key(key_path.to_str().unwrap()); - assert!(result.is_err()); - - let error = result.unwrap_err(); - assert!(error.to_string().contains("failed to parse private key in")); - } - - #[test] - fn test_load_private_key_invalid_format() { - let temp_dir = TempDir::new().unwrap(); - let key_path = temp_dir.path().join("invalid_key.pem"); - fs::write(&key_path, "invalid private key content").unwrap(); - - let result = load_private_key(key_path.to_str().unwrap()); - assert!(result.is_err()); - - let error = result.unwrap_err(); - assert!(error.to_string().contains("failed to parse private key in")); - } - - #[test] - fn test_load_all_certs_from_directory_not_exists() { - let result = load_all_certs_from_directory(default_load_options("/non/existent/directory")); - assert!(result.is_err()); - - let error = result.unwrap_err(); - assert!(error.to_string().contains("does not exist or is not a directory")); - } - #[test] fn test_load_all_certs_from_directory_empty() { - let temp_dir = TempDir::new().unwrap(); - + let temp_dir = TempDir::new().expect("tempdir should create"); let result = load_all_certs_from_directory(default_load_options(temp_dir.path())); assert!(result.is_err()); - - let error = result.unwrap_err(); + let error = result.expect_err("empty directory should error"); + assert_eq!(error.kind(), ErrorKind::NotFound); assert!(error.to_string().contains("No valid certificate/private key pair found")); } - #[test] - fn test_load_all_certs_from_directory_file_instead_of_dir() { - let temp_dir = TempDir::new().unwrap(); - let file_path = temp_dir.path().join("not_a_directory.txt"); - fs::write(&file_path, "content").unwrap(); - - let result = load_all_certs_from_directory(default_load_options(&file_path)); - assert!(result.is_err()); - - let error = result.unwrap_err(); - assert!(error.to_string().contains("does not exist or is not a directory")); - } - - #[test] - fn test_load_cert_key_pair_missing_cert() { - let temp_dir = TempDir::new().unwrap(); - let key_path = temp_dir.path().join("test_key.pem"); - fs::write(&key_path, "dummy key content").unwrap(); - - let result = load_cert_key_pair("non_existent_cert.pem", key_path.to_str().unwrap()); - assert!(result.is_err()); - } - - #[test] - fn test_load_cert_key_pair_missing_key() { - let temp_dir = TempDir::new().unwrap(); - let cert_path = temp_dir.path().join("test_cert.pem"); - fs::write(&cert_path, "dummy cert content").unwrap(); - - let result = load_cert_key_pair(cert_path.to_str().unwrap(), "non_existent_key.pem"); - assert!(result.is_err()); - } - - #[test] - fn test_create_multi_cert_resolver_empty_map() { - let empty_map = HashMap::new(); - let result = create_multi_cert_resolver(empty_map); - - // Should succeed even with empty map - assert!(result.is_ok()); - } - - #[test] - fn test_error_message_formatting() { - let test_cases = vec![ - ("file not found", "failed to open test.pem: file not found"), - ("permission denied", "failed to open key.pem: permission denied"), - ("invalid format", "certificate file cert.pem format error:invalid format"), - ]; - - for (input, _expected_pattern) in test_cases { - let error1 = certs_error(format!("failed to open test.pem: {input}")); - assert!(error1.to_string().contains(input)); - - let error2 = certs_error(format!("failed to open key.pem: {input}")); - assert!(error2.to_string().contains(input)); - } - } - - #[test] - fn test_path_handling_edge_cases() { - // Test with various path formats - let path_cases = vec![ - "", // Empty path - ".", // Current directory - "..", // Parent directory - "/", // Root directory (Unix) - "relative/path", // Relative path - "/absolute/path", // Absolute path - ]; - - for path in path_cases { - let result = load_all_certs_from_directory(default_load_options(path)); - // All should fail since these are not valid cert directories - assert!(result.is_err()); - } - } - - #[test] - fn test_directory_structure_validation() { - let temp_dir = TempDir::new().unwrap(); - - // Create a subdirectory without certificates - let sub_dir = temp_dir.path().join("example.com"); - fs::create_dir(&sub_dir).unwrap(); - - // Should fail because no certificates found - let result = load_all_certs_from_directory(default_load_options(temp_dir.path())); - assert!(result.is_err()); - assert!( - result - .unwrap_err() - .to_string() - .contains("No valid certificate/private key pair found") - ); - } - #[test] fn test_load_all_certs_skips_kubernetes_secret_projection_dirs() { - let temp_dir = TempDir::new().unwrap(); + let temp_dir = TempDir::new().expect("tempdir should create"); write_test_cert_pair(temp_dir.path()); let domain_dir = temp_dir.path().join("example.com"); - fs::create_dir(&domain_dir).unwrap(); + fs::create_dir(&domain_dir).expect("domain dir should create"); write_test_cert_pair(&domain_dir); for internal_dir_name in ["..data", "..2026_04_28_18_33_53.4209048473"] { let internal_dir = temp_dir.path().join(internal_dir_name); - fs::create_dir(&internal_dir).unwrap(); + fs::create_dir(&internal_dir).expect("internal dir should create"); write_test_cert_pair(&internal_dir); } - let certs = load_all_certs_from_directory(default_load_options(temp_dir.path())).unwrap(); - + let certs = load_all_certs_from_directory(default_load_options(temp_dir.path())).expect("certs should load"); assert!(certs.contains_key("default")); assert!(certs.contains_key("example.com")); assert!(!certs.contains_key("..data")); - assert!(!certs.contains_key("..2026_04_28_18_33_53.4209048473")); assert_eq!(certs.len(), 2); } - - #[test] - fn test_unicode_path_handling() { - let temp_dir = TempDir::new().unwrap(); - - // Create directory with Unicode characters - let unicode_dir = temp_dir.path().join("test_directory"); - fs::create_dir(&unicode_dir).unwrap(); - - let result = load_all_certs_from_directory(default_load_options(&unicode_dir)); - assert!(result.is_err()); - assert!( - result - .unwrap_err() - .to_string() - .contains("No valid certificate/private key pair found") - ); - } - - #[test] - fn test_concurrent_access_safety() { - use std::sync::Arc; - use std::thread; - - let temp_dir = TempDir::new().unwrap(); - let dir_path = Arc::new(temp_dir.path().to_string_lossy().to_string()); - - let handles: Vec<_> = (0..5) - .map(|_| { - let path = Arc::clone(&dir_path); - thread::spawn(move || { - let result = load_all_certs_from_directory(default_load_options(path.as_str())); - // All should fail since directory is empty - assert!(result.is_err()); - }) - }) - .collect(); - - for handle in handles { - handle.join().expect("Thread should complete successfully"); - } - } - - #[test] - fn test_memory_efficiency() { - let error = certs_error("test".to_string()); - let error_size = std::mem::size_of_val(&error); - - // Error should not be excessively large - assert!(error_size < 1024, "Error size should be reasonable, got {error_size} bytes"); - } } diff --git a/crates/tls-runtime/src/config.rs b/crates/tls-runtime/src/config.rs new file mode 100644 index 000000000..9162b82b7 --- /dev/null +++ b/crates/tls-runtime/src/config.rs @@ -0,0 +1,51 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::time::Duration; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ReloadDetectMode { + Poll, + Watch, // TODO: implement fs::watch-based reload + Hybrid, // TODO: implement poll + fs::watch hybrid +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ReloadApplyHint { + Lazy, + SoftReconnect, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TlsReloadOptions { + pub enabled: bool, + pub detect_mode: ReloadDetectMode, + pub interval: Duration, + pub debounce: Duration, + pub min_stable_age: Duration, + pub apply_hint: ReloadApplyHint, +} + +impl Default for TlsReloadOptions { + fn default() -> Self { + Self { + enabled: true, + detect_mode: ReloadDetectMode::Poll, + interval: Duration::from_secs(15), + debounce: Duration::from_secs(2), + min_stable_age: Duration::from_secs(1), + apply_hint: ReloadApplyHint::Lazy, + } + } +} diff --git a/crates/tls-runtime/src/coordinator.rs b/crates/tls-runtime/src/coordinator.rs new file mode 100644 index 000000000..da65167ed --- /dev/null +++ b/crates/tls-runtime/src/coordinator.rs @@ -0,0 +1,226 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::config::{ReloadDetectMode, TlsReloadOptions}; +use crate::error::TlsRuntimeError; +use crate::material::TlsMaterialSnapshot; +use crate::metrics::{ + TLS_RUNTIME_FOUNDATION_CONSUMER, record_tls_generation, record_tls_publication_fail, record_tls_reload_result, + record_tls_reload_skipped, +}; +use crate::source::TlsSource; +use crate::state::{ + TlsGeneration, TlsPublishedState, TlsReloadRuntimeState, TlsRuntimeConsumerSection, TlsRuntimeOutboundSection, + TlsRuntimeRuntimeSection, TlsRuntimeServerSection, TlsRuntimeStatusSnapshot, detect_mode_label, +}; +use std::sync::Arc; +use std::time::{SystemTime, UNIX_EPOCH}; +use tokio::task::JoinHandle; +use tracing::{debug, info, warn}; + +pub trait TlsConsumer: Send + Sync + 'static { + fn on_publish(&self, generation: TlsGeneration, state: Arc>) -> Result<(), TlsRuntimeError>; +} + +#[derive(Debug)] +pub struct TlsReloadCoordinator { + source: TlsSource, + options: TlsReloadOptions, +} + +impl TlsReloadCoordinator { + pub fn new(source: TlsSource, options: TlsReloadOptions) -> Self { + Self { source, options } + } + + pub fn source(&self) -> &TlsSource { + &self.source + } + + pub fn options(&self) -> &TlsReloadOptions { + &self.options + } + + pub async fn status_snapshot(&self, runtime_state: &TlsReloadRuntimeState) -> TlsRuntimeStatusSnapshot { + let current = runtime_state.current.load(); + let last_attempt = runtime_state.last_attempt_unix_ms(); + let last_success = runtime_state.last_success_unix_ms(); + + TlsRuntimeStatusSnapshot { + runtime: TlsRuntimeRuntimeSection { + generation: current.generation.0, + reload_enabled: self.options.enabled, + detect_mode: detect_mode_label(self.options.detect_mode), + last_attempt_time: (last_attempt != 0).then_some(last_attempt), + last_success_time: (last_success != 0).then_some(last_success), + last_error: runtime_state.last_error.read().await.clone(), + source_path: self.source.base_dir.display().to_string(), + }, + outbound: TlsRuntimeOutboundSection { + has_roots: !current.material.outbound.root_ca_pem.is_empty(), + has_mtls_identity: current.material.outbound.mtls_identity.is_some(), + }, + server: TlsRuntimeServerSection { + has_material: current.material.server.is_some(), + }, + consumer: TlsRuntimeConsumerSection { stale_generation: false }, + } + } + + pub async fn load_initial_snapshot(&self) -> Result { + TlsMaterialSnapshot::load(&self.source).await + } + + pub async fn publish_initial_state(&self, snapshot: TlsMaterialSnapshot) -> Arc> { + let published = Arc::new(TlsPublishedState { + generation: TlsGeneration(1), + fingerprint: snapshot.fingerprint.clone(), + material: Arc::new(snapshot), + loaded_at_unix_ms: unix_time_ms(), + }); + record_tls_generation(TLS_RUNTIME_FOUNDATION_CONSUMER, published.generation.0); + published + } + + pub async fn reload_once( + &self, + runtime_state: &TlsReloadRuntimeState, + consumer: &C, + ) -> Result>>, TlsRuntimeError> + where + C: TlsConsumer, + { + runtime_state.mark_attempt(unix_time_ms()); + let started_at = std::time::Instant::now(); + + let snapshot = self.load_initial_snapshot().await?; + let current = runtime_state.current.load(); + if current.fingerprint == snapshot.fingerprint { + debug!(source = %self.source.base_dir.display(), "TLS material unchanged; skipping publication"); + record_tls_reload_skipped(TLS_RUNTIME_FOUNDATION_CONSUMER, "unchanged"); + return Ok(None); + } + + let published = Arc::new(TlsPublishedState { + generation: runtime_state.bump_generation(), + fingerprint: snapshot.fingerprint.clone(), + material: Arc::new(snapshot), + loaded_at_unix_ms: unix_time_ms(), + }); + + if let Err(err) = consumer.on_publish(published.generation, published.clone()) { + record_tls_publication_fail(TLS_RUNTIME_FOUNDATION_CONSUMER); + return Err(err); + } + runtime_state.current.store(published.clone()); + runtime_state.last_good.store(published.clone()); + runtime_state.mark_success(unix_time_ms()); + *runtime_state.last_error.write().await = None; + record_tls_reload_result( + TLS_RUNTIME_FOUNDATION_CONSUMER, + "ok", + Some(started_at.elapsed().as_secs_f64()), + Some(published.generation.0), + ); + + Ok(Some(published)) + } + + pub fn spawn_poll_loop( + self: Arc, + runtime_state: Arc>, + consumer: Arc, + ) -> Option> + where + C: TlsConsumer, + { + if !self.options.enabled { + debug!(source = %self.source.base_dir.display(), "TLS reload disabled; poll loop not started"); + return None; + } + + if !matches!(self.options.detect_mode, ReloadDetectMode::Poll | ReloadDetectMode::Hybrid) { + debug!(source = %self.source.base_dir.display(), "TLS poll loop skipped for non-poll detect mode"); + return None; + } + + let interval_duration = self.options.interval; + info!( + source = %self.source.base_dir.display(), + interval_secs = interval_duration.as_secs(), + "TLS poll reload loop enabled" + ); + + Some(tokio::spawn(async move { + let mut interval = tokio::time::interval(interval_duration); + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + interval.tick().await; + + loop { + interval.tick().await; + if let Err(err) = self.reload_once(runtime_state.as_ref(), consumer.as_ref()).await { + warn!( + source = %self.source.base_dir.display(), + error = %err, + "TLS reload failed (will retry)" + ); + *runtime_state.last_error.write().await = Some(err.to_string()); + } + } + })) + } +} + +fn unix_time_ms() -> u64 { + SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_millis() as u64 +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::source::TlsSource; + use std::sync::Arc; + + struct NopConsumer; + + impl TlsConsumer for NopConsumer { + fn on_publish( + &self, + _generation: TlsGeneration, + _state: Arc>, + ) -> Result<(), TlsRuntimeError> { + Ok(()) + } + } + + #[tokio::test] + async fn reload_once_skips_when_fingerprint_unchanged() { + let temp = tempfile::tempdir().expect("tempdir"); + let source = TlsSource::from_directory(temp.path().to_path_buf()); + let options = TlsReloadOptions::default(); + let coordinator = TlsReloadCoordinator::new(source.clone(), options); + + // Load the actual snapshot from the (empty) temp dir so its fingerprint + // matches what reload_once will observe on the next load. + let initial_snapshot = coordinator.load_initial_snapshot().await.expect("initial load"); + let initial = coordinator.publish_initial_state(initial_snapshot).await; + let runtime_state = TlsReloadRuntimeState::new(initial); + + let consumer = NopConsumer; + let result = coordinator.reload_once(&runtime_state, &consumer).await; + // Fingerprint has not changed → should skip and return Ok(None). + assert!(result.is_ok(), "reload_once should succeed: {:?}", result.err()); + assert!(result.unwrap().is_none(), "should skip when fingerprint unchanged"); + } +} diff --git a/crates/tls-runtime/src/debug.rs b/crates/tls-runtime/src/debug.rs new file mode 100644 index 000000000..4475d0bbe --- /dev/null +++ b/crates/tls-runtime/src/debug.rs @@ -0,0 +1,106 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::state::TlsRuntimeStatusSnapshot; +use serde::Serialize; + +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +pub struct TlsConsumerStatusItem { + pub consumer: &'static str, + pub generation: u64, + pub has_root_ca: bool, + pub has_mtls_identity: bool, +} + +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +pub struct TlsDebugStatusResponse { + pub foundation: TlsRuntimeStatusSnapshot, + pub consumers: Vec, +} + +#[derive(Debug, Clone)] +pub struct TlsDebugStatusResponseBuilder { + foundation: TlsRuntimeStatusSnapshot, + consumers: Vec, +} + +impl TlsDebugStatusResponse { + pub fn builder(foundation: TlsRuntimeStatusSnapshot) -> TlsDebugStatusResponseBuilder { + TlsDebugStatusResponseBuilder { + foundation, + consumers: Vec::new(), + } + } +} + +impl TlsDebugStatusResponseBuilder { + pub fn push_consumers(mut self, sources: I) -> Self + where + I: IntoIterator, + { + self.consumers.extend(sources); + self + } + + pub fn build(self) -> TlsDebugStatusResponse { + TlsDebugStatusResponse { + foundation: self.foundation, + consumers: self.consumers, + } + } +} + +#[cfg(test)] +mod tests { + use super::{TlsConsumerStatusItem, TlsDebugStatusResponse}; + use crate::state::{ + TlsRuntimeConsumerSection, TlsRuntimeOutboundSection, TlsRuntimeRuntimeSection, TlsRuntimeServerSection, + TlsRuntimeStatusSnapshot, + }; + + #[test] + fn builder_produces_structured_response() { + let foundation = TlsRuntimeStatusSnapshot { + runtime: TlsRuntimeRuntimeSection { + generation: 5, + reload_enabled: true, + detect_mode: "poll", + last_attempt_time: Some(1), + last_success_time: Some(2), + last_error: None, + source_path: "/tmp/tls".to_string(), + }, + outbound: TlsRuntimeOutboundSection { + has_roots: true, + has_mtls_identity: false, + }, + server: TlsRuntimeServerSection { has_material: true }, + consumer: TlsRuntimeConsumerSection { stale_generation: false }, + }; + + let response = TlsDebugStatusResponse::builder(foundation) + .push_consumers([TlsConsumerStatusItem { + consumer: "test_consumer", + generation: 7, + has_root_ca: true, + has_mtls_identity: false, + }]) + .build(); + + let json = serde_json::to_value(response).expect("response should serialize"); + assert!(json.get("foundation").is_some()); + assert!(json.get("consumers").is_some()); + assert!(json["consumers"].is_array()); + } +} diff --git a/crates/tls-runtime/src/error.rs b/crates/tls-runtime/src/error.rs new file mode 100644 index 000000000..ac7d50691 --- /dev/null +++ b/crates/tls-runtime/src/error.rs @@ -0,0 +1,32 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::path::PathBuf; +use thiserror::Error; + +#[derive(Debug, Error)] +pub enum TlsRuntimeError { + #[error("TLS source path is empty")] + EmptySourcePath, + #[error("TLS directory does not exist: {path}")] + DirectoryNotFound { path: PathBuf }, + #[error("TLS path is not a directory: {path}")] + NotADirectory { path: PathBuf }, + #[error("TLS material error: {0}")] + Material(String), + #[error("TLS publication error: {0}")] + Publication(String), + #[error("I/O error: {0}")] + Io(#[from] std::io::Error), +} diff --git a/crates/tls-runtime/src/fingerprint.rs b/crates/tls-runtime/src/fingerprint.rs new file mode 100644 index 000000000..0e7857025 --- /dev/null +++ b/crates/tls-runtime/src/fingerprint.rs @@ -0,0 +1,48 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use sha2::{Digest, Sha256}; + +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub struct TlsFingerprint { + pub server_sha256: Option<[u8; 32]>, + pub public_ca_sha256: Option<[u8; 32]>, + pub client_ca_sha256: Option<[u8; 32]>, + pub client_cert_sha256: Option<[u8; 32]>, + pub client_key_sha256: Option<[u8; 32]>, +} + +fn digest_bytes(bytes: &[u8]) -> [u8; 32] { + let mut hasher = Sha256::new(); + hasher.update(bytes); + hasher.finalize().into() +} + +impl TlsFingerprint { + pub fn from_optional_bytes( + server: Option<&[u8]>, + public_ca: Option<&[u8]>, + client_ca: Option<&[u8]>, + client_cert: Option<&[u8]>, + client_key: Option<&[u8]>, + ) -> Self { + Self { + server_sha256: server.map(digest_bytes), + public_ca_sha256: public_ca.map(digest_bytes), + client_ca_sha256: client_ca.map(digest_bytes), + client_cert_sha256: client_cert.map(digest_bytes), + client_key_sha256: client_key.map(digest_bytes), + } + } +} diff --git a/crates/tls-runtime/src/lib.rs b/crates/tls-runtime/src/lib.rs new file mode 100644 index 000000000..dea4daf74 --- /dev/null +++ b/crates/tls-runtime/src/lib.rs @@ -0,0 +1,126 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +pub mod certs; +pub mod config; +pub mod coordinator; +pub mod debug; +pub mod error; +pub mod fingerprint; +pub mod material; +pub mod metrics; +pub mod outbound; +pub mod server; +pub mod source; +pub mod state; + +pub use certs::{ + CertDirectoryLoadOptions, WebPkiClientVerifierOptions, build_webpki_client_verifier, create_multi_cert_resolver, + load_all_certs_from_directory, load_cert_bundle_der_bytes, load_certs, load_private_key, +}; +pub use config::{ReloadApplyHint, ReloadDetectMode, TlsReloadOptions}; +pub use coordinator::{TlsConsumer, TlsReloadCoordinator}; +pub use debug::{TlsConsumerStatusItem, TlsDebugStatusResponse, TlsDebugStatusResponseBuilder}; +pub use error::TlsRuntimeError; +pub use fingerprint::TlsFingerprint; +pub use material::{OutboundTlsMaterial, ServerTlsMaterial, TlsMaterialSnapshot}; +pub use metrics::{ + TLS_OUTBOUND_GLOBAL_CONSUMER, TLS_RUNTIME_FOUNDATION_CONSUMER, init_tls_metrics, record_tls_consumer_stale_generation, + record_tls_generation, record_tls_publication_fail, record_tls_reload_result, record_tls_reload_skipped, +}; +pub use outbound::{ + GlobalOutboundTlsStateSummary, GlobalPublishedOutboundTlsState, load_global_outbound_tls_generation, + load_global_outbound_tls_state, publish_global_outbound_tls_state, summarize_global_outbound_tls_state, +}; +pub use server::{ReloadableServerCertResolver, spawn_server_cert_reload_loop}; +pub use source::{TlsFileLayout, TlsSource, TlsSourceKind}; +pub use state::OutboundOnlySnapshotArgs; +pub use state::{TlsGeneration, TlsPublishedState, TlsReloadRuntimeState, TlsRuntimeStatusSnapshot}; +pub use state::{TlsRuntimeConsumerSection, TlsRuntimeOutboundSection, TlsRuntimeRuntimeSection, TlsRuntimeServerSection}; + +#[cfg(test)] +mod tests { + use super::*; + use rcgen::generate_simple_self_signed; + use std::collections::HashMap; + + #[test] + fn tls_source_requires_existing_directory() { + let source = TlsSource::from_directory("/definitely/missing/rustfs/tls-runtime-test"); + let err = source.validate_directory().expect_err("missing directory should fail"); + assert!(matches!(err, TlsRuntimeError::DirectoryNotFound { .. })); + } + + #[test] + fn fingerprint_changes_when_server_material_changes() { + let cert_a = generate_simple_self_signed(vec!["a.example.com".to_string()]).expect("cert A should generate"); + let cert_b = generate_simple_self_signed(vec!["b.example.com".to_string()]).expect("cert B should generate"); + + let single_a = ServerTlsMaterial::SingleCert { + certs: vec![cert_a.cert.der().clone()], + key: rustls::pki_types::PrivateKeyDer::try_from(cert_a.signing_key.serialize_der()).expect("key A should convert"), + }; + let single_b = ServerTlsMaterial::SingleCert { + certs: vec![cert_b.cert.der().clone()], + key: rustls::pki_types::PrivateKeyDer::try_from(cert_b.signing_key.serialize_der()).expect("key B should convert"), + }; + + let bytes_a = match &single_a { + ServerTlsMaterial::SingleCert { certs, key } => { + let mut bytes = Vec::new(); + for cert in certs { + bytes.extend_from_slice(cert.as_ref()); + } + bytes.extend_from_slice(key.secret_der()); + bytes + } + ServerTlsMaterial::MultiCert { .. } => unreachable!(), + }; + let bytes_b = match &single_b { + ServerTlsMaterial::SingleCert { certs, key } => { + let mut bytes = Vec::new(); + for cert in certs { + bytes.extend_from_slice(cert.as_ref()); + } + bytes.extend_from_slice(key.secret_der()); + bytes + } + ServerTlsMaterial::MultiCert { .. } => unreachable!(), + }; + + let fp_a = TlsFingerprint::from_optional_bytes(Some(&bytes_a), None, None, None, None); + let fp_b = TlsFingerprint::from_optional_bytes(Some(&bytes_b), None, None, None, None); + assert_ne!(fp_a, fp_b); + } + + #[tokio::test] + async fn coordinator_can_publish_initial_state() { + let source = TlsSource::from_directory(std::env::temp_dir()); + let coordinator = TlsReloadCoordinator::new(source.clone(), TlsReloadOptions::default()); + let snapshot = TlsMaterialSnapshot { + source, + server: Some(ServerTlsMaterial::MultiCert { + cert_key_pairs: HashMap::new(), + }), + outbound: OutboundTlsMaterial { + root_ca_pem: Vec::new(), + mtls_identity: None, + }, + fingerprint: TlsFingerprint::default(), + }; + + let published = coordinator.publish_initial_state(snapshot).await; + assert_eq!(published.generation, TlsGeneration(1)); + } +} diff --git a/crates/tls-runtime/src/material.rs b/crates/tls-runtime/src/material.rs new file mode 100644 index 000000000..f6b3fab99 --- /dev/null +++ b/crates/tls-runtime/src/material.rs @@ -0,0 +1,200 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::certs::{CertDirectoryLoadOptions, load_all_certs_from_directory, load_certs, load_private_key}; +use crate::error::TlsRuntimeError; +use crate::fingerprint::TlsFingerprint; +use crate::source::TlsSource; +use rustfs_common::MtlsIdentityPem; +use rustls::pki_types::{CertificateDer, PrivateKeyDer, pem::PemObject}; +use std::collections::HashMap; +use std::io::Cursor; +use std::io::ErrorKind; +use std::path::Path; + +#[derive(Debug, Clone)] +pub struct OutboundTlsMaterial { + pub root_ca_pem: Vec, + pub mtls_identity: Option, +} + +#[derive(Debug)] +pub enum ServerTlsMaterial { + SingleCert { + certs: Vec>, + key: PrivateKeyDer<'static>, + }, + MultiCert { + cert_key_pairs: HashMap>, PrivateKeyDer<'static>)>, + }, +} + +#[derive(Debug)] +pub struct TlsMaterialSnapshot { + pub source: TlsSource, + pub server: Option, + pub outbound: OutboundTlsMaterial, + pub fingerprint: TlsFingerprint, +} + +impl TlsMaterialSnapshot { + pub async fn load(source: &TlsSource) -> Result { + let base_dir = source.validate_directory()?.to_path_buf(); + + let server = load_server_material(source, &base_dir)?; + + let public_ca_path = base_dir.join(&source.layout.public_ca_filename); + let client_ca_path = base_dir.join(&source.fallback_ca_filename); + let client_cert_path = base_dir.join(&source.layout.client_cert_filename); + let client_key_path = base_dir.join(&source.layout.client_key_filename); + + let public_ca_pem = tokio::fs::read(&public_ca_path).await.ok(); + let client_ca_pem = tokio::fs::read(&client_ca_path).await.ok(); + let root_ca_pem = combine_optional_pem(public_ca_pem.as_deref(), client_ca_pem.as_deref()); + + let mtls_identity = match ( + tokio::fs::read(&client_cert_path).await.ok(), + tokio::fs::read(&client_key_path).await.ok(), + ) { + (Some(cert_pem), Some(key_pem)) => { + let mut cert_reader = Cursor::new(&cert_pem); + if CertificateDer::pem_reader_iter(&mut cert_reader).next().is_none() { + return Err(TlsRuntimeError::Material("no valid certificate in client cert PEM".to_string())); + } + + let mut key_reader = Cursor::new(&key_pem); + PrivateKeyDer::from_pem_reader(&mut key_reader) + .map_err(|e| TlsRuntimeError::Material(format!("invalid client key PEM: {e}")))?; + + Some(MtlsIdentityPem { cert_pem, key_pem }) + } + _ => None, + }; + + let outbound = OutboundTlsMaterial { + root_ca_pem: root_ca_pem.clone(), + mtls_identity: mtls_identity.clone(), + }; + + let server_fingerprint_bytes = server.as_ref().map(serialize_server_material_for_fingerprint); + let fingerprint = TlsFingerprint::from_optional_bytes( + server_fingerprint_bytes.as_deref(), + public_ca_pem.as_deref(), + client_ca_pem.as_deref(), + mtls_identity.as_ref().map(|identity| identity.cert_pem.as_slice()), + mtls_identity.as_ref().map(|identity| identity.key_pem.as_slice()), + ); + + Ok(Self { + source: source.clone(), + server, + outbound, + fingerprint, + }) + } +} + +fn load_server_material(source: &TlsSource, base_dir: &Path) -> Result, TlsRuntimeError> { + let root_cert = base_dir.join(&source.layout.server_cert_filename); + let root_key = base_dir.join(&source.layout.server_key_filename); + + let has_root_pair = root_cert.exists() && root_key.exists(); + let cert_key_pairs = load_all_certs_from_directory( + CertDirectoryLoadOptions::builder(base_dir, &source.layout.server_cert_filename, &source.layout.server_key_filename) + .build(), + ); + + match cert_key_pairs { + Ok(cert_key_pairs) if cert_key_pairs.len() > 1 || cert_key_pairs.keys().any(|key| key != "default") => { + Ok(Some(ServerTlsMaterial::MultiCert { cert_key_pairs })) + } + Ok(cert_key_pairs) if !cert_key_pairs.is_empty() => { + if let Some((certs, key)) = cert_key_pairs.get("default") { + return Ok(Some(ServerTlsMaterial::SingleCert { + certs: certs.clone(), + key: key.clone_key(), + })); + } + Ok(Some(ServerTlsMaterial::MultiCert { cert_key_pairs })) + } + Ok(_) => Ok(None), + Err(_err) if has_root_pair => { + let root_cert_path = path_to_utf8_str(&root_cert, "root TLS certificate")?; + let root_key_path = path_to_utf8_str(&root_key, "root TLS private key")?; + let certs = load_certs(root_cert_path) + .map_err(|e| TlsRuntimeError::Material(format!("load root TLS certificate {}: {e}", root_cert.display())))?; + let key = load_private_key(root_key_path) + .map_err(|e| TlsRuntimeError::Material(format!("load root TLS private key {}: {e}", root_key.display())))?; + Ok(Some(ServerTlsMaterial::SingleCert { certs, key })) + } + Err(err) if err.kind() == ErrorKind::NotFound => Ok(None), + Err(err) => Err(TlsRuntimeError::Material(format!( + "discover server TLS certificates under '{}': {err}", + base_dir.display() + ))), + } +} + +fn path_to_utf8_str<'a>(path: &'a Path, description: &str) -> Result<&'a str, TlsRuntimeError> { + path.to_str() + .ok_or_else(|| TlsRuntimeError::Material(format!("{description} path '{}' is not valid UTF-8", path.display()))) +} + +fn combine_optional_pem(primary: Option<&[u8]>, fallback: Option<&[u8]>) -> Vec { + let mut combined = Vec::new(); + + for pem in [primary, fallback].into_iter().flatten() { + if pem.iter().all(|&b| b.is_ascii_whitespace()) { + continue; + } + combined.extend_from_slice(pem); + if !combined.ends_with(b"\n") { + combined.push(b'\n'); + } + } + + combined +} + +fn serialize_server_material_for_fingerprint(material: &ServerTlsMaterial) -> Vec { + match material { + ServerTlsMaterial::SingleCert { certs, key } => { + let mut bytes = Vec::new(); + for cert in certs { + bytes.extend_from_slice(cert.as_ref()); + } + bytes.extend_from_slice(key.secret_der()); + bytes + } + ServerTlsMaterial::MultiCert { cert_key_pairs } => { + let mut entries = cert_key_pairs.iter().collect::>(); + entries.sort_by_key(|(left, _)| *left); + + let mut bytes = Vec::new(); + for (domain, (certs, key)) in entries { + bytes.extend_from_slice(domain.as_bytes()); + for cert in certs { + bytes.extend_from_slice(cert.as_ref()); + } + bytes.extend_from_slice(key.secret_der()); + } + bytes + } + } +} + +pub(crate) fn server_material_fingerprint(material: &ServerTlsMaterial) -> TlsFingerprint { + let bytes = serialize_server_material_for_fingerprint(material); + TlsFingerprint::from_optional_bytes(Some(&bytes), None, None, None, None) +} diff --git a/crates/tls-runtime/src/metrics.rs b/crates/tls-runtime/src/metrics.rs new file mode 100644 index 000000000..75692621f --- /dev/null +++ b/crates/tls-runtime/src/metrics.rs @@ -0,0 +1,88 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use metrics::{counter, describe_counter, describe_gauge, describe_histogram, gauge, histogram}; + +pub const TLS_RUNTIME_FOUNDATION_CONSUMER: &str = "tls_runtime_foundation"; +pub const TLS_OUTBOUND_GLOBAL_CONSUMER: &str = "outbound_global"; + +const CONSUMER_LABEL: &str = "consumer"; +const RESULT_LABEL: &str = "result"; +const REASON_LABEL: &str = "reason"; + +const TLS_OUTBOUND_PUBLICATIONS_TOTAL: &str = "rustfs_tls_outbound_publications_total"; +const TLS_OUTBOUND_GENERATION: &str = "rustfs_tls_outbound_generation"; +const TLS_OUTBOUND_HAS_ROOT_CA: &str = "rustfs_tls_outbound_has_root_ca"; +const TLS_OUTBOUND_HAS_MTLS_IDENTITY: &str = "rustfs_tls_outbound_has_mtls_identity"; +const TLS_GENERATION: &str = "rustfs_tls_generation"; +const TLS_RELOAD_TOTAL: &str = "rustfs_tls_reload_total"; +const TLS_RELOAD_DURATION_SECONDS: &str = "rustfs_tls_reload_duration_seconds"; +const TLS_RELOAD_GENERATION: &str = "rustfs_tls_reload_generation"; +const TLS_RELOAD_SKIPPED_TOTAL: &str = "rustfs_tls_reload_skipped_total"; +const TLS_PUBLICATION_FAIL_TOTAL: &str = "rustfs_tls_publication_fail_total"; +const TLS_CONSUMER_STALE_GENERATION_TOTAL: &str = "rustfs_tls_consumer_stale_generation_total"; + +pub fn record_outbound_tls_publication(generation: u64, has_root_ca: bool, has_mtls_identity: bool) { + counter!(TLS_OUTBOUND_PUBLICATIONS_TOTAL, "result" => "ok").increment(1); + gauge!(TLS_OUTBOUND_GENERATION).set(generation as f64); + gauge!(TLS_OUTBOUND_HAS_ROOT_CA).set(if has_root_ca { 1.0 } else { 0.0 }); + gauge!(TLS_OUTBOUND_HAS_MTLS_IDENTITY).set(if has_mtls_identity { 1.0 } else { 0.0 }); + record_tls_generation(TLS_OUTBOUND_GLOBAL_CONSUMER, generation); +} + +pub fn record_tls_generation(consumer: &'static str, generation: u64) { + gauge!(TLS_GENERATION, CONSUMER_LABEL => consumer).set(generation as f64); +} + +pub fn record_tls_reload_result( + consumer: &'static str, + result: &'static str, + duration_secs: Option, + generation: Option, +) { + counter!(TLS_RELOAD_TOTAL, CONSUMER_LABEL => consumer, RESULT_LABEL => result).increment(1); + if let Some(duration_secs) = duration_secs { + histogram!(TLS_RELOAD_DURATION_SECONDS, CONSUMER_LABEL => consumer).record(duration_secs); + } + if let Some(generation) = generation { + gauge!(TLS_RELOAD_GENERATION, CONSUMER_LABEL => consumer).set(generation as f64); + record_tls_generation(consumer, generation); + } +} + +pub fn record_tls_reload_skipped(consumer: &'static str, reason: &'static str) { + counter!(TLS_RELOAD_SKIPPED_TOTAL, CONSUMER_LABEL => consumer, REASON_LABEL => reason).increment(1); +} + +pub fn record_tls_publication_fail(consumer: &'static str) { + counter!(TLS_PUBLICATION_FAIL_TOTAL, CONSUMER_LABEL => consumer).increment(1); +} + +pub fn record_tls_consumer_stale_generation(consumer: &'static str) { + counter!(TLS_CONSUMER_STALE_GENERATION_TOTAL, CONSUMER_LABEL => consumer).increment(1); +} + +pub fn init_tls_metrics() { + describe_counter!(TLS_OUTBOUND_PUBLICATIONS_TOTAL, "Total TLS outbound publications, labeled by result."); + describe_gauge!(TLS_OUTBOUND_GENERATION, "Current outbound TLS generation."); + describe_gauge!(TLS_OUTBOUND_HAS_ROOT_CA, "Whether outbound TLS roots are configured."); + describe_gauge!(TLS_OUTBOUND_HAS_MTLS_IDENTITY, "Whether outbound mTLS identity is configured."); + describe_gauge!(TLS_GENERATION, "Current TLS generation by consumer."); + describe_counter!(TLS_RELOAD_TOTAL, "Total TLS reload attempts, labeled by consumer and result."); + describe_histogram!(TLS_RELOAD_DURATION_SECONDS, "TLS reload duration by consumer (seconds)."); + describe_gauge!(TLS_RELOAD_GENERATION, "TLS generation after reload by consumer."); + describe_counter!(TLS_RELOAD_SKIPPED_TOTAL, "Total skipped TLS reloads, labeled by consumer and reason."); + describe_counter!(TLS_PUBLICATION_FAIL_TOTAL, "Total TLS publication failures by consumer."); + describe_counter!(TLS_CONSUMER_STALE_GENERATION_TOTAL, "Total stale TLS consumer generations observed."); +} diff --git a/crates/tls-runtime/src/outbound.rs b/crates/tls-runtime/src/outbound.rs new file mode 100644 index 000000000..b1bc95f14 --- /dev/null +++ b/crates/tls-runtime/src/outbound.rs @@ -0,0 +1,67 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::material::OutboundTlsMaterial; +use crate::metrics::record_outbound_tls_publication; +use crate::state::TlsGeneration; +use rustfs_common::{ + GLOBAL_MTLS_IDENTITY, GLOBAL_ROOT_CERT, MtlsIdentityPem, get_global_outbound_tls_generation, set_global_mtls_identity, + set_global_outbound_tls_generation, set_global_root_cert, +}; + +#[derive(Debug, Clone)] +pub struct GlobalPublishedOutboundTlsState { + pub generation: TlsGeneration, + pub root_ca_pem: Option>, + pub mtls_identity: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct GlobalOutboundTlsStateSummary { + pub generation: TlsGeneration, + pub has_root_ca: bool, + pub has_mtls_identity: bool, +} + +pub async fn publish_global_outbound_tls_state(generation: TlsGeneration, material: &OutboundTlsMaterial) { + if !material.root_ca_pem.is_empty() { + set_global_root_cert(material.root_ca_pem.clone()).await; + } else { + *GLOBAL_ROOT_CERT.write().await = None; + } + set_global_mtls_identity(material.mtls_identity.clone()).await; + set_global_outbound_tls_generation(generation.0); + record_outbound_tls_publication(generation.0, !material.root_ca_pem.is_empty(), material.mtls_identity.is_some()); +} + +pub async fn load_global_outbound_tls_state() -> GlobalPublishedOutboundTlsState { + GlobalPublishedOutboundTlsState { + generation: TlsGeneration(get_global_outbound_tls_generation()), + root_ca_pem: GLOBAL_ROOT_CERT.read().await.clone(), + mtls_identity: GLOBAL_MTLS_IDENTITY.read().await.clone(), + } +} + +pub fn load_global_outbound_tls_generation() -> TlsGeneration { + TlsGeneration(get_global_outbound_tls_generation()) +} + +pub async fn summarize_global_outbound_tls_state() -> GlobalOutboundTlsStateSummary { + let state = load_global_outbound_tls_state().await; + GlobalOutboundTlsStateSummary { + generation: state.generation, + has_root_ca: state.root_ca_pem.as_ref().is_some_and(|pem| !pem.is_empty()), + has_mtls_identity: state.mtls_identity.is_some(), + } +} diff --git a/crates/protocols/src/tls_hot_reload.rs b/crates/tls-runtime/src/server.rs similarity index 59% rename from crates/protocols/src/tls_hot_reload.rs rename to crates/tls-runtime/src/server.rs index f1dd2f1d7..088d1d721 100644 --- a/crates/protocols/src/tls_hot_reload.rs +++ b/crates/tls-runtime/src/server.rs @@ -12,19 +12,19 @@ // See the License for the specific language governing permissions and // limitations under the License. -use rustfs_config::{ - DEFAULT_TLS_RELOAD_ENABLE, DEFAULT_TLS_RELOAD_INTERVAL, ENV_TLS_RELOAD_ENABLE, ENV_TLS_RELOAD_INTERVAL, RUSTFS_TLS_CERT, - RUSTFS_TLS_KEY, -}; +use crate::certs::{CertDirectoryLoadOptions, load_all_certs_from_directory}; +use crate::config::TlsReloadOptions; +use crate::error::TlsRuntimeError; +use crate::material::{ServerTlsMaterial, server_material_fingerprint}; +use crate::metrics::{record_tls_generation, record_tls_publication_fail, record_tls_reload_result, record_tls_reload_skipped}; +use crate::source::TlsSource; use rustls::pki_types::{CertificateDer, PrivateKeyDer}; use rustls::server::{ClientHello, ResolvesServerCert, ResolvesServerCertUsingSni}; use rustls::sign::CertifiedKey; use std::collections::HashMap; -use std::collections::hash_map::DefaultHasher; -use std::hash::Hasher; -use std::io::{self, Error}; +use std::io; +use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, RwLock}; -use std::time::Duration; use tokio::sync::watch; use tokio::task::JoinHandle; use tokio::time::MissedTickBehavior; @@ -35,16 +35,18 @@ struct ResolverState { cert_resolver: ResolvesServerCertUsingSni, default_cert: Option>, cert_count: usize, - fingerprint: u64, + fingerprint: crate::fingerprint::TlsFingerprint, } impl ResolverState { - fn load_from_directory(cert_dir: &str) -> io::Result { - let cert_key_pairs = rustfs_utils::load_all_certs_from_directory( - rustfs_utils::CertDirectoryLoadOptions::builder(cert_dir, RUSTFS_TLS_CERT, RUSTFS_TLS_KEY).build(), + fn load_from_source(source: &TlsSource) -> Result { + let base_dir = source.validate_directory()?; + let cert_key_pairs = load_all_certs_from_directory( + CertDirectoryLoadOptions::builder(base_dir, &source.layout.server_cert_filename, &source.layout.server_key_filename) + .build(), )?; if cert_key_pairs.is_empty() { - return Err(Error::other("No valid certificates found in directory")); + return Err(TlsRuntimeError::Material("No valid certificates found in directory".to_string())); } Self::from_cert_key_pairs(cert_key_pairs) @@ -52,17 +54,23 @@ impl ResolverState { fn from_cert_key_pairs( cert_key_pairs: HashMap>, PrivateKeyDer<'static>)>, - ) -> io::Result { + ) -> Result { let cert_count = cert_key_pairs.len(); let mut cert_resolver = ResolvesServerCertUsingSni::new(); let mut default_cert = None; let mut entries = cert_key_pairs.into_iter().collect::>(); entries.sort_by(|(left_domain, _), (right_domain, _)| left_domain.cmp(right_domain)); - let fingerprint = fingerprint_tls_entries(&entries); + let material = ServerTlsMaterial::MultiCert { + cert_key_pairs: entries + .iter() + .map(|(domain, (certs, key))| (domain.clone(), (certs.clone(), key.clone_key()))) + .collect(), + }; + let fingerprint = server_material_fingerprint(&material); for (domain, (certs, key)) in entries { let signing_key = rustls::crypto::aws_lc_rs::sign::any_supported_type(&key) - .map_err(|e| Error::other(format!("unsupported private key type for {domain}: {e:?}")))?; + .map_err(|e| io::Error::other(format!("unsupported private key type for {domain}: {e:?}")))?; let certified_key = CertifiedKey::new(certs, signing_key); if domain.as_str() == "default" { @@ -70,7 +78,7 @@ impl ResolverState { } else { cert_resolver .add(&domain, certified_key) - .map_err(|e| Error::other(format!("failed to add certificate for {domain}: {e:?}")))?; + .map_err(|e| io::Error::other(format!("failed to add certificate for {domain}: {e:?}")))?; } } @@ -83,39 +91,30 @@ impl ResolverState { } } -fn fingerprint_tls_entries(entries: &[(String, (Vec>, PrivateKeyDer<'static>))]) -> u64 { - let mut hasher = DefaultHasher::new(); - - for (domain, (certs, key)) in entries { - hasher.write_usize(domain.len()); - hasher.write(domain.as_bytes()); - hasher.write_usize(certs.len()); - for cert in certs { - hasher.write_usize(cert.as_ref().len()); - hasher.write(cert.as_ref()); - } - hasher.write_usize(key.secret_der().len()); - hasher.write(key.secret_der()); - } - - hasher.finish() -} - #[derive(Debug)] -pub(crate) struct ReloadableCertResolver { +pub struct ReloadableServerCertResolver { + source: TlsSource, current: RwLock, + generation: AtomicU64, } -impl ReloadableCertResolver { - pub(crate) fn load_from_directory(cert_dir: &str) -> io::Result> { - let state = ResolverState::load_from_directory(cert_dir)?; +impl ReloadableServerCertResolver { + pub fn load_from_source(source: TlsSource) -> Result, TlsRuntimeError> { + let state = ResolverState::load_from_source(&source)?; + record_tls_generation("server_resolver", 1); Ok(Arc::new(Self { + source, current: RwLock::new(state), + generation: AtomicU64::new(1), })) } - pub(crate) fn reload_from_directory(&self, cert_dir: &str) -> io::Result> { - let new_state = ResolverState::load_from_directory(cert_dir)?; + pub fn load_from_directory(cert_dir: &str) -> Result, TlsRuntimeError> { + Self::load_from_source(TlsSource::from_directory(cert_dir)) + } + + pub fn reload(&self) -> Result, TlsRuntimeError> { + let new_state = ResolverState::load_from_source(&self.source)?; match self.current.write() { Ok(mut guard) => { @@ -124,6 +123,7 @@ impl ReloadableCertResolver { } let cert_count = new_state.cert_count; *guard = new_state; + self.generation.fetch_add(1, Ordering::Relaxed); Ok(Some(cert_count)) } Err(poisoned) => { @@ -133,14 +133,19 @@ impl ReloadableCertResolver { } let cert_count = new_state.cert_count; *guard = new_state; + self.generation.fetch_add(1, Ordering::Relaxed); Ok(Some(cert_count)) } } } + + pub fn generation(&self) -> u64 { + self.generation.load(Ordering::Relaxed) + } } -impl ResolvesServerCert for ReloadableCertResolver { - fn resolve(&self, client_hello: ClientHello) -> Option> { +impl ResolvesServerCert for ReloadableServerCertResolver { + fn resolve(&self, client_hello: ClientHello<'_>) -> Option> { let guard = match self.current.read() { Ok(guard) => guard, Err(poisoned) => poisoned.into_inner(), @@ -153,31 +158,26 @@ impl ResolvesServerCert for ReloadableCertResolver { } } -pub(crate) fn spawn_cert_reload_loop( +pub fn spawn_server_cert_reload_loop( protocol: &'static str, - cert_dir: String, - resolver: Arc, + resolver: Arc, + options: TlsReloadOptions, mut shutdown_rx: watch::Receiver, ) -> Option> { - let enabled = rustfs_utils::get_env_bool(ENV_TLS_RELOAD_ENABLE, DEFAULT_TLS_RELOAD_ENABLE); - if !enabled { - debug!( - protocol, - "TLS certificate hot reload is disabled (set {}=1 to enable)", ENV_TLS_RELOAD_ENABLE - ); + if !options.enabled { + debug!(protocol, "TLS certificate hot reload is disabled"); return None; } - let interval_secs = rustfs_utils::get_env_u64(ENV_TLS_RELOAD_INTERVAL, DEFAULT_TLS_RELOAD_INTERVAL).max(5); info!( protocol, - cert_dir = %cert_dir, + cert_dir = %resolver.source.base_dir.display(), "TLS certificate hot reload enabled, checking every {}s", - interval_secs + options.interval.as_secs() ); Some(tokio::spawn(async move { - let mut interval = tokio::time::interval(Duration::from_secs(interval_secs)); + let mut interval = tokio::time::interval(options.interval); interval.set_missed_tick_behavior(MissedTickBehavior::Delay); interval.tick().await; @@ -187,7 +187,7 @@ pub(crate) fn spawn_cert_reload_loop( match changed { Ok(()) => { if *shutdown_rx.borrow() { - info!(protocol, cert_dir = %cert_dir, "TLS certificate hot reload task stopped"); + info!(protocol, cert_dir = %resolver.source.base_dir.display(), "TLS certificate hot reload task stopped"); break; } continue; @@ -195,7 +195,7 @@ pub(crate) fn spawn_cert_reload_loop( Err(_) => { info!( protocol, - cert_dir = %cert_dir, + cert_dir = %resolver.source.base_dir.display(), "TLS certificate hot reload task stopped because the shutdown channel closed" ); break; @@ -205,22 +205,29 @@ pub(crate) fn spawn_cert_reload_loop( _ = interval.tick() => {} } - match resolver.reload_from_directory(&cert_dir) { + match resolver.reload() { Ok(Some(cert_count)) => { + record_tls_reload_result(protocol, "ok", None, Some(resolver.generation())); info!( protocol, - cert_dir = %cert_dir, + cert_dir = %resolver.source.base_dir.display(), cert_count, "TLS certificates reloaded successfully" ); } Ok(None) => { - debug!(protocol, cert_dir = %cert_dir, "TLS certificate material unchanged; skipping reload"); + record_tls_reload_skipped(protocol, "unchanged"); + debug!( + protocol, + cert_dir = %resolver.source.base_dir.display(), + "TLS certificate material unchanged; skipping reload" + ); } Err(e) => { + record_tls_publication_fail(protocol); warn!( protocol, - cert_dir = %cert_dir, + cert_dir = %resolver.source.base_dir.display(), "TLS certificate reload failed (will retry): {}", e ); @@ -238,10 +245,10 @@ mod tests { use tempfile::TempDir; fn cert_key_pair(san: &str) -> (Vec>, PrivateKeyDer<'static>) { - let cert = generate_simple_self_signed(vec![san.to_string()]).unwrap(); + let cert = generate_simple_self_signed(vec![san.to_string()]).expect("cert should generate"); ( vec![cert.cert.der().clone()], - PrivateKeyDer::try_from(cert.signing_key.serialize_der()).unwrap(), + PrivateKeyDer::try_from(cert.signing_key.serialize_der()).expect("key should convert"), ) } @@ -252,42 +259,44 @@ mod tests { } fn write_default_cert(dir: &std::path::Path, san: &str) { - let cert = generate_simple_self_signed(vec![san.to_string()]).unwrap(); - fs::write(dir.join(RUSTFS_TLS_CERT), cert.cert.pem()).unwrap(); - fs::write(dir.join(RUSTFS_TLS_KEY), cert.signing_key.serialize_pem()).unwrap(); + let cert = generate_simple_self_signed(vec![san.to_string()]).expect("cert should generate"); + fs::write(dir.join(rustfs_config::RUSTFS_TLS_CERT), cert.cert.pem()).expect("cert should write"); + fs::write(dir.join(rustfs_config::RUSTFS_TLS_KEY), cert.signing_key.serialize_pem()).expect("key should write"); } #[test] - fn reload_from_directory_replaces_default_certificate() { - let temp_dir = TempDir::new().unwrap(); + fn reload_replaces_default_certificate() { + let temp_dir = TempDir::new().expect("tempdir should create"); write_default_cert(temp_dir.path(), "localhost"); - let resolver = ReloadableCertResolver::load_from_directory(temp_dir.path().to_str().unwrap()).unwrap(); + let resolver = ReloadableServerCertResolver::load_from_directory(temp_dir.path().to_str().expect("path should utf8")) + .expect("resolver should load"); let before = { - let guard = resolver.current.read().unwrap(); - guard.default_cert.as_ref().unwrap().clone() + let guard = resolver.current.read().expect("lock should acquire"); + guard.default_cert.as_ref().expect("default cert should exist").clone() }; write_default_cert(temp_dir.path(), "rotated.local"); - let cert_count = resolver.reload_from_directory(temp_dir.path().to_str().unwrap()).unwrap(); + let cert_count = resolver.reload().expect("reload should succeed"); assert_eq!(cert_count, Some(1)); let after = { - let guard = resolver.current.read().unwrap(); - guard.default_cert.as_ref().unwrap().clone() + let guard = resolver.current.read().expect("lock should acquire"); + guard.default_cert.as_ref().expect("default cert should exist").clone() }; assert_ne!(before.cert[0].as_ref(), after.cert[0].as_ref()); } #[test] - fn reload_from_directory_skips_when_material_is_unchanged() { - let temp_dir = TempDir::new().unwrap(); + fn reload_skips_when_material_is_unchanged() { + let temp_dir = TempDir::new().expect("tempdir should create"); write_default_cert(temp_dir.path(), "localhost"); - let resolver = ReloadableCertResolver::load_from_directory(temp_dir.path().to_str().unwrap()).unwrap(); - let outcome = resolver.reload_from_directory(temp_dir.path().to_str().unwrap()).unwrap(); + let resolver = ReloadableServerCertResolver::load_from_directory(temp_dir.path().to_str().expect("path should utf8")) + .expect("resolver should load"); + let outcome = resolver.reload().expect("reload should succeed"); assert_eq!(outcome, None); } @@ -307,8 +316,8 @@ mod tests { second.insert("default".to_string(), clone_cert_key_pair(&default_cert)); second.insert("api.example.com".to_string(), clone_cert_key_pair(&api_cert)); - let first_state = ResolverState::from_cert_key_pairs(first).unwrap(); - let second_state = ResolverState::from_cert_key_pairs(second).unwrap(); + let first_state = ResolverState::from_cert_key_pairs(first).expect("first state should build"); + let second_state = ResolverState::from_cert_key_pairs(second).expect("second state should build"); assert_eq!(first_state.cert_count, 3); assert_eq!(second_state.cert_count, 3); diff --git a/crates/tls-runtime/src/source.rs b/crates/tls-runtime/src/source.rs new file mode 100644 index 000000000..dee0b0277 --- /dev/null +++ b/crates/tls-runtime/src/source.rs @@ -0,0 +1,94 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::error::TlsRuntimeError; +use rustfs_config::{ + RUSTFS_CA_CERT, RUSTFS_CLIENT_CA_CERT_FILENAME, RUSTFS_CLIENT_CERT_FILENAME, RUSTFS_CLIENT_KEY_FILENAME, RUSTFS_PUBLIC_CERT, + RUSTFS_TLS_CERT, RUSTFS_TLS_KEY, +}; +use std::path::{Path, PathBuf}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TlsSourceKind { + Directory, + ExplicitFiles, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TlsFileLayout { + pub server_cert_filename: String, + pub server_key_filename: String, + pub public_ca_filename: String, + pub client_ca_filename: String, + pub client_cert_filename: String, + pub client_key_filename: String, +} + +impl Default for TlsFileLayout { + fn default() -> Self { + Self { + server_cert_filename: RUSTFS_TLS_CERT.to_string(), + server_key_filename: RUSTFS_TLS_KEY.to_string(), + public_ca_filename: RUSTFS_PUBLIC_CERT.to_string(), + client_ca_filename: RUSTFS_CLIENT_CA_CERT_FILENAME.to_string(), + client_cert_filename: RUSTFS_CLIENT_CERT_FILENAME.to_string(), + client_key_filename: RUSTFS_CLIENT_KEY_FILENAME.to_string(), + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TlsSource { + pub kind: TlsSourceKind, + pub base_dir: PathBuf, + pub layout: TlsFileLayout, + pub fallback_ca_filename: String, + pub trust_system_ca: bool, + pub trust_leaf_as_ca: bool, + pub server_mtls_enabled: bool, +} + +impl TlsSource { + pub fn from_directory(base_dir: impl Into) -> Self { + Self { + kind: TlsSourceKind::Directory, + base_dir: base_dir.into(), + layout: TlsFileLayout::default(), + fallback_ca_filename: RUSTFS_CA_CERT.to_string(), + trust_system_ca: false, + trust_leaf_as_ca: false, + server_mtls_enabled: false, + } + } + + pub fn validate_directory(&self) -> Result<&Path, TlsRuntimeError> { + if self.base_dir.as_os_str().is_empty() { + return Err(TlsRuntimeError::EmptySourcePath); + } + + if !self.base_dir.exists() { + return Err(TlsRuntimeError::DirectoryNotFound { + path: self.base_dir.clone(), + }); + } + + if !self.base_dir.is_dir() { + return Err(TlsRuntimeError::NotADirectory { + path: self.base_dir.clone(), + }); + } + + Ok(self.base_dir.as_path()) + } +} diff --git a/crates/tls-runtime/src/state.rs b/crates/tls-runtime/src/state.rs new file mode 100644 index 000000000..ae2fd5fbc --- /dev/null +++ b/crates/tls-runtime/src/state.rs @@ -0,0 +1,234 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::fingerprint::TlsFingerprint; +use arc_swap::ArcSwap; +use serde::Serialize; +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; +use tokio::sync::RwLock; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct TlsGeneration(pub u64); + +#[derive(Debug)] +pub struct TlsPublishedState { + pub generation: TlsGeneration, + pub material: Arc, + pub fingerprint: TlsFingerprint, + pub loaded_at_unix_ms: u64, +} + +#[derive(Debug)] +pub struct TlsReloadRuntimeState { + pub current: ArcSwap>, + pub last_good: ArcSwap>, + pub last_attempt_unix_ms: AtomicU64, + pub last_success_unix_ms: AtomicU64, + pub last_error: RwLock>, +} + +impl TlsReloadRuntimeState { + pub fn new(initial: Arc>) -> Self { + Self { + current: ArcSwap::from(initial.clone()), + last_good: ArcSwap::from(initial), + last_attempt_unix_ms: AtomicU64::new(0), + last_success_unix_ms: AtomicU64::new(0), + last_error: RwLock::new(None), + } + } + + pub fn current_generation(&self) -> TlsGeneration { + self.current.load().generation + } + + pub fn bump_generation(&self) -> TlsGeneration { + TlsGeneration(self.current_generation().0.saturating_add(1)) + } + + pub fn mark_attempt(&self, unix_ms: u64) { + self.last_attempt_unix_ms.store(unix_ms, Ordering::Relaxed); + } + + pub fn mark_success(&self, unix_ms: u64) { + self.last_success_unix_ms.store(unix_ms, Ordering::Relaxed); + } + + pub fn last_attempt_unix_ms(&self) -> u64 { + self.last_attempt_unix_ms.load(Ordering::Relaxed) + } + + pub fn last_success_unix_ms(&self) -> u64 { + self.last_success_unix_ms.load(Ordering::Relaxed) + } +} + +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +pub struct TlsRuntimeStatusSnapshot { + pub runtime: TlsRuntimeRuntimeSection, + pub outbound: TlsRuntimeOutboundSection, + pub server: TlsRuntimeServerSection, + pub consumer: TlsRuntimeConsumerSection, +} + +impl TlsRuntimeStatusSnapshot { + pub fn is_complete(&self) -> bool { + self.server.has_material || self.outbound.has_roots || self.outbound.has_mtls_identity + } + + pub fn from_outbound_only(args: OutboundOnlySnapshotArgs) -> Self { + Self { + runtime: TlsRuntimeRuntimeSection { + generation: args.generation, + reload_enabled: args.reload_enabled, + detect_mode: args.detect_mode, + last_attempt_time: args.last_attempt_time, + last_success_time: args.last_success_time, + last_error: args.last_error, + source_path: args.source_path, + }, + outbound: TlsRuntimeOutboundSection { + has_roots: args.has_roots, + has_mtls_identity: args.has_mtls_identity, + }, + server: TlsRuntimeServerSection { has_material: false }, + consumer: TlsRuntimeConsumerSection { stale_generation: false }, + } + } +} + +#[derive(Debug, Clone)] +pub struct OutboundOnlySnapshotArgs { + pub source_path: String, + pub generation: u64, + pub reload_enabled: bool, + pub detect_mode: &'static str, + pub last_attempt_time: Option, + pub last_success_time: Option, + pub last_error: Option, + pub has_roots: bool, + pub has_mtls_identity: bool, +} + +pub fn detect_mode_label(mode: crate::config::ReloadDetectMode) -> &'static str { + match mode { + crate::config::ReloadDetectMode::Poll => "poll", + crate::config::ReloadDetectMode::Watch => "watch", + crate::config::ReloadDetectMode::Hybrid => "hybrid", + } +} + +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +pub struct TlsRuntimeRuntimeSection { + pub generation: u64, + pub reload_enabled: bool, + pub detect_mode: &'static str, + pub last_attempt_time: Option, + pub last_success_time: Option, + pub last_error: Option, + pub source_path: String, +} + +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +pub struct TlsRuntimeOutboundSection { + pub has_roots: bool, + pub has_mtls_identity: bool, +} + +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +pub struct TlsRuntimeServerSection { + pub has_material: bool, +} + +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +pub struct TlsRuntimeConsumerSection { + pub stale_generation: bool, +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::Arc; + + fn make_published(generation: u64, fingerprint_bytes: &[u8]) -> Arc> { + Arc::new(TlsPublishedState { + generation: TlsGeneration(generation), + material: Arc::new("test".to_string()), + fingerprint: TlsFingerprint::from_optional_bytes(Some(fingerprint_bytes), None, None, None, None), + loaded_at_unix_ms: 0, + }) + } + + #[test] + fn runtime_state_tracks_generation_and_timestamps() { + let initial = make_published(1, b"aaa"); + let state = TlsReloadRuntimeState::new(initial); + + assert_eq!(state.current_generation(), TlsGeneration(1)); + assert_eq!(state.bump_generation(), TlsGeneration(2)); + assert_eq!(state.last_attempt_unix_ms(), 0); + assert_eq!(state.last_success_unix_ms(), 0); + + state.mark_attempt(100); + assert_eq!(state.last_attempt_unix_ms(), 100); + + state.mark_success(200); + assert_eq!(state.last_success_unix_ms(), 200); + } + + #[test] + fn bump_generation_saturates_at_max() { + let initial = Arc::new(TlsPublishedState { + generation: TlsGeneration(u64::MAX), + material: Arc::new("max".to_string()), + fingerprint: TlsFingerprint::default(), + loaded_at_unix_ms: 0, + }); + let state = TlsReloadRuntimeState::new(initial); + assert_eq!(state.bump_generation(), TlsGeneration(u64::MAX)); + } + + #[test] + fn status_snapshot_is_complete_with_outbound_roots() { + let snap = TlsRuntimeStatusSnapshot::from_outbound_only(OutboundOnlySnapshotArgs { + source_path: "/tmp".to_string(), + generation: 1, + reload_enabled: true, + detect_mode: "poll", + last_attempt_time: None, + last_success_time: None, + last_error: None, + has_roots: true, + has_mtls_identity: false, + }); + assert!(snap.is_complete()); + } + + #[test] + fn status_snapshot_is_not_complete_when_empty() { + let snap = TlsRuntimeStatusSnapshot::from_outbound_only(OutboundOnlySnapshotArgs { + source_path: "/tmp".to_string(), + generation: 1, + reload_enabled: true, + detect_mode: "poll", + last_attempt_time: None, + last_success_time: None, + last_error: None, + has_roots: false, + has_mtls_identity: false, + }); + assert!(!snap.is_complete()); + } +} diff --git a/crates/utils/Cargo.toml b/crates/utils/Cargo.toml index d7ecdd63e..2257c5d06 100644 --- a/crates/utils/Cargo.toml +++ b/crates/utils/Cargo.toml @@ -32,22 +32,17 @@ bytes = { workspace = true, optional = true } crc-fast = { workspace = true, optional = true } flate2 = { workspace = true, optional = true } futures = { workspace = true, optional = true } -hashbrown = { workspace = true, optional = true } hex-simd = { workspace = true, optional = true } highway = { workspace = true, optional = true } hmac = { workspace = true, optional = true } http = { workspace = true, optional = true } hyper = { workspace = true, optional = true } -libc = { workspace = true, optional = true } local-ip-address = { workspace = true, optional = true } lz4 = { workspace = true, optional = true } md-5 = { workspace = true, optional = true } netif = { workspace = true, optional = true } regex = { workspace = true, optional = true } rustix = { workspace = true, optional = true } -rustls = { workspace = true, optional = true } -rustls-pki-types = { workspace = true, optional = true } -s3s = { workspace = true, optional = true } serde = { workspace = true, optional = true } sha1 = { workspace = true, optional = true } sha2 = { workspace = true, optional = true } @@ -55,7 +50,6 @@ convert_case = { workspace = true, optional = true } siphasher = { workspace = true, optional = true } snap = { workspace = true, optional = true } tempfile = { workspace = true, optional = true } -thiserror = { workspace = true, optional = true } tokio = { workspace = true, optional = true, features = ["io-util", "macros"] } tracing = { workspace = true } transform-stream = { workspace = true, optional = true } @@ -63,7 +57,6 @@ url = { workspace = true, optional = true } zstd = { workspace = true, optional = true } [dev-dependencies] -rcgen = { workspace = true } tempfile = { workspace = true } tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } temp-env = { workspace = true } @@ -77,11 +70,9 @@ workspace = true [features] default = ["ip"] # features that are enabled by default ip = ["dep:local-ip-address"] # ip characteristics and their dependencies -tls = ["dep:rustls", "dep:rustls-pki-types"] # tls characteristics and their dependencies -net = ["ip", "dep:url", "dep:netif", "dep:futures", "dep:transform-stream", "dep:bytes", "dep:s3s", "dep:hyper", "dep:thiserror", "dep:tokio"] # network features with DNS resolver +net = ["ip", "dep:url", "dep:netif", "dep:futures", "dep:transform-stream", "dep:bytes", "dep:hyper", "dep:tokio"] # network features with DNS resolver io = ["dep:tokio"] path = [] # path manipulation features -notify = ["dep:hyper", "dep:s3s", "dep:hashbrown", "dep:thiserror", "dep:serde", "dep:libc", "dep:url", "dep:regex"] # file system notification features compress = ["dep:flate2", "dep:brotli", "dep:snap", "dep:lz4", "dep:zstd"] string = ["dep:regex"] crypto = ["dep:base64-simd", "dep:hex-simd", "dep:hmac", "dep:hyper", "dep:sha1", "dep:sha2"] @@ -90,4 +81,4 @@ os = ["dep:rustix", "dep:tempfile", "dep:windows"] # operating system utilities integration = [] # integration test features http = ["dep:convert_case", "dep:http", "dep:regex"] obj = ["http"] # object storage features -full = ["ip", "tls", "net", "io", "hash", "os", "integration", "path", "crypto", "string", "compress", "notify", "http", "obj"] # all features +full = ["ip", "net", "io", "hash", "os", "integration", "path", "crypto", "string", "compress", "http", "obj"] # all features diff --git a/crates/utils/src/lib.rs b/crates/utils/src/lib.rs index 722b81b43..70ee86461 100644 --- a/crates/utils/src/lib.rs +++ b/crates/utils/src/lib.rs @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#[cfg(feature = "tls")] -pub mod certs; #[cfg(feature = "ip")] pub mod ip; #[cfg(feature = "net")] @@ -52,9 +50,6 @@ pub mod compress; #[cfg(feature = "path")] pub mod dirs; -#[cfg(feature = "tls")] -pub use certs::*; - #[cfg(feature = "hash")] pub use hash::*; @@ -70,12 +65,6 @@ pub use crypto::*; #[cfg(feature = "compress")] pub use compress::*; -#[cfg(feature = "notify")] -mod notify; - -#[cfg(feature = "notify")] -pub use notify::*; - #[cfg(feature = "obj")] pub mod obj; diff --git a/crates/utils/src/notify/mod.rs b/crates/utils/src/notify/mod.rs deleted file mode 100644 index 1d0ddb4d0..000000000 --- a/crates/utils/src/notify/mod.rs +++ /dev/null @@ -1,205 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -mod net; - -use hashbrown::HashMap; -use hyper::HeaderMap; -use s3s::{S3Request, S3Response}; - -pub use net::*; - -/// Extract request parameters from S3Request, mainly header information. -pub fn extract_req_params(req: &S3Request) -> HashMap { - extract_params_header(&req.headers) -} - -/// Extract request parameters from hyper::HeaderMap, mainly header information. -/// This function is useful when you have a raw HTTP request and need to extract parameters. -#[deprecated(since = "0.1.0", note = "Use extract_params_header instead")] -pub fn extract_req_params_header(head: &HeaderMap) -> HashMap { - extract_params_header(head) -} - -/// Extract parameters from hyper::HeaderMap, mainly header information. -/// This function is useful when you have a raw HTTP request and need to extract parameters. -pub fn extract_params_header(head: &HeaderMap) -> HashMap { - let mut params = HashMap::new(); - for (key, value) in head.iter() { - if let Ok(val_str) = value.to_str() { - params.insert(key.as_str().to_string(), val_str.to_string()); - } - } - params -} - -/// Extract response elements from S3Response, mainly header information. -pub fn extract_resp_elements(resp: &S3Response) -> HashMap { - extract_params_header(&resp.headers) -} - -/// Get host from header information. -pub fn get_request_host(headers: &HeaderMap) -> String { - headers - .get("host") - .and_then(|v| v.to_str().ok()) - .unwrap_or_default() - .to_string() -} - -/// Get Port from header information. -/// Priority: -/// 1. x-forwarded-port -/// 2. host header (parse port) -/// If host has no port, try to deduce from x-forwarded-proto (http->80, https->443) -/// 3. port header -/// -/// If the port cannot be determined, returns 0. -pub fn get_request_port(headers: &HeaderMap) -> u16 { - // 1. Try x-forwarded-port - if let Some(port) = headers - .get("x-forwarded-port") - .and_then(|v| v.to_str().ok()) - .and_then(|s| s.parse::().ok()) - { - return port; - } - - // 2. Try host header - if let Some(host) = headers.get("host").and_then(|v| v.to_str().ok()) { - if let Some(idx) = host.rfind(':') { - // Check if it's an IPv6 address with port, e.g., [::1]:8080 - // If ']' is present, the colon must be after it. - let valid_colon = match host.rfind(']') { - Some(close_bracket_idx) => idx > close_bracket_idx, - None => true, - }; - - if valid_colon - && let Ok(port) = host[idx + 1..].parse::() - && port > 0 - { - return port; - } - } - - // If host is present but no port found (or parsing failed, or port is 0), - // try to deduce from x-forwarded-proto - if let Some(proto) = headers.get("x-forwarded-proto").and_then(|v| v.to_str().ok()) { - match proto { - "http" => return 80, - "https" => return 443, - _ => {} - } - } - } - - // 3. Fallback to "port" header - headers - .get("port") - .and_then(|v| v.to_str().ok()) - .and_then(|s| s.parse::().ok()) - .unwrap_or(0) -} - -/// Get content-length from header information. -pub fn get_request_content_length(headers: &HeaderMap) -> u64 { - headers - .get("content-length") - .and_then(|v| v.to_str().ok()) - .and_then(|s| s.parse::().ok()) - .unwrap_or(0) -} - -/// Get referer from header information. -/// If the referer header is not present, returns an empty string. -pub fn get_request_referer(headers: &HeaderMap) -> String { - headers - .get("referer") - .and_then(|v| v.to_str().ok()) - .unwrap_or_default() - .to_string() -} - -/// Get user-agent from header information. -pub fn get_request_user_agent(headers: &HeaderMap) -> String { - headers - .get("user-agent") - .and_then(|v| v.to_str().ok()) - .unwrap_or_default() - .to_string() -} - -#[cfg(test)] -mod tests { - use super::*; - use hyper::header::HeaderValue; - - #[test] - fn test_get_request_port() { - let mut headers = HeaderMap::new(); - - // Case 1: No port info - assert_eq!(get_request_port(&headers), 0); - - // Case 2: port header - headers.insert("port", HeaderValue::from_static("8080")); - assert_eq!(get_request_port(&headers), 8080); - - // Case 3: host header with port - headers.remove("port"); - headers.insert("host", HeaderValue::from_static("example.com:9000")); - assert_eq!(get_request_port(&headers), 9000); - - // Case 4: host header without port, no proto - headers.insert("host", HeaderValue::from_static("example.com")); - assert_eq!(get_request_port(&headers), 0); - - // Case 5: IPv6 host with port - headers.insert("host", HeaderValue::from_static("[::1]:9001")); - assert_eq!(get_request_port(&headers), 9001); - - // Case 6: IPv6 host without port - headers.insert("host", HeaderValue::from_static("[::1]")); - assert_eq!(get_request_port(&headers), 0); - - // Case 7: x-forwarded-port - headers.insert("x-forwarded-port", HeaderValue::from_static("7000")); - // Even if host is present, x-forwarded-port takes precedence - assert_eq!(get_request_port(&headers), 7000); - - // Case 8: host without port, but x-forwarded-proto is http - headers.remove("x-forwarded-port"); - headers.insert("host", HeaderValue::from_static("example.com")); - headers.insert("x-forwarded-proto", HeaderValue::from_static("http")); - assert_eq!(get_request_port(&headers), 80); - - // Case 9: host without port, but x-forwarded-proto is https - headers.insert("x-forwarded-proto", HeaderValue::from_static("https")); - assert_eq!(get_request_port(&headers), 443); - - // Case 10: host without port, unknown proto - headers.insert("x-forwarded-proto", HeaderValue::from_static("ftp")); - assert_eq!(get_request_port(&headers), 0); - - // Case 11: host with port 0, should fallback to proto - headers.insert("host", HeaderValue::from_static("example.com:0")); - headers.insert("x-forwarded-proto", HeaderValue::from_static("https")); - assert_eq!(get_request_port(&headers), 443); - - // Case 12: host with port 0, no proto - headers.remove("x-forwarded-proto"); - assert_eq!(get_request_port(&headers), 0); - } -} diff --git a/rustfs/Cargo.toml b/rustfs/Cargo.toml index 2fa4f2dda..ce6a1078b 100644 --- a/rustfs/Cargo.toml +++ b/rustfs/Cargo.toml @@ -85,6 +85,7 @@ rustfs-data-usage = { workspace = true } rustfs-s3select-api = { workspace = true } rustfs-s3select-query = { workspace = true } rustfs-targets = { workspace = true } +rustfs-tls-runtime = { workspace = true } rustfs-trusted-proxies = { workspace = true } rustfs-utils = { workspace = true, features = ["full"] } rustfs-zip = { workspace = true } @@ -127,6 +128,7 @@ serde_urlencoded = { workspace = true } # Cryptography and Security rustls = { workspace = true } +rustls-pki-types = { workspace = true } subtle = { workspace = true } jiff = { workspace = true } time = { workspace = true, features = ["parsing", "formatting", "serde"] } @@ -173,9 +175,7 @@ libsystemd.workspace = true [target.'cfg(not(all(target_os = "linux", target_env = "gnu", target_arch = "x86_64")))'.dependencies] mimalloc = { workspace = true } -libmimalloc-sys = { version = "0.1.48", features = ["extended"] } - - +libmimalloc-sys = { version = "0.1.49", features = ["extended"] } # Only enable pprof-based profiling on linux + gnu + x86_64. [target.'cfg(all(target_os = "linux", target_env = "gnu", target_arch = "x86_64"))'.dependencies] diff --git a/rustfs/src/admin/handlers/mod.rs b/rustfs/src/admin/handlers/mod.rs index e5285ab0f..ee95db6c0 100644 --- a/rustfs/src/admin/handlers/mod.rs +++ b/rustfs/src/admin/handlers/mod.rs @@ -45,6 +45,7 @@ pub mod sts; pub mod system; mod target_descriptor; pub mod tier; +pub mod tls_debug; pub mod trace; pub mod user; pub mod user_iam; @@ -75,6 +76,7 @@ mod tests { let _metrics_handler = metrics::MetricsHandler {}; let _profile_handler = profile_admin::ProfileHandler {}; let _profile_status_handler = profile_admin::ProfileStatusHandler {}; + let _tls_status_handler = tls_debug::TlsStatusHandler {}; let _heal_handler = heal::HealHandler {}; let _bg_heal_handler = heal::BackgroundHealStatusHandler {}; let _replication_metrics_handler = replication::GetReplicationMetricsHandler {}; diff --git a/rustfs/src/admin/handlers/site_replication.rs b/rustfs/src/admin/handlers/site_replication.rs index eb9f63c2d..7a105be82 100644 --- a/rustfs/src/admin/handlers/site_replication.rs +++ b/rustfs/src/admin/handlers/site_replication.rs @@ -29,10 +29,7 @@ use http::header::{CONTENT_TYPE, HOST}; use http::{HeaderMap, HeaderValue, Uri}; use hyper::{Method, StatusCode}; use matchit::Params; -use rustfs_config::{ - DEFAULT_DELIMITER, DEFAULT_RUSTFS_TLS_PATH, DEFAULT_TRUST_LEAF_CERT_AS_CA, ENV_RUSTFS_TLS_PATH, ENV_TRUST_LEAF_CERT_AS_CA, - MAX_ADMIN_REQUEST_BODY_SIZE, RUSTFS_CA_CERT, RUSTFS_TLS_CERT, -}; +use rustfs_config::{DEFAULT_DELIMITER, DEFAULT_RUSTFS_TLS_PATH, ENV_RUSTFS_TLS_PATH, MAX_ADMIN_REQUEST_BODY_SIZE}; use rustfs_ecstore::bucket::bucket_target_sys::BucketTargetSys; use rustfs_ecstore::bucket::metadata::{ BUCKET_CORS_CONFIG, BUCKET_LIFECYCLE_CONFIG, BUCKET_POLICY_CONFIG, BUCKET_QUOTA_CONFIG_FILE, BUCKET_REPLICATION_CONFIG, @@ -67,7 +64,9 @@ use rustfs_policy::policy::{ }; use rustfs_signer::constants::UNSIGNED_PAYLOAD; use rustfs_signer::sign_v4; +use rustfs_tls_runtime::load_global_outbound_tls_state; use rustfs_utils::http::get_source_scheme; +use rustls_pki_types::pem::PemObject; use s3s::dto::{ BucketVersioningStatus, DeleteMarkerReplication, DeleteMarkerReplicationStatus, DeleteReplication, DeleteReplicationStatus, Destination, ExistingObjectReplication, ExistingObjectReplicationStatus, ReplicationConfiguration, ReplicationRule, @@ -80,9 +79,9 @@ use serde::de::DeserializeOwned; use serde_json::Value; use sha2::{Digest, Sha256}; use std::collections::{BTreeMap, HashMap, HashSet}; -use std::sync::OnceLock; use std::time::{Duration, Instant}; use time::OffsetDateTime; +use tokio::sync::OnceCell; use tracing::warn; use url::{Url, form_urlencoded}; use uuid::Uuid; @@ -103,7 +102,7 @@ const SITE_REPLICATOR_SERVICE_ACCOUNT: &str = "site-replicator-0"; const SITE_REPLICATION_PEER_JOIN_PATH: &str = "/rustfs/admin/v3/site-replication/peer/join"; const SITE_REPLICATION_PEER_EDIT_PATH: &str = "/rustfs/admin/v3/site-replication/peer/edit"; const SITE_REPLICATION_PEER_REMOVE_PATH: &str = "/rustfs/admin/v3/site-replication/peer/remove"; -static SITE_REPLICATION_PEER_CLIENT: OnceLock> = OnceLock::new(); +static SITE_REPLICATION_PEER_CLIENT: OnceCell> = OnceCell::const_new(); #[derive(Debug, Clone, Serialize, Deserialize, Default)] struct SiteReplicationState { @@ -448,56 +447,32 @@ async fn persist_site_replication_state(state: &SiteReplicationState) -> S3Resul } } -fn add_root_certificates_from_file( - mut builder: reqwest::ClientBuilder, - cert_path: &std::path::Path, - description: &str, -) -> S3Result { - if !cert_path.exists() { - return Ok(builder); - } - - std::fs::read(cert_path).map_err(|e| { - S3Error::with_message( - S3ErrorCode::InternalError, - format!("failed to read {description} {}: {e}", cert_path.display()), - ) - })?; - - let certs_der = rustfs_utils::load_cert_bundle_der_bytes(cert_path.to_string_lossy().as_ref()).map_err(|e| { - S3Error::with_message( - S3ErrorCode::InternalError, - format!("failed to parse {description} {}: {e}", cert_path.display()), - ) - })?; - - for cert_der in certs_der { - let cert = reqwest::Certificate::from_der(&cert_der).map_err(|e| { - S3Error::with_message( - S3ErrorCode::InternalError, - format!("failed to load {description} {}: {e}", cert_path.display()), - ) - })?; - builder = builder.add_root_certificate(cert); - } - - Ok(builder) -} - -fn build_site_replication_peer_client() -> S3Result { +async fn build_site_replication_peer_client() -> S3Result { let mut builder = reqwest::Client::builder() .timeout(SITE_REPLICATION_PEER_REQUEST_TIMEOUT) .connect_timeout(SITE_REPLICATION_PEER_CONNECT_TIMEOUT) .pool_idle_timeout(Some(Duration::from_secs(60))); - let tls_path = rustfs_utils::get_env_str(ENV_RUSTFS_TLS_PATH, DEFAULT_RUSTFS_TLS_PATH); - if !tls_path.is_empty() { - let tls_dir = std::path::Path::new(&tls_path); - builder = add_root_certificates_from_file(builder, &tls_dir.join(RUSTFS_CA_CERT), "site-replication CA cert")?; + let outbound_tls = load_global_outbound_tls_state().await; + if let Some(root_ca_pem) = outbound_tls.root_ca_pem.as_ref() { + let mut reader = std::io::BufReader::new(root_ca_pem.as_slice()); + let certs_der = rustls_pki_types::CertificateDer::pem_reader_iter(&mut reader) + .collect::, _>>() + .map_err(|e| { + S3Error::with_message( + S3ErrorCode::InternalError, + format!("failed to parse published site-replication CA certs: {e}"), + ) + })?; - if rustfs_utils::get_env_bool(ENV_TRUST_LEAF_CERT_AS_CA, DEFAULT_TRUST_LEAF_CERT_AS_CA) { - builder = - add_root_certificates_from_file(builder, &tls_dir.join(RUSTFS_TLS_CERT), "site-replication leaf cert as CA")?; + for cert_der in certs_der { + let cert = reqwest::Certificate::from_der(cert_der.as_ref()).map_err(|e| { + S3Error::with_message( + S3ErrorCode::InternalError, + format!("failed to load published site-replication CA cert: {e}"), + ) + })?; + builder = builder.add_root_certificate(cert); } } @@ -506,8 +481,10 @@ fn build_site_replication_peer_client() -> S3Result { .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("build site replication peer client failed: {e}"))) } -fn site_replication_peer_client() -> S3Result<&'static reqwest::Client> { - let result = SITE_REPLICATION_PEER_CLIENT.get_or_init(|| build_site_replication_peer_client().map_err(|e| e.to_string())); +async fn site_replication_peer_client() -> S3Result<&'static reqwest::Client> { + let result = SITE_REPLICATION_PEER_CLIENT + .get_or_init(|| async { build_site_replication_peer_client().await.map_err(|e| e.to_string()) }) + .await; result.as_ref().map_err(|err| { S3Error::with_message( S3ErrorCode::InternalError, @@ -969,7 +946,7 @@ async fn send_peer_admin_request( .unwrap_or("us-east-1"), ); - let mut req = site_replication_peer_client()?.request(reqwest::Method::PUT, &url); + let mut req = site_replication_peer_client().await?.request(reqwest::Method::PUT, &url); for (name, value) in signed.headers() { req = req.header(name, value); } diff --git a/rustfs/src/admin/handlers/tls_debug.rs b/rustfs/src/admin/handlers/tls_debug.rs new file mode 100644 index 000000000..fef4cb7d0 --- /dev/null +++ b/rustfs/src/admin/handlers/tls_debug.rs @@ -0,0 +1,154 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::profile::authorize_profile_request; +use crate::admin::router::{AdminOperation, Operation, S3Router}; +use crate::server::ADMIN_PREFIX; +use http::StatusCode; +use http::{HeaderMap, HeaderValue}; +use hyper::Method; +use matchit::Params; +use rustfs_tls_runtime::{ + OutboundOnlySnapshotArgs, TlsConsumerStatusItem, TlsDebugStatusResponse, TlsRuntimeStatusSnapshot, + summarize_global_outbound_tls_state, +}; +use s3s::header::CONTENT_TYPE; +use s3s::{Body, S3Error, S3ErrorCode, S3Request, S3Response, S3Result}; + +pub fn register_tls_debug_route(r: &mut S3Router) -> std::io::Result<()> { + r.insert( + Method::GET, + format!("{}{}", ADMIN_PREFIX, "/debug/tls/status").as_str(), + AdminOperation(&TlsStatusHandler {}), + )?; + Ok(()) +} + +pub struct TlsStatusHandler {} + +const PROTOS_GRPC_CHANNEL_CONSUMER: &str = "protos_grpc_channel"; +const RIO_HTTP_READER_CONSUMER: &str = "rio_http_reader"; +const ECSTORE_TRANSITION_CLIENT_CONSUMER: &str = "ecstore_transition_client"; + +#[async_trait::async_trait] +impl Operation for TlsStatusHandler { + async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { + authorize_profile_request(&req).await?; + + let tls_path = rustfs_utils::get_env_opt_str(rustfs_config::ENV_RUSTFS_TLS_PATH).unwrap_or_default(); + let outbound = summarize_global_outbound_tls_state().await; + let status = TlsRuntimeStatusSnapshot::from_outbound_only(OutboundOnlySnapshotArgs { + source_path: tls_path, + generation: outbound.generation.0, + reload_enabled: rustfs_utils::get_env_bool( + rustfs_config::ENV_TLS_RELOAD_ENABLE, + rustfs_config::DEFAULT_TLS_RELOAD_ENABLE, + ), + detect_mode: "poll", + last_attempt_time: None, + last_success_time: None, + last_error: None, + has_roots: outbound.has_root_ca, + has_mtls_identity: outbound.has_mtls_identity, + }); + let payload = TlsDebugStatusResponse::builder(status) + .push_consumers([ + TlsConsumerStatusItem { + consumer: PROTOS_GRPC_CHANNEL_CONSUMER, + generation: outbound.generation.0, + has_root_ca: outbound.has_root_ca, + has_mtls_identity: outbound.has_mtls_identity, + }, + TlsConsumerStatusItem { + consumer: RIO_HTTP_READER_CONSUMER, + generation: outbound.generation.0, + has_root_ca: outbound.has_root_ca, + has_mtls_identity: outbound.has_mtls_identity, + }, + TlsConsumerStatusItem { + consumer: ECSTORE_TRANSITION_CLIENT_CONSUMER, + generation: outbound.generation.0, + has_root_ca: outbound.has_root_ca, + has_mtls_identity: outbound.has_mtls_identity, + }, + ]) + .build(); + let body = serde_json::to_vec(&payload) + .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("serialize tls status failed: {e}")))?; + + let mut headers = HeaderMap::new(); + headers.insert(CONTENT_TYPE, HeaderValue::from_static("application/json")); + Ok(S3Response::with_headers((StatusCode::OK, Body::from(body)), headers)) + } +} + +#[cfg(test)] +mod tests { + use super::TlsStatusHandler; + use crate::admin::router::Operation; + use http::{Extensions, HeaderMap, Uri}; + use hyper::Method; + use matchit::Params; + use rustfs_tls_runtime::{OutboundOnlySnapshotArgs, TlsConsumerStatusItem, TlsRuntimeStatusSnapshot}; + use s3s::{Body, S3ErrorCode, S3Request}; + + fn build_tls_status_request() -> S3Request { + S3Request { + input: Body::empty(), + method: Method::GET, + uri: Uri::from_static("/rustfs/admin/debug/tls/status"), + headers: HeaderMap::new(), + extensions: Extensions::new(), + credentials: None, + region: None, + service: None, + trailing_headers: None, + } + } + + #[tokio::test] + async fn tls_status_handler_rejects_missing_credentials() { + let result = TlsStatusHandler {}.call(build_tls_status_request(), Params::new()).await; + let err = result.expect_err("tls status handler must reject unauthenticated requests"); + assert_eq!(err.code(), &S3ErrorCode::AccessDenied); + } + + #[test] + fn tls_debug_response_schema_is_structured() { + let status = TlsRuntimeStatusSnapshot::from_outbound_only(OutboundOnlySnapshotArgs { + source_path: "/tmp/tls".to_string(), + generation: 7, + reload_enabled: true, + detect_mode: "poll", + last_attempt_time: Some(1), + last_success_time: Some(2), + last_error: Some("x".to_string()), + has_roots: true, + has_mtls_identity: false, + }); + let payload = rustfs_tls_runtime::TlsDebugStatusResponse::builder(status) + .push_consumers([TlsConsumerStatusItem { + consumer: "protos_grpc_channel", + generation: 7, + has_root_ca: true, + has_mtls_identity: false, + }]) + .build(); + + let json = serde_json::to_value(payload).expect("json should serialize"); + assert!(json.get("foundation").is_some()); + assert!(json.get("consumers").is_some()); + assert!(json["consumers"].is_array()); + } +} diff --git a/rustfs/src/admin/mod.rs b/rustfs/src/admin/mod.rs index 4b59a54ba..4bc9cf63a 100644 --- a/rustfs/src/admin/mod.rs +++ b/rustfs/src/admin/mod.rs @@ -28,7 +28,7 @@ mod route_registration_test; use handlers::{ audit, bucket_meta, heal, health, kms, module_switch, oidc, plugins_catalog, plugins_instances, pools, profile_admin, quota, - rebalance, replication, site_replication, sts, system, tier, user, + rebalance, replication, site_replication, sts, system, tier, tls_debug, user, }; use router::{AdminOperation, S3Router}; use s3s::route::S3Route; @@ -65,6 +65,7 @@ pub fn make_admin_route(console_enabled: bool) -> std::io::Result replication::register_replication_route(&mut r)?; site_replication::register_site_replication_route(&mut r)?; profile_admin::register_profiling_route(&mut r)?; + tls_debug::register_tls_debug_route(&mut r)?; kms::register_kms_route(&mut r)?; oidc::register_oidc_route(&mut r)?; diff --git a/rustfs/src/admin/route_registration_test.rs b/rustfs/src/admin/route_registration_test.rs index 8d444b377..d88a4ea64 100644 --- a/rustfs/src/admin/route_registration_test.rs +++ b/rustfs/src/admin/route_registration_test.rs @@ -15,7 +15,7 @@ use crate::admin::{ handlers::{ audit, bucket_meta, heal, health, kms, module_switch, oidc, plugins_catalog, plugins_instances, pools, profile_admin, - quota, rebalance, replication, site_replication, sts, system, tier, user, + quota, rebalance, replication, site_replication, sts, system, tier, tls_debug, user, }, router::{AdminOperation, S3Router}, }; @@ -59,6 +59,7 @@ fn register_admin_routes(router: &mut S3Router) { replication::register_replication_route(router).expect("register replication route"); site_replication::register_site_replication_route(router).expect("register site replication route"); profile_admin::register_profiling_route(router).expect("register profile route"); + tls_debug::register_tls_debug_route(router).expect("register tls debug route"); kms::register_kms_route(router).expect("register kms route"); oidc::register_oidc_route(router).expect("register oidc route"); } @@ -157,6 +158,7 @@ fn test_register_routes_cover_representative_admin_paths() { assert_route(&router, Method::PUT, &admin_path("/v3/site-replication/resync/op")); assert_route(&router, Method::PUT, &admin_path("/v3/site-replication/state/edit")); assert_route(&router, Method::GET, &admin_path("/debug/pprof/profile")); + assert_route(&router, Method::GET, &admin_path("/debug/tls/status")); assert_route(&router, Method::POST, &admin_path("/v3/kms/create-key")); assert_route(&router, Method::POST, &admin_path("/v3/kms/key/create")); diff --git a/rustfs/src/app/object_usecase.rs b/rustfs/src/app/object_usecase.rs index 56e9a881a..8b72016f3 100644 --- a/rustfs/src/app/object_usecase.rs +++ b/rustfs/src/app/object_usecase.rs @@ -92,7 +92,10 @@ use rustfs_s3select_api::{ query::{Context, Query}, }; use rustfs_s3select_query::get_global_db; -use rustfs_targets::EventName; +use rustfs_targets::{ + EventName, extract_params_header, extract_resp_elements, get_request_host, get_request_port, get_request_user_agent, +}; +use rustfs_utils::CompressionAlgorithm; use rustfs_utils::http::{ AMZ_BUCKET_REPLICATION_STATUS, AMZ_CHECKSUM_MODE, AMZ_CHECKSUM_TYPE, AMZ_WEBSITE_REDIRECT_LOCATION, CONTENT_TYPE, SUFFIX_ACTUAL_SIZE, SUFFIX_COMPRESSION, SUFFIX_COMPRESSION_SIZE, SUFFIX_REPLICATION_STATUS, SUFFIX_REPLICATION_TIMESTAMP, @@ -108,10 +111,6 @@ use rustfs_utils::http::{ insert_str, remove_str, }; use rustfs_utils::path::{is_dir_object, path_join_buf}; -use rustfs_utils::{ - CompressionAlgorithm, extract_params_header, extract_resp_elements, get_request_host, get_request_port, - get_request_user_agent, -}; use rustfs_zip::CompressionFormat; use s3s::dto::*; use s3s::header::{X_AMZ_RESTORE, X_AMZ_RESTORE_OUTPUT_PATH}; diff --git a/rustfs/src/main.rs b/rustfs/src/main.rs index fefce4ff1..0a3bcf005 100644 --- a/rustfs/src/main.rs +++ b/rustfs/src/main.rs @@ -221,12 +221,16 @@ async fn async_main() -> Result<()> { debug!("rustls crypto provider already installed, skipping aws-lc-rs default install"); } // Initialize TLS outbound material (root CAs, mTLS identity) if configured. - // Server-side TLS acceptor is built separately inside start_http_server() - // using the same TlsMaterialSnapshot loading logic. + // Server-side TLS acceptor is built separately inside start_http_server(). + // Single load via tls-runtime; outbound is enriched with platform CAs and + // published to the global state before any HTTP listener starts. if let Some(tls_path) = config.tls_path.as_deref().map(str::trim).filter(|path| !path.is_empty()) { - match rustfs::server::tls_material::TlsMaterialSnapshot::load(tls_path).await { + match rustfs::server::tls_material::load_tls_material(tls_path).await { Ok(snapshot) => { - snapshot.apply_outbound().await; + use rustfs_tls_runtime::{TlsGeneration, publish_global_outbound_tls_state, record_tls_generation}; + let generation = TlsGeneration(rustfs_common::get_global_outbound_tls_generation().saturating_add(1)); + publish_global_outbound_tls_state(generation, &snapshot.outbound).await; + record_tls_generation("rustfs_server_startup", generation.0); info!(target: "rustfs::main", "TLS outbound material initialized from {}", tls_path); } Err(e) => { @@ -234,6 +238,9 @@ async fn async_main() -> Result<()> { return Err(Error::other(e.to_string())); } } + if rustfs_obs::observability_metric_enabled() { + rustfs_tls_runtime::init_tls_metrics(); + } } // Run parameters diff --git a/rustfs/src/server/http.rs b/rustfs/src/server/http.rs index 01c6bd760..accf6bbed 100644 --- a/rustfs/src/server/http.rs +++ b/rustfs/src/server/http.rs @@ -25,7 +25,9 @@ use crate::server::{ BodylessStatusFixLayer, ConditionalCorsLayer, EmptyBodyContentLengthCompatLayer, HeadRequestBodyFixLayer, ObjectAttributesEtagFixLayer, PublicHealthEndpointLayer, RedirectLayer, RequestContextLayer, S3ErrorMessageCompatLayer, }, - tls_material::{TlsAcceptorHolder, TlsHandshakeFailureKind, TlsMaterialSnapshot, spawn_reload_loop}, + tls_material::{ + TlsAcceptorHolder, TlsHandshakeFailureKind, build_acceptor_from_loaded, load_tls_material, spawn_reload_loop, + }, }; use crate::storage; use crate::storage::rpc::InternodeRpcService; @@ -217,22 +219,34 @@ pub async fn start_http_server( let tls_path = config.tls_path.as_deref().map(str::trim).unwrap_or_default(); let tls_path_configured = !tls_path.is_empty(); - // Load TLS materials and build server acceptor. - // Note: outbound material (root CAs, mTLS identity) is already applied in main.rs. - let tls_snapshot = TlsMaterialSnapshot::load(tls_path) - .await - .map_err(|e| Error::other(e.to_string()))?; - - let tls_acceptor = tls_snapshot.build_tls_acceptor(tls_path).await.map_err(|e| { - if tls_path_configured { + // Load TLS materials and build server acceptor in a single pass. + // Outbound material (root CAs, mTLS identity) was already published in main.rs; + // this load is needed for the server-side TLS acceptor and reload loop. + let tls_acceptor = if tls_path_configured { + let snapshot = load_tls_material(tls_path).await.map_err(|e| { Error::other(format!( "TLS is explicitly configured via RUSTFS_TLS_PATH/tls_path='{}' but TLS acceptor initialization failed: {}", tls_path, e )) - } else { - Error::other(e.to_string()) + })?; + let acceptor = build_acceptor_from_loaded(snapshot.server, std::path::Path::new(tls_path)) + .await + .map_err(|e| Error::other(e.to_string()))?; + + // Fail closed: if TLS was explicitly configured but no server certificates + // were found, refuse to start rather than silently falling back to plain HTTP. + match acceptor { + None => { + return Err(Error::other(format!( + "TLS is explicitly configured via RUSTFS_TLS_PATH/tls_path='{}' but no server certificates were found", + tls_path + ))); + } + Some(a) => Some(a), } - })?; + } else { + None + }; let tls_enabled = tls_acceptor.is_some(); let protocol = if tls_enabled { "https" } else { "http" }; diff --git a/rustfs/src/server/tls_material.rs b/rustfs/src/server/tls_material.rs index c5593c4fa..44146d7a7 100644 --- a/rustfs/src/server/tls_material.rs +++ b/rustfs/src/server/tls_material.rs @@ -12,32 +12,32 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Unified TLS Material Snapshot +//! TLS material loading, enrichment, and acceptor construction. //! -//! Provides a single loading point for all TLS materials, eliminating duplicate -//! directory scanning and PEM parsing between outbound and inbound paths. -//! -//! Usage: -//! 1. Call `TlsMaterialSnapshot::load(tls_path)` once at startup. -//! 2. Call `snapshot.apply_outbound()` to set global root CAs and mTLS identity. -//! 3. TLS acceptor construction is handled internally during server startup. +//! Single-load architecture: `rustfs_tls_runtime::TlsMaterialSnapshot` loads all +//! TLS materials (server certs + outbound CAs + mTLS identity) from disk in one +//! pass. This module enriches the outbound material with platform-specific CAs +//! (system roots, leaf-as-CA, mTLS env-var path overrides) and builds the server +//! TLS acceptor directly from the pre-loaded server material — no double reads. -use rustfs_common::{MtlsIdentityPem, set_global_mtls_identity, set_global_root_cert}; +use rustfs_common::{MtlsIdentityPem, get_global_outbound_tls_generation}; use rustfs_config::{ DEFAULT_SERVER_MTLS_ENABLE, DEFAULT_TLS_KEYLOG, DEFAULT_TLS_RELOAD_ENABLE, DEFAULT_TLS_RELOAD_INTERVAL, DEFAULT_TRUST_LEAF_CERT_AS_CA, DEFAULT_TRUST_SYSTEM_CA, ENV_MTLS_CLIENT_CERT, ENV_MTLS_CLIENT_KEY, ENV_SERVER_MTLS_ENABLE, ENV_TLS_KEYLOG, ENV_TLS_RELOAD_ENABLE, ENV_TLS_RELOAD_INTERVAL, ENV_TRUST_LEAF_CERT_AS_CA, ENV_TRUST_SYSTEM_CA, - RUSTFS_CA_CERT, RUSTFS_CLIENT_CA_CERT_FILENAME, RUSTFS_CLIENT_CERT_FILENAME, RUSTFS_CLIENT_KEY_FILENAME, RUSTFS_PUBLIC_CERT, - RUSTFS_TLS_CERT, RUSTFS_TLS_KEY, + RUSTFS_CA_CERT, RUSTFS_CLIENT_CA_CERT_FILENAME, RUSTFS_CLIENT_CERT_FILENAME, RUSTFS_CLIENT_KEY_FILENAME, RUSTFS_TLS_CERT, +}; +use rustfs_tls_runtime::{ + ServerTlsMaterial as RuntimeServerTlsMaterial, TlsGeneration, TlsSource, WebPkiClientVerifierOptions, + build_webpki_client_verifier, create_multi_cert_resolver, publish_global_outbound_tls_state, record_tls_generation, + record_tls_reload_result, record_tls_reload_skipped, }; use rustfs_utils::{get_env_bool, get_env_opt_str}; use rustls::pki_types::{CertificateDer, PrivateKeyDer, pem::PemObject}; -use std::collections::HashMap; use std::path::{Path, PathBuf}; use std::sync::Arc; use std::sync::RwLock; use std::time::Duration; -use std::{fs, io}; use tokio_rustls::TlsAcceptor; use tracing::{debug, info, warn}; @@ -54,310 +54,86 @@ const SYSTEM_CA_PATHS: &[&str] = &[ "/usr/share/pki/ca-trust-legacy/ca-bundle.legacy.crt", // RHEL legacy ]; -/// Outbound TLS material for client connections (inter-node RPC). -#[derive(Debug, Clone)] -pub struct OutboundTlsMaterial { - /// Concatenated PEM-encoded root CA certificates. - pub root_ca_pem: Vec, - /// Optional mTLS client identity. - pub mtls_identity: Option, -} +// ── Public API ── -/// Complete TLS material snapshot loaded once at startup. -#[derive(Debug)] -pub struct TlsMaterialSnapshot { - /// Material for outbound client connections. - pub outbound: OutboundTlsMaterial, -} - -impl TlsMaterialSnapshot { - /// Load all TLS materials from the given directory. - /// - /// This is the single entry point that replaces both the old - /// `cert.rs::init_cert()` and `http.rs::setup_tls_acceptor()` loading logic. - pub async fn load(tls_path: &str) -> Result { - if tls_path.is_empty() { - info!("No TLS path configured; skipping TLS material loading"); - return Ok(Self::empty()); - } - - let tls_dir = PathBuf::from(tls_path); - - // Load outbound material (root CAs + mTLS identity) - let outbound = load_outbound_material(&tls_dir).await?; - - Ok(Self { outbound }) +/// Load all TLS materials from the given directory in a single pass. +/// +/// Uses `rustfs_tls_runtime::TlsMaterialSnapshot` as the single disk-read point, +/// then enriches the outbound material with platform-specific CAs that the runtime +/// crate does not handle (system roots, leaf-as-CA, mTLS env-var path overrides). +/// +/// Returns the fully enriched snapshot ready for both outbound publishing and +/// acceptor construction. +pub async fn load_tls_material(tls_path: &str) -> Result { + if tls_path.is_empty() { + return Err(TlsMaterialError::Io("TLS path is empty".into())); } - /// Apply outbound material to global state (root CAs, mTLS identity). - pub async fn apply_outbound(&self) { - if !self.outbound.root_ca_pem.is_empty() { - set_global_root_cert(self.outbound.root_ca_pem.clone()).await; - info!("Configured custom root certificates for inter-node communication"); - } - set_global_mtls_identity(self.outbound.mtls_identity.clone()).await; - } + let tls_source = TlsSource::from_directory(tls_path); + let mut snapshot = rustfs_tls_runtime::TlsMaterialSnapshot::load(&tls_source) + .await + .map_err(map_runtime_tls_error)?; - /// Build a `TlsAcceptorHolder` from the loaded snapshot. - /// - /// This is the single place that constructs the server `ServerConfig`, - /// handling both multi-cert (SNI resolver) and single-cert fallback. - /// Returns `None` if no TLS certificates are available. - pub(crate) async fn build_tls_acceptor(&self, tls_path: &str) -> Result>, TlsMaterialError> { - if tls_path.is_empty() { - return Ok(None); - } + enrich_outbound(&mut snapshot.outbound, &PathBuf::from(tls_path)).await?; - let tls_dir = validate_server_tls_directory(tls_path)?; - - let mtls_verifier = rustfs_utils::build_webpki_client_verifier( - rustfs_utils::WebPkiClientVerifierOptions::builder(&tls_dir, RUSTFS_CLIENT_CA_CERT_FILENAME, RUSTFS_CA_CERT) - .enabled(get_env_bool(ENV_SERVER_MTLS_ENABLE, DEFAULT_SERVER_MTLS_ENABLE)) - .build(), - ) - .map_err(|e| TlsMaterialError::Io(format!("build mTLS verifier: {e}")))?; - - match load_server_certificate_material(&tls_dir)? { - ServerCertificateMaterial::SingleCert { certs, key } => { - let config = build_server_config(ServerCertSource::SingleCert { certs, key }, mtls_verifier)?; - info!("Created TLS acceptor with root single certificate"); - let acceptor = Arc::new(TlsAcceptor::from(Arc::new(config))); - Ok(Some(Arc::new(TlsAcceptorHolder::new(acceptor)))) - } - ServerCertificateMaterial::MultiCert { cert_key_pairs } => { - let resolver = rustfs_utils::create_multi_cert_resolver(cert_key_pairs) - .map_err(|e| TlsMaterialError::Parse(format!("build multi-cert resolver: {e}")))?; - let config = build_server_config(ServerCertSource::Resolver(Arc::new(resolver)), mtls_verifier)?; - info!("Created TLS acceptor with SNI resolver"); - let acceptor = Arc::new(TlsAcceptor::from(Arc::new(config))); - Ok(Some(Arc::new(TlsAcceptorHolder::new(acceptor)))) - } - } - } - - fn empty() -> Self { - Self { - outbound: OutboundTlsMaterial { - root_ca_pem: Vec::new(), - mtls_identity: None, - }, - } - } + Ok(snapshot) } -fn validate_server_tls_directory(tls_path: &str) -> Result { - let tls_dir = PathBuf::from(tls_path); - let metadata = fs::metadata(&tls_dir).map_err(|e| { - let kind = if e.kind() == io::ErrorKind::NotFound { - "TLS directory does not exist" - } else { - "Failed to inspect TLS directory" - }; - TlsMaterialError::Io(format!("{kind}: {} ({e})", tls_dir.display())) - })?; - - if !metadata.is_dir() { - return Err(TlsMaterialError::Io(format!("TLS path is not a directory: {}", tls_dir.display()))); - } - - Ok(tls_dir) -} - -fn load_server_certificate_material(tls_dir: &Path) -> Result { - let root_cert_pair = inspect_root_server_cert_pair(tls_dir); - if has_discoverable_domain_cert_pair(tls_dir)? { - return load_multi_cert_material(tls_dir); - } - - match root_cert_pair { - RootCertPairState::Complete { cert_path, key_path } => load_root_single_cert_material(&cert_path, &key_path), - RootCertPairState::MissingCert { cert_path } => Err(TlsMaterialError::Io(format!( - "TLS root certificate file missing: {}", - cert_path.display() - ))), - RootCertPairState::MissingKey { key_path } => { - Err(TlsMaterialError::Io(format!("TLS root private key file missing: {}", key_path.display()))) - } - RootCertPairState::Absent => Err(TlsMaterialError::Io(format!( - "No usable TLS certificate material found under '{}'", - tls_dir.display() - ))), - } -} - -fn inspect_root_server_cert_pair(tls_dir: &Path) -> RootCertPairState { - let cert_path = tls_dir.join(RUSTFS_TLS_CERT); - let key_path = tls_dir.join(RUSTFS_TLS_KEY); - let cert_exists = cert_path.exists(); - let key_exists = key_path.exists(); - - match (cert_exists, key_exists) { - (true, true) => RootCertPairState::Complete { cert_path, key_path }, - (false, false) => RootCertPairState::Absent, - (false, true) => RootCertPairState::MissingCert { cert_path }, - (true, false) => RootCertPairState::MissingKey { key_path }, - } -} - -fn has_discoverable_domain_cert_pair(tls_dir: &Path) -> Result { - for entry in - fs::read_dir(tls_dir).map_err(|e| TlsMaterialError::Io(format!("read TLS directory {}: {e}", tls_dir.display())))? - { - let entry = entry.map_err(|e| TlsMaterialError::Io(format!("read TLS directory entry in {}: {e}", tls_dir.display())))?; - let path = entry.path(); - if !path.is_dir() { - continue; - } - - let Some(domain_name) = path.file_name().and_then(|name| name.to_str()) else { - continue; - }; - if !is_discoverable_cert_domain_dir(domain_name) { - continue; - } - - if path.join(RUSTFS_TLS_CERT).exists() && path.join(RUSTFS_TLS_KEY).exists() { - return Ok(true); - } - } - - Ok(false) -} - -fn is_discoverable_cert_domain_dir(domain_name: &str) -> bool { - !domain_name.starts_with('.') -} - -fn load_root_single_cert_material(cert_path: &Path, key_path: &Path) -> Result { - let cert_path_str = cert_path - .to_str() - .ok_or_else(|| TlsMaterialError::Io(format!("invalid UTF-8 in TLS root certificate path: {cert_path:?}")))?; - let key_path_str = key_path - .to_str() - .ok_or_else(|| TlsMaterialError::Io(format!("invalid UTF-8 in TLS root private key path: {key_path:?}")))?; - let certs = rustfs_utils::load_certs(cert_path_str) - .map_err(|e| TlsMaterialError::Io(format!("load root TLS certificate {}: {e}", cert_path.display())))?; - let key = rustfs_utils::load_private_key(key_path_str) - .map_err(|e| TlsMaterialError::Io(format!("load root TLS private key {}: {e}", key_path.display())))?; - - Ok(ServerCertificateMaterial::SingleCert { certs, key }) -} - -fn load_multi_cert_material(tls_dir: &Path) -> Result { - let cert_key_pairs = rustfs_utils::load_all_certs_from_directory( - rustfs_utils::CertDirectoryLoadOptions::builder(tls_dir, RUSTFS_TLS_CERT, RUSTFS_TLS_KEY).build(), +/// Build a TLS acceptor from pre-loaded server material. +/// +/// Takes the server material already loaded by `load_tls_material` and constructs +/// the `rustls::ServerConfig` without any additional disk reads (except the mTLS +/// client CA verifier, which reads the client CA cert from disk). +pub(crate) async fn build_acceptor_from_loaded( + server: Option, + tls_dir: &Path, +) -> Result>, TlsMaterialError> { + let mtls_verifier = build_webpki_client_verifier( + WebPkiClientVerifierOptions::builder(tls_dir, RUSTFS_CLIENT_CA_CERT_FILENAME, RUSTFS_CA_CERT) + .enabled(get_env_bool(ENV_SERVER_MTLS_ENABLE, DEFAULT_SERVER_MTLS_ENABLE)) + .build(), ) - .map_err(|e| TlsMaterialError::Io(format!("discover multi-cert TLS certificates under '{}': {e}", tls_dir.display())))?; + .map_err(|e| TlsMaterialError::Io(format!("build mTLS verifier: {e}")))?; - Ok(ServerCertificateMaterial::MultiCert { cert_key_pairs }) -} - -enum RootCertPairState { - Complete { cert_path: PathBuf, key_path: PathBuf }, - MissingCert { cert_path: PathBuf }, - MissingKey { key_path: PathBuf }, - Absent, -} - -enum ServerCertificateMaterial { - SingleCert { - certs: Vec>, - key: PrivateKeyDer<'static>, - }, - MultiCert { - cert_key_pairs: HashMap>, PrivateKeyDer<'static>)>, - }, -} - -// ── Server Config Construction ── - -/// Certificate source for building a `ServerConfig`. -enum ServerCertSource { - /// Pre-built SNI resolver from multi-cert directory. - Resolver(Arc), - /// Single certificate/key pair. - SingleCert { - certs: Vec>, - key: PrivateKeyDer<'static>, - }, -} - -/// Build a `ServerConfig` with standardized ALPN, session cache, and key log settings. -/// -/// This is the single place for `ServerConfig` construction, used by both -/// initial startup and hot-reload. -fn build_server_config( - cert_source: ServerCertSource, - mtls_verifier: Option>, -) -> Result { - let mut config = match cert_source { - ServerCertSource::Resolver(resolver) => { - if let Some(verifier) = mtls_verifier { - rustls::ServerConfig::builder() - .with_client_cert_verifier(verifier) - .with_cert_resolver(resolver) - } else { - rustls::ServerConfig::builder() - .with_no_client_auth() - .with_cert_resolver(resolver) - } + match server { + Some(RuntimeServerTlsMaterial::SingleCert { certs, key }) => { + let config = build_server_config(ServerCertSource::SingleCert { certs, key }, mtls_verifier)?; + info!("Created TLS acceptor with root single certificate"); + let acceptor = Arc::new(TlsAcceptor::from(Arc::new(config))); + Ok(Some(Arc::new(TlsAcceptorHolder::new(acceptor)))) } - ServerCertSource::SingleCert { certs, key } => { - if let Some(verifier) = mtls_verifier { - rustls::ServerConfig::builder() - .with_client_cert_verifier(verifier) - .with_single_cert(certs, key) - .map_err(|e| TlsMaterialError::Io(format!("configure single cert with mTLS: {e}")))? - } else { - rustls::ServerConfig::builder() - .with_no_client_auth() - .with_single_cert(certs, key) - .map_err(|e| TlsMaterialError::Io(format!("configure single cert: {e}")))? - } + Some(RuntimeServerTlsMaterial::MultiCert { cert_key_pairs }) => { + let resolver = create_multi_cert_resolver(cert_key_pairs) + .map_err(|e| TlsMaterialError::Parse(format!("build multi-cert resolver: {e}")))?; + let config = build_server_config(ServerCertSource::Resolver(Arc::new(resolver)), mtls_verifier)?; + info!("Created TLS acceptor with SNI resolver"); + let acceptor = Arc::new(TlsAcceptor::from(Arc::new(config))); + Ok(Some(Arc::new(TlsAcceptorHolder::new(acceptor)))) } - }; - - config.alpn_protocols = vec![b"h2".to_vec(), b"http/1.1".to_vec(), b"http/1.0".to_vec()]; - config.session_storage = rustls::server::ServerSessionMemoryCache::new(10000); - - if tls_key_log() { - config.key_log = Arc::new(rustls::KeyLogFile::new()); + None => Ok(None), } - - Ok(config) } -/// Checks if TLS key logging is enabled. -/// -/// # Returns -/// * A boolean indicating whether TLS key logging is enabled based on the `RUSTFS_TLS_KEYLOG` environment variable. -/// -fn tls_key_log() -> bool { - get_env_bool(ENV_TLS_KEYLOG, DEFAULT_TLS_KEYLOG) -} - -// ── Outbound Material Loading ── - -/// Load root CA certificates and mTLS identity for outbound connections. -async fn load_outbound_material(tls_dir: &Path) -> Result { - let mut root_ca_pem = Vec::new(); +// ── Outbound Enrichment ── +/// Enrich the outbound TLS material with platform-specific additions that the +/// tls-runtime crate does not handle: +/// 1. Optional: server leaf cert as root CA (`RUSTFS_TRUST_LEAF_CERT_AS_CA`) +/// 2. Optional: system root CAs from platform paths (`RUSTFS_TRUST_SYSTEM_CA`) +/// 3. Optional: mTLS identity from env-var-overridden paths +async fn enrich_outbound(outbound: &mut rustfs_tls_runtime::OutboundTlsMaterial, tls_dir: &Path) -> Result<(), TlsMaterialError> { // 1. Optional: load leaf certs as root CAs if get_env_bool(ENV_TRUST_LEAF_CERT_AS_CA, DEFAULT_TRUST_LEAF_CERT_AS_CA) - && load_cert_file_by_name(tls_dir, RUSTFS_TLS_CERT, &mut root_ca_pem).await + && load_cert_file_by_name(tls_dir, RUSTFS_TLS_CERT, &mut outbound.root_ca_pem).await { info!("Loaded leaf certificate(s) as root CA as per RUSTFS_TRUST_LEAF_CERT_AS_CA"); } - // 2. Load public.crt and ca.crt - load_cert_file(&tls_dir.join(RUSTFS_PUBLIC_CERT), &mut root_ca_pem, "CA certificate").await; - load_cert_file(&tls_dir.join(RUSTFS_CA_CERT), &mut root_ca_pem, "CA certificate").await; - - // 3. Optional: load system root CAs + // 2. Optional: load system root CAs if get_env_bool(ENV_TRUST_SYSTEM_CA, DEFAULT_TRUST_SYSTEM_CA) { let mut system_loaded = false; for path in SYSTEM_CA_PATHS { - if load_cert_file(Path::new(path), &mut root_ca_pem, "system root certificates").await { + if load_cert_file(Path::new(path), &mut outbound.root_ca_pem, "system root certificates").await { system_loaded = true; info!("Loaded system root certificates from {}", path); break; @@ -370,23 +146,31 @@ async fn load_outbound_material(tls_dir: &Path) -> Result Result, TlsMaterialError> { - let client_cert_path = match get_env_opt_str(ENV_MTLS_CLIENT_CERT) { +/// Load mTLS client identity when env-var path overrides are set. +/// If neither env var is set, returns Ok(None) (the tls-runtime already loaded +/// identity from the default path). +async fn load_mtls_identity_from_overridden_paths( + env_cert: Option<&str>, + env_key: Option<&str>, + tls_dir: &Path, +) -> Result, TlsMaterialError> { + let client_cert_path = match env_cert { Some(p) => PathBuf::from(p), None => tls_dir.join(RUSTFS_CLIENT_CERT_FILENAME), }; - - let client_key_path = match get_env_opt_str(ENV_MTLS_CLIENT_KEY) { + let client_key_path = match env_key { Some(p) => PathBuf::from(p), None => tls_dir.join(RUSTFS_CLIENT_KEY_FILENAME), }; @@ -406,7 +190,6 @@ async fn load_mtls_identity(tls_dir: &Path) -> Result, T .await .map_err(|e| TlsMaterialError::Io(format!("read client key {client_key_path:?}: {e}")))?; - // Validate parse-ability let mut reader = std::io::Cursor::new(&cert_pem); if CertificateDer::pem_reader_iter(&mut reader).next().is_none() { return Err(TlsMaterialError::Parse("no valid certificate in client cert PEM".into())); @@ -418,6 +201,17 @@ async fn load_mtls_identity(tls_dir: &Path) -> Result, T Ok(Some(MtlsIdentityPem { cert_pem, key_pem })) } +// ── Helpers ── + +fn map_runtime_tls_error(err: rustfs_tls_runtime::TlsRuntimeError) -> TlsMaterialError { + match &err { + rustfs_tls_runtime::TlsRuntimeError::Material(msg) | rustfs_tls_runtime::TlsRuntimeError::Publication(msg) => { + TlsMaterialError::Parse(msg.clone()) + } + _ => TlsMaterialError::Io(err.to_string()), + } +} + /// Load a single certificate file and append PEM data. /// Returns true if the file was successfully loaded. async fn load_cert_file(path: &Path, pem_data: &mut Vec, desc: &str) -> bool { @@ -457,17 +251,16 @@ async fn load_cert_file_by_name(dir: &Path, cert_name: &str, pem_data: &mut Vec< if fname == cert_name && load_cert_file(&entry.path(), pem_data, "certificate").await { loaded = true; } - } else if ft.is_dir() { - // Only check direct subdirectories (one level deep) - if let Ok(mut sub_rd) = tokio::fs::read_dir(&entry.path()).await { - while let Ok(Some(sub_entry)) = sub_rd.next_entry().await { - if let Ok(sub_ft) = sub_entry.file_type().await - && sub_ft.is_file() - { - let fname = sub_entry.file_name().to_string_lossy().to_string(); - if fname == cert_name && load_cert_file(&sub_entry.path(), pem_data, "certificate").await { - loaded = true; - } + } else if ft.is_dir() + && let Ok(mut sub_rd) = tokio::fs::read_dir(&entry.path()).await + { + while let Ok(Some(sub_entry)) = sub_rd.next_entry().await { + if let Ok(sub_ft) = sub_entry.file_type().await + && sub_ft.is_file() + { + let fname = sub_entry.file_name().to_string_lossy().to_string(); + if fname == cert_name && load_cert_file(&sub_entry.path(), pem_data, "certificate").await { + loaded = true; } } } @@ -476,12 +269,66 @@ async fn load_cert_file_by_name(dir: &Path, cert_name: &str, pem_data: &mut Vec< loaded } -/// Errors that can occur during TLS material loading. +// ── Server Config Construction ── + +enum ServerCertSource { + Resolver(Arc), + SingleCert { + certs: Vec>, + key: PrivateKeyDer<'static>, + }, +} + +fn build_server_config( + cert_source: ServerCertSource, + mtls_verifier: Option>, +) -> Result { + let mut config = match cert_source { + ServerCertSource::Resolver(resolver) => { + if let Some(verifier) = mtls_verifier { + rustls::ServerConfig::builder() + .with_client_cert_verifier(verifier) + .with_cert_resolver(resolver) + } else { + rustls::ServerConfig::builder() + .with_no_client_auth() + .with_cert_resolver(resolver) + } + } + ServerCertSource::SingleCert { certs, key } => { + if let Some(verifier) = mtls_verifier { + rustls::ServerConfig::builder() + .with_client_cert_verifier(verifier) + .with_single_cert(certs, key) + .map_err(|e| TlsMaterialError::Io(format!("configure single cert with mTLS: {e}")))? + } else { + rustls::ServerConfig::builder() + .with_no_client_auth() + .with_single_cert(certs, key) + .map_err(|e| TlsMaterialError::Io(format!("configure single cert: {e}")))? + } + } + }; + + config.alpn_protocols = vec![b"h2".to_vec(), b"http/1.1".to_vec(), b"http/1.0".to_vec()]; + config.session_storage = rustls::server::ServerSessionMemoryCache::new(10000); + + if tls_key_log() { + config.key_log = Arc::new(rustls::KeyLogFile::new()); + } + + Ok(config) +} + +fn tls_key_log() -> bool { + get_env_bool(ENV_TLS_KEYLOG, DEFAULT_TLS_KEYLOG) +} + +// ── Errors ── + #[derive(Debug)] pub enum TlsMaterialError { - /// I/O error (file read, directory access). Io(String), - /// PEM parsing error. Parse(String), } @@ -498,7 +345,6 @@ impl std::error::Error for TlsMaterialError {} // ── TLS Handshake Error Classification ── -/// Structured classification of TLS handshake failures. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) enum TlsHandshakeFailureKind { UnexpectedEof, @@ -509,7 +355,6 @@ pub(crate) enum TlsHandshakeFailureKind { } impl TlsHandshakeFailureKind { - /// Classify a TLS accept error into a structured failure kind. pub(crate) fn classify(err_msg: &str) -> Self { if err_msg.contains("unexpected EOF") || err_msg.contains("handshake eof") { Self::UnexpectedEof @@ -524,7 +369,6 @@ impl TlsHandshakeFailureKind { } } - /// Metric label string for Prometheus. pub(crate) fn as_str(self) -> &'static str { match self { Self::UnexpectedEof => "UNEXPECTED_EOF", @@ -538,10 +382,6 @@ impl TlsHandshakeFailureKind { // ── TLS Acceptor Holder (for hot reload) ── -/// Holds the current TLS acceptor and supports atomic swap for certificate rotation. -/// -/// Uses `RwLock` so that multiple readers (per-connection `get()` calls) -/// do not block each other. The write lock is held only briefly during swap. pub(crate) struct TlsAcceptorHolder { current: RwLock>, } @@ -553,7 +393,6 @@ impl TlsAcceptorHolder { } } - /// Get the current TLS acceptor for handling a new connection. #[inline] pub(crate) fn get(&self) -> Arc { match self.current.read() { @@ -562,7 +401,6 @@ impl TlsAcceptorHolder { } } - /// Atomically replace the TLS acceptor with a new one. fn swap(&self, new_holder: &TlsAcceptorHolder) { let new_acceptor = new_holder.get(); match self.current.write() { @@ -575,7 +413,11 @@ impl TlsAcceptorHolder { } } +// ── Reload Loop ── + /// Spawn a background task that periodically checks for TLS certificate changes. +/// Single load per tick: loads once via tls-runtime, enriches, publishes outbound, +/// and builds acceptor — no double reads. pub(crate) fn spawn_reload_loop(tls_path: String, holder: Arc) { let enabled = get_env_bool(ENV_TLS_RELOAD_ENABLE, DEFAULT_TLS_RELOAD_ENABLE); if !enabled { @@ -584,31 +426,49 @@ pub(crate) fn spawn_reload_loop(tls_path: String, holder: Arc } let interval_secs = rustfs_utils::get_env_u64(ENV_TLS_RELOAD_INTERVAL, DEFAULT_TLS_RELOAD_INTERVAL).max(5); + let tls_source = TlsSource::from_directory(&tls_path); info!("TLS certificate hot reload enabled, checking every {}s", interval_secs); tokio::spawn(async move { let mut interval = tokio::time::interval(Duration::from_secs(interval_secs)); + let tls_dir = PathBuf::from(&tls_path); loop { interval.tick().await; - match TlsMaterialSnapshot::load(&tls_path).await { - Ok(snapshot) => { - // Always refresh outbound material (root CAs, mTLS identity) on reload. - snapshot.apply_outbound().await; + match rustfs_tls_runtime::TlsMaterialSnapshot::load(&tls_source).await { + Ok(mut snapshot) => { + if let Err(e) = enrich_outbound(&mut snapshot.outbound, &tls_dir).await { + record_tls_reload_result("rustfs_server_reload_loop", "enrich_err", None, None); + warn!("TLS outbound enrichment failed (will retry): {}", e); + continue; + } - match snapshot.build_tls_acceptor(&tls_path).await { + let generation = get_global_outbound_tls_generation().saturating_add(1); + publish_global_outbound_tls_state(TlsGeneration(generation), &snapshot.outbound).await; + record_tls_generation("rustfs_server_reload_loop", generation); + if !snapshot.outbound.root_ca_pem.is_empty() { + info!("Configured custom root certificates for inter-node communication"); + } + + match build_acceptor_from_loaded(snapshot.server, &tls_dir).await { Ok(Some(new_holder)) => { info!("TLS certificates reloaded successfully"); holder.swap(&new_holder); + record_tls_reload_result("rustfs_server_reload_loop", "ok", None, Some(generation)); } Ok(None) => { + record_tls_reload_skipped("rustfs_server_reload_loop", "no_acceptor"); warn!("TLS reload returned no acceptor despite configured TLS path; keeping previous acceptor") } - Err(e) => warn!("TLS certificate reload failed (will retry): {}", e), + Err(e) => { + record_tls_reload_result("rustfs_server_reload_loop", "acceptor_err", None, Some(generation)); + warn!("TLS certificate reload failed (will retry): {}", e) + } } } Err(e) => { + record_tls_reload_result("rustfs_server_reload_loop", "load_err", None, None); warn!("TLS material reload failed (will retry): {}", e); } } @@ -636,20 +496,19 @@ mod tests { fn write_test_cert_pair(dir: &Path, subject: &str) { let CertifiedKey { cert, signing_key } = rcgen::generate_simple_self_signed(vec![subject.to_string()]).unwrap(); fs::write(dir.join(RUSTFS_TLS_CERT), cert.pem()).unwrap(); - fs::write(dir.join(RUSTFS_TLS_KEY), signing_key.serialize_pem()).unwrap(); + fs::write(dir.join(rustfs_config::RUSTFS_TLS_KEY), signing_key.serialize_pem()).unwrap(); } #[tokio::test] - async fn build_tls_acceptor_accepts_root_single_cert_with_trailing_slash() { + async fn build_acceptor_accepts_root_single_cert_with_trailing_slash() { ensure_rustls_crypto_provider(); let temp_dir = TempDir::new().unwrap(); write_test_cert_pair(temp_dir.path(), "localhost"); - let snapshot = TlsMaterialSnapshot::load(&format!("{}/", temp_dir.path().display())) + let snapshot = load_tls_material(&format!("{}/", temp_dir.path().display())) .await .expect("TLS material load should succeed"); - let acceptor = snapshot - .build_tls_acceptor(&format!("{}/", temp_dir.path().display())) + let acceptor = build_acceptor_from_loaded(snapshot.server, temp_dir.path()) .await .expect("root single-cert TLS acceptor should build"); @@ -657,7 +516,7 @@ mod tests { } #[tokio::test] - async fn build_tls_acceptor_accepts_symlinked_root_single_cert_directory() { + async fn build_acceptor_accepts_symlinked_root_single_cert_directory() { ensure_rustls_crypto_provider(); let temp_dir = TempDir::new().unwrap(); let real_dir = temp_dir.path().join("tls-real"); @@ -670,11 +529,10 @@ mod tests { #[cfg(windows)] std::os::windows::fs::symlink_dir(&real_dir, &symlink_dir).unwrap(); - let snapshot = TlsMaterialSnapshot::load(symlink_dir.to_str().unwrap()) + let snapshot = load_tls_material(symlink_dir.to_str().unwrap()) .await .expect("TLS material load through symlink should succeed"); - let acceptor = snapshot - .build_tls_acceptor(symlink_dir.to_str().unwrap()) + let acceptor = build_acceptor_from_loaded(snapshot.server, &symlink_dir) .await .expect("TLS acceptor should build through symlink"); @@ -682,62 +540,55 @@ mod tests { } #[tokio::test] - async fn build_tls_acceptor_rejects_missing_tls_directory() { + async fn build_acceptor_rejects_missing_tls_directory() { let temp_dir = TempDir::new().unwrap(); let missing_dir = temp_dir.path().join("missing"); - let snapshot = TlsMaterialSnapshot::empty(); - let err = match snapshot.build_tls_acceptor(missing_dir.to_str().unwrap()).await { - Ok(_) => panic!("missing TLS directory should fail"), - Err(err) => err, - }; + let err = load_tls_material(missing_dir.to_str().unwrap()) + .await + .expect_err("missing TLS directory should fail"); assert!(err.to_string().contains("TLS directory does not exist")); } #[tokio::test] - async fn build_tls_acceptor_rejects_tls_path_that_is_not_directory() { + async fn build_acceptor_rejects_tls_path_that_is_not_directory() { let temp_dir = TempDir::new().unwrap(); let file_path = temp_dir.path().join("tls-file"); fs::write(&file_path, "not-a-directory").unwrap(); - let snapshot = TlsMaterialSnapshot::empty(); - let err = match snapshot.build_tls_acceptor(file_path.to_str().unwrap()).await { - Ok(_) => panic!("regular file TLS path should fail"), - Err(err) => err, - }; + let err = load_tls_material(file_path.to_str().unwrap()) + .await + .expect_err("regular file TLS path should fail"); assert!(err.to_string().contains("TLS path is not a directory")); } #[tokio::test] - async fn build_tls_acceptor_rejects_empty_tls_directory() { + async fn build_acceptor_returns_none_for_empty_tls_directory() { let temp_dir = TempDir::new().unwrap(); - let snapshot = TlsMaterialSnapshot::load(temp_dir.path().to_str().unwrap()) + let snapshot = load_tls_material(temp_dir.path().to_str().unwrap()) .await - .expect("outbound TLS load for empty dir should succeed"); + .expect("TLS material load for empty dir should succeed"); - let err = match snapshot.build_tls_acceptor(temp_dir.path().to_str().unwrap()).await { - Ok(_) => panic!("empty TLS directory should fail"), - Err(err) => err, - }; - - assert!(err.to_string().contains("No usable TLS certificate material found")); + let acceptor = build_acceptor_from_loaded(snapshot.server, temp_dir.path()) + .await + .expect("empty TLS directory should be treated as no acceptor"); + assert!(acceptor.is_none()); } #[tokio::test] - async fn build_tls_acceptor_still_supports_multi_cert_directories() { + async fn build_acceptor_still_supports_multi_cert_directories() { ensure_rustls_crypto_provider(); let temp_dir = TempDir::new().unwrap(); let domain_dir = temp_dir.path().join("example.com"); fs::create_dir(&domain_dir).unwrap(); write_test_cert_pair(&domain_dir, "example.com"); - let snapshot = TlsMaterialSnapshot::load(temp_dir.path().to_str().unwrap()) + let snapshot = load_tls_material(temp_dir.path().to_str().unwrap()) .await .expect("TLS material load for multi-cert dir should succeed"); - let acceptor = snapshot - .build_tls_acceptor(temp_dir.path().to_str().unwrap()) + let acceptor = build_acceptor_from_loaded(snapshot.server, temp_dir.path()) .await .expect("multi-cert TLS acceptor should build"); @@ -745,7 +596,7 @@ mod tests { } #[tokio::test] - async fn build_tls_acceptor_prefers_multi_cert_when_root_and_domain_pairs_both_exist() { + async fn build_acceptor_prefers_multi_cert_when_root_and_domain_pairs_both_exist() { ensure_rustls_crypto_provider(); let temp_dir = TempDir::new().unwrap(); write_test_cert_pair(temp_dir.path(), "default.local"); @@ -754,11 +605,10 @@ mod tests { fs::create_dir(&domain_dir).unwrap(); write_test_cert_pair(&domain_dir, "example.com"); - let snapshot = TlsMaterialSnapshot::load(temp_dir.path().to_str().unwrap()) + let snapshot = load_tls_material(temp_dir.path().to_str().unwrap()) .await .expect("TLS material load for mixed root/domain layout should succeed"); - let acceptor = snapshot - .build_tls_acceptor(temp_dir.path().to_str().unwrap()) + let acceptor = build_acceptor_from_loaded(snapshot.server, temp_dir.path()) .await .expect("multi-cert TLS acceptor should still build when root pair also exists"); diff --git a/rustfs/src/storage/helper.rs b/rustfs/src/storage/helper.rs index 2a1c5e510..995d7e590 100644 --- a/rustfs/src/storage/helper.rs +++ b/rustfs/src/storage/helper.rs @@ -27,10 +27,10 @@ use rustfs_io_metrics::record_s3_op; use rustfs_notify::{EventArgsBuilder, notifier_global}; use rustfs_s3_ops::{S3Operation, operation_matches_event_name}; use rustfs_s3_types::EventName; -use rustfs_utils::{ +use rustfs_targets::{ extract_params_header, extract_req_params, extract_resp_elements, get_request_host, get_request_port, get_request_user_agent, - http::headers::AMZ_REQUEST_ID, }; +use rustfs_utils::http::headers::AMZ_REQUEST_ID; use s3s::{S3Request, S3Response, S3Result}; use serde_json::Value; use std::future::Future;