diff --git a/.github/workflows/nix-flake-update.yml b/.github/workflows/nix-flake-update.yml index ea936a0bf..3d74456c3 100644 --- a/.github/workflows/nix-flake-update.yml +++ b/.github/workflows/nix-flake-update.yml @@ -31,7 +31,7 @@ jobs: update-flake: name: Update flake.lock runs-on: ubuntu-latest - timeout-minutes: 30 + timeout-minutes: 45 steps: - name: Checkout repository uses: actions/checkout@v6 diff --git a/.github/workflows/nix.yml b/.github/workflows/nix.yml index 8552a8dd2..ac14ddbae 100644 --- a/.github/workflows/nix.yml +++ b/.github/workflows/nix.yml @@ -43,7 +43,7 @@ jobs: nix-validation: name: Nix Build & Check runs-on: ubuntu-latest - timeout-minutes: 30 + timeout-minutes: 45 steps: - name: Checkout repository uses: actions/checkout@v6 @@ -57,7 +57,7 @@ jobs: - name: Setup Magic Nix Cache uses: DeterminateSystems/magic-nix-cache-action@v13 - + - name: Setup Flake Checker uses: DeterminateSystems/flake-checker-action@v12 diff --git a/Cargo.lock b/Cargo.lock index 9a9b87330..c879160c0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -148,9 +148,9 @@ checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" [[package]] name = "anstream" -version = "0.6.21" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43d5b281e737544384e969a5ccad3f1cdd24b48086a0fc1b2a5262a26b8f4f4a" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" dependencies = [ "anstyle", "anstyle-parse", @@ -169,9 +169,9 @@ checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78" [[package]] name = "anstyle-parse" -version = "0.2.7" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" dependencies = [ "utf8parse", ] @@ -706,9 +706,9 @@ dependencies = [ [[package]] name = "aws-sdk-s3" -version = "1.125.0" +version = "1.126.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "223f5c95650d9557925a91f4c2db3def189e8f659452134a29e5cd2d37d708ed" +checksum = "7878050a2321d215eec9db8be09f8db59418b53860ae86cc7042b4094d6cb2bb" dependencies = [ "aws-credential-types", "aws-runtime", @@ -1470,7 +1470,7 @@ version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" dependencies = [ - "crypto-common 0.1.7", + "crypto-common 0.1.6", "inout 0.1.4", ] @@ -1487,9 +1487,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.60" +version = "4.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2797f34da339ce31042b27d23607e051786132987f595b02ba4f6a6dffb7030a" +checksum = "b193af5b67834b676abd72466a96c1024e6a6ad978a1f484bd90b85c94041351" dependencies = [ "clap_builder", "clap_derive", @@ -1497,9 +1497,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.60" +version = "4.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24a241312cea5059b13574bb9b3861cabf758b879c15190b37b6d6fd63ab6876" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" dependencies = [ "anstream", "anstyle", @@ -1509,9 +1509,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.55" +version = "4.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a92793da1a46a5f2a02a6f4c46c6496b28c43638adea8306fcb0caa1634f24e5" +checksum = "1110bd8a634a1ab8cb04345d8d878267d57c3cf1b38d91b71af6686408bbca6a" dependencies = [ "heck", "proc-macro2", @@ -1521,9 +1521,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "1.0.0" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a822ea5bc7590f9d40f1ba12c0dc3c2760f3482c6984db1573ad11031420831" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" [[package]] name = "clocksource" @@ -1927,9 +1927,9 @@ dependencies = [ [[package]] name = "crypto-bigint" -version = "0.7.0-rc.28" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96dacf199529fb801ae62a9aafdc01b189e9504c0d1ee1512a4c16bcd8666a93" +checksum = "9fde2467e74147f492aebb834985186b2c74761927b8b9b3bd303bcb2e72199d" dependencies = [ "cpubits", "ctutils", @@ -1941,9 +1941,9 @@ dependencies = [ [[package]] name = "crypto-common" -version = "0.1.7" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" dependencies = [ "generic-array", "typenum", @@ -1962,11 +1962,11 @@ dependencies = [ [[package]] name = "crypto-primes" -version = "0.7.0-pre.9" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6081ce8b60c0e533e2bba42771b94eb6149052115f4179744d5779883dc98583" +checksum = "21f41f23de7d24cdbda7f0c4d9c0351f99a4ceb258ef30e5c1927af8987ffe5a" dependencies = [ - "crypto-bigint 0.7.0-rc.28", + "crypto-bigint 0.7.1", "libm", "rand_core 0.10.0", ] @@ -2204,9 +2204,9 @@ checksum = "d7a1e2f27636f116493b8b860f5546edb47c8d8f8ea73e1d2a20be88e28d1fea" [[package]] name = "datafusion" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "503f1f4a9060ae6e650d3dff5dc7a21266fea1302d890768d45b4b28586e830f" +checksum = "ea28305c211e3541c9cfcf06a23d0d8c7c824b4502ed1fdf0a6ff4ad24ee531c" dependencies = [ "arrow", "arrow-schema", @@ -2259,9 +2259,9 @@ dependencies = [ [[package]] name = "datafusion-catalog" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14417a3ee4ae3d092b56cd6c1d32e8ff3e2c9ec130ecb2276ec91c89fd599399" +checksum = "78ab99b6df5f60a6ddbc515e4c05caee1192d395cf3cb67ce5d1c17e3c9b9b74" dependencies = [ "arrow", "async-trait", @@ -2284,9 +2284,9 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d0eba824adb45a4b3ac6f0251d40df3f6a9382371cad136f4f14ac9ebc6bc10" +checksum = "77ae3d14912c0d779ada98d30dc60f3244f3c26c2446b87394629ea5c076a31c" dependencies = [ "arrow", "async-trait", @@ -2307,9 +2307,9 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0039deefbd00c56adf5168b7ca58568fb058e4ba4c5a03b09f8be371b4e434b6" +checksum = "ea2df29b9592a5d55b8238eaf67d2f21963d5a08cd1a8b7670134405206caabd" dependencies = [ "ahash", "arrow", @@ -2331,9 +2331,9 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ec7e3e60b813048331f8fb9673583173e5d2dd8fef862834ee871fc98b57ca7" +checksum = "42639baa0049d5fffd7e283504b9b5e7b9b2e7a2dea476eed60ab0d40d999b85" dependencies = [ "futures", "log", @@ -2342,9 +2342,9 @@ dependencies = [ [[package]] name = "datafusion-datasource" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "802068957f620302ecf05f84ff4019601aeafd36f5f3f1334984af2e34265129" +checksum = "25951b617bb22a9619e1520450590cb2004bfcad10bcb396b961f4a1a10dcec5" dependencies = [ "arrow", "async-compression", @@ -2377,9 +2377,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-arrow" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90fc387d5067c62d494a6647d29c5ad4fcdd5a6e50ab4ea1d2568caa2d66f2cc" +checksum = "dc0b28226960ba99c50d78ac6f736ebe09eb5cb3bb9bb58194266278000ca41f" dependencies = [ "arrow", "arrow-ipc", @@ -2401,9 +2401,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efd5e20579bb6c8bd4e6c620253972fb723822030c280dd6aa047f660d09eeba" +checksum = "f538b57b052a678b1ce860181c65d3ace5a8486312dc50b41c01dd585a773a51" dependencies = [ "arrow", "async-trait", @@ -2424,9 +2424,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0788b0d48fcef31880a02013ea3cc18e5a4e0eacc3b0abdd2cd0597b99dc96e" +checksum = "89fbc1d32b1b03c9734e27c0c5f041232b68621c8455f22769838634750a196c" dependencies = [ "arrow", "async-trait", @@ -2446,9 +2446,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-parquet" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66639b70f1f363f5f0950733170100e588f1acfacac90c1894e231194aa35957" +checksum = "203271d31fe5613a5943181db70ec98162121d1de94a9a300d5e5f19f9500a32" dependencies = [ "arrow", "async-trait", @@ -2476,15 +2476,15 @@ dependencies = [ [[package]] name = "datafusion-doc" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e44b41f3e8267c6cf3eec982d63f34db9f1dd5f30abfd2e1f124f0871708952e" +checksum = "5b6450dc702b3d39e8ced54c3356abb453bd2f3cea86d90d555a4b92f7a38462" [[package]] name = "datafusion-execution" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e456f60e5d38db45335e84617006d90af14a8c8c5b8e959add708b2daaa0e2c" +checksum = "e66a02fa601de49da5181dbdcf904a18b16a184db2b31f5e5534552ea2d5e660" dependencies = [ "arrow", "async-trait", @@ -2503,9 +2503,9 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6507c719804265a58043134580c1c20767e7c23ba450724393f03ec982769ad9" +checksum = "cdf59a9b308a1a07dc2eb2f85e6366bc0226dc390b40f3aa0a72d79f1cfe2465" dependencies = [ "arrow", "async-trait", @@ -2526,9 +2526,9 @@ dependencies = [ [[package]] name = "datafusion-expr-common" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a413caa9c5885072b539337aed68488f0291653e8edd7d676c92df2480f6cab0" +checksum = "bd99eac4c6538c708638db43e7a3bd88e0e57955ddb722d420fb9a6d38dfc28f" dependencies = [ "arrow", "datafusion-common", @@ -2539,9 +2539,9 @@ dependencies = [ [[package]] name = "datafusion-functions" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "189256495dc9cbbb8e20dbcf161f60422e628d201a78df8207e44bd4baefadb6" +checksum = "11aa2c492ac046397b36d57c62a72982aad306495bbcbcdbcabd424d4a2fe245" dependencies = [ "arrow", "arrow-buffer", @@ -2570,9 +2570,9 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12e73dfee4cd67c4a507ffff4c5a711d39983adf544adbc09c09bf06f789f413" +checksum = "325a00081898945d48d6194d9ca26120e523c993be3bb7c084061a5a2a72e787" dependencies = [ "ahash", "arrow", @@ -2591,9 +2591,9 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87727bd9e65f4f9ac6d608c9810b7da9eaa3b18b26a4a4b76520592d49020acf" +checksum = "809bbcb1e0dbec5d0ce30d493d135aea7564f1ba4550395f7f94321223df2dae" dependencies = [ "ahash", "arrow", @@ -2604,9 +2604,9 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e5ef761359224b7c2b5a1bfad6296ac63225f8583d08ad18af9ba1a89ac3887" +checksum = "29ebaa5d7024ef45973e0a7db1e9aeaa647936496f4d4061c0448f23d77d6320" dependencies = [ "arrow", "arrow-ord", @@ -2627,9 +2627,9 @@ dependencies = [ [[package]] name = "datafusion-functions-table" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b17dac25dfda2d2a90ff0ad1c054a11fb1523766226bec6e9bd8c410daee2ae" +checksum = "60eab6f39df9ee49a2c7fa38eddc01fa0086ee31b29c7d19f38e72f479609752" dependencies = [ "arrow", "async-trait", @@ -2643,9 +2643,9 @@ dependencies = [ [[package]] name = "datafusion-functions-window" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c594a29ddb22cbdbce500e4d99b5b2392c5cecb4c1086298b41d1ffec14dbb77" +checksum = "e00b2c15e342a90e65a846199c9e49293dd09fe1bcd63d8be2544604892f7eb8" dependencies = [ "arrow", "datafusion-common", @@ -2661,9 +2661,9 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9aa1b15ed81c7543f62264a30dd49dec4b1b0b698053b968f53be32dfba4f729" +checksum = "493e2e1d1f4753dfc139a5213f1b5d0b97eea46a82d9bda3c7908aa96981b74b" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -2671,9 +2671,9 @@ dependencies = [ [[package]] name = "datafusion-macros" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c00c31c4795597aa25b74cab5174ac07a53051f27ce1e011ecaffa9eaeecef81" +checksum = "ba01c55ade8278a791b429f7bf5cb1de64de587a342d084b18245edfae7096e2" dependencies = [ "datafusion-doc", "quote", @@ -2682,9 +2682,9 @@ dependencies = [ [[package]] name = "datafusion-optimizer" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80ccf60767c09302b2e0fc3afebb3761a6d508d07316fab8c5e93312728a21bb" +checksum = "a80c6dfbba6a2163a9507f6353ac78c69d8deb26232c9e419160e58ff7c3e047" dependencies = [ "arrow", "chrono", @@ -2702,9 +2702,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c64b7f277556944e4edd3558da01d9e9ff9f5416f1c0aa7fee088e57bd141a7e" +checksum = "5d3a86264bb9163e7360b6622e789bc7fcbb43672e78a8493f0bc369a41a57c6" dependencies = [ "ahash", "arrow", @@ -2726,9 +2726,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-adapter" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7abaee372ea2d19c016ee9ef8629c4415257d291cdd152bc7f0b75f28af1b63" +checksum = "3f5e00e524ac33500be6c5eeac940bd3f6b984ba9b7df0cd5f6c34a8a2cc4d6b" dependencies = [ "arrow", "datafusion-common", @@ -2741,9 +2741,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42237efe621f92adc22d111b531fdbc2cc38ca9b5e02327535628fb103ae2157" +checksum = "2ae769ea5d688b4e74e9be5cad6f9d9f295b540825355868a3ab942380dd97ce" dependencies = [ "ahash", "arrow", @@ -2758,9 +2758,9 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd093498bd1319c6e5c76e9dfa905e78486f01b34579ce97f2e3a49f84c37fac" +checksum = "f3588753ab2b47b0e43cd823fe5e7944df6734dabd6dafb72e2cc1c2a22f1944" dependencies = [ "arrow", "datafusion-common", @@ -2777,9 +2777,9 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cbe61b12daf81a9f20ba03bd3541165d51f86e004ef37426b11881330eed261" +checksum = "79949cbb109c2a45c527bfe0d956b9f2916807c05d4d2e66f3fd0af827ac2b61" dependencies = [ "ahash", "arrow", @@ -2808,9 +2808,9 @@ dependencies = [ [[package]] name = "datafusion-pruning" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0124331116db7f79df92ebfd2c3b11a8f90240f253555c9bb084f10b6fecf1dd" +checksum = "6434e2ee8a39d04b95fed688ff34dc251af6e4a0c2e1714716b6e3846690d589" dependencies = [ "arrow", "datafusion-common", @@ -2825,9 +2825,9 @@ dependencies = [ [[package]] name = "datafusion-session" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1673e3c58ba618a6ea0568672f00664087b8982c581e9afd5aa6c3c79c9b431f" +checksum = "c91efb8302b4877d499c37e9a71886b90236ab27d9cc42fd51112febf341abd6" dependencies = [ "async-trait", "datafusion-common", @@ -2839,9 +2839,9 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5272d256dab5347bb39d2040589f45d8c6b715b27edcb5fffe88cc8b9c3909cb" +checksum = "3f01eef7bcf4d00e87305b55f1b75792384e130fe0258bac02cd48378ae5ff87" dependencies = [ "arrow", "bigdecimal", @@ -3025,7 +3025,7 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer 0.10.4", "const-oid 0.9.6", - "crypto-common 0.1.7", + "crypto-common 0.1.6", "subtle", ] @@ -3606,9 +3606,9 @@ dependencies = [ [[package]] name = "generic-array" -version = "0.14.7" +version = "0.14.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +checksum = "4bb6743198531e02858aeaea5398fcc883e71851fcbcb5a2f773e2fb6cb1edf2" dependencies = [ "typenum", "version_check", @@ -3703,15 +3703,18 @@ checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" [[package]] name = "google-cloud-auth" -version = "1.6.0" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36ad774d41426ab205eeec577540f209a5485c366814dd5c89a7e3018fe84e7c" +checksum = "1f83bc5c208df4a6b38ad2a8d2b01c0d377811f9efe9b0733171f28dd74db9c3" dependencies = [ "async-trait", "aws-lc-rs", "base64 0.22.1", "bytes", + "chrono", "google-cloud-gax", + "hex", + "hmac 0.12.1", "http 1.4.0", "jsonwebtoken", "reqwest 0.13.2", @@ -3720,16 +3723,18 @@ dependencies = [ "rustls-pki-types", "serde", "serde_json", + "sha2 0.10.9", "thiserror 2.0.18", "time", "tokio", + "url", ] [[package]] name = "google-cloud-gax" -version = "1.7.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2973715fe664ecb0d883926c8b5f66cb9d52a44add1d0be1cad1907d832bf0af" +checksum = "188909653b7c484e43695325c0324804b5645d568f8d2e4c8a6f520231d50956" dependencies = [ "base64 0.22.1", "bytes", @@ -3747,9 +3752,9 @@ dependencies = [ [[package]] name = "google-cloud-gax-internal" -version = "0.7.9" +version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "598e5ffec2c1c9b43e83847b2badc0f128a03e541b75f1ecd8acf0a2605c40cf" +checksum = "7395094c1dc7284155a48530aa1a49d52c876ce1b392dfcdf7e6b32540d042a2" dependencies = [ "bytes", "futures", @@ -3761,6 +3766,8 @@ dependencies = [ "http-body 1.0.1", "http-body-util", "hyper", + "lazy_static", + "opentelemetry", "opentelemetry-semantic-conventions", "percent-encoding", "pin-project", @@ -3781,9 +3788,9 @@ dependencies = [ [[package]] name = "google-cloud-iam-v1" -version = "1.5.0" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b41025359d1f52c24966c24d5ecb7f91198ded13c9cbf57c2d2e735dae93c814" +checksum = "f436945fb3c581a3ef32f37e47756690006de7177934c6405cc0d4db799c8975" dependencies = [ "async-trait", "bytes", @@ -3800,9 +3807,9 @@ dependencies = [ [[package]] name = "google-cloud-longrunning" -version = "1.6.0" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "daafe6c2859976ca51571db6c7bbf96bdce0122e81e6854a6c62554c4f63bae8" +checksum = "767db5d07fff5361d01058764e724c4335b9899aa6c973c3bb2ae8d3bd4eacd8" dependencies = [ "async-trait", "bytes", @@ -3846,9 +3853,9 @@ dependencies = [ [[package]] name = "google-cloud-storage" -version = "1.8.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ce225eccfece751251e07ac787199cfc53bc2b5d60ac0570f58404507ceb309" +checksum = "1be397108904bd24fb7b1518b68fc26a70589f3718fa8c47e9af5f09c4fc6e88" dependencies = [ "async-trait", "base64 0.22.1", @@ -4168,9 +4175,9 @@ checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" [[package]] name = "hybrid-array" -version = "0.4.7" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1b229d73f5803b562cc26e4da0396c8610a4ee209f4fac8fa4f8d709166dc45" +checksum = "8655f91cd07f2b9d0c24137bd650fe69617773435ee5ec83022377777ce65ef1" dependencies = [ "typenum", ] @@ -5501,18 +5508,18 @@ dependencies = [ [[package]] name = "objc2-core-foundation" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c10c2894a6fed806ade6027bcd50662746363a9589d3ec9d9bef30a4e4bc166" +checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" dependencies = [ "bitflags 2.11.0", ] [[package]] name = "objc2-io-kit" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71c1c64d6120e51cd86033f67176b1cb66780c2efe34dec55176f77befd93c0a" +checksum = "33fafba39597d6dc1fb709123dfa8289d39406734be322956a69f0931c73bb15" dependencies = [ "libc", "objc2-core-foundation", @@ -5572,9 +5579,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.21.3" +version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "once_cell_polyfill" @@ -5865,9 +5872,9 @@ dependencies = [ [[package]] name = "password-hash" -version = "0.6.0-rc.12" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fa9e3d1c7b6f3e230b60fa44adc855cb8e24eede37236621f2cc1940d95564f" +checksum = "ccbd25f71dd5249dba9ed843d52500c8757a25511560d01a94f4abf56b52a1d5" dependencies = [ "getrandom 0.4.2", "phc", @@ -5990,13 +5997,13 @@ dependencies = [ [[package]] name = "phc" -version = "0.6.0-rc.1" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71d390c5fe8d102c2c18ff39f1e72b9ad5996de282c2d831b0312f56910f5508" +checksum = "44dc769b75f93afdddd8c7fa12d685292ddeff1e66f7f0f3a234cf1818afe892" dependencies = [ "base64ct", + "ctutils", "getrandom 0.4.2", - "subtle", ] [[package]] @@ -6474,9 +6481,9 @@ dependencies = [ [[package]] name = "pyroscope" -version = "1.0.2" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9a34d9b74a7a13140e626ed568fc984f520747ed6cf9e83bb532bb2f311ec75" +checksum = "85eebd4bcbf45db75f67d2ba20ea0207bd111d2029c07a7db3229289173d4387" dependencies = [ "lazy_static", "libc", @@ -6982,9 +6989,9 @@ checksum = "3582f63211428f83597b51b2ddb88e2a91a9d52d12831f9d08f5e624e8977422" [[package]] name = "rmcp" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2cb14cb9278a12eae884c9f3c0cfeca2cc28f361211206424a1d7abed95f090" +checksum = "ba6b9d2f0efe2258b23767f1f9e0054cfbcac9c2d6f81a031214143096d7864f" dependencies = [ "async-trait", "base64 0.22.1", @@ -7004,9 +7011,9 @@ dependencies = [ [[package]] name = "rmcp-macros" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a02ea81d9482b07e1fe156ac7cf98b6823d51fb84531936a5e1cbb4eec31ad5" +checksum = "ab9d95d7ed26ad8306352b0d5f05b593222b272790564589790d210aa15caa9e" dependencies = [ "darling 0.23.0", "proc-macro2", @@ -7056,12 +7063,12 @@ dependencies = [ [[package]] name = "rsa" -version = "0.10.0-rc.16" +version = "0.10.0-rc.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fb9fd8c1edd9e6a2693623baf0fe77ff05ce022a5d7746900ffc38a15c233de" +checksum = "87ed3e93fc7e473e464b9726f4759659e72bc8665e4b8ea227547024f416d905" dependencies = [ "const-oid 0.10.2", - "crypto-bigint 0.7.0-rc.28", + "crypto-bigint 0.7.1", "crypto-primes", "digest 0.11.1", "pkcs1 0.8.0-rc.4", @@ -7260,7 +7267,7 @@ version = "0.0.5" dependencies = [ "base64-simd", "rand 0.10.0", - "rsa 0.10.0-rc.16", + "rsa 0.10.0-rc.17", "serde", "serde_json", ] @@ -7675,6 +7682,7 @@ version = "0.0.5" dependencies = [ "flate2", "glob", + "jiff", "metrics", "nvml-wrapper", "opentelemetry", @@ -7697,7 +7705,6 @@ dependencies = [ "tracing-error", "tracing-opentelemetry", "tracing-subscriber", - "walkdir", ] [[package]] @@ -8203,7 +8210,7 @@ checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" [[package]] name = "s3s" version = "0.14.0-dev" -source = "git+https://github.com/s3s-project/s3s?rev=4189c86154ed1fa4b89d2eaf2acfe04b7dcd0990#4189c86154ed1fa4b89d2eaf2acfe04b7dcd0990" +source = "git+https://github.com/s3s-project/s3s?rev=c2dc7b16535659904d4efff52c558fc039be1ef3#c2dc7b16535659904d4efff52c558fc039be1ef3" dependencies = [ "arc-swap", "arrayvec", @@ -8271,9 +8278,9 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.28" +version = "0.1.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "891d81b926048e76efe18581bf793546b4c0eaf8448d72be8de2bbee5fd166e1" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" dependencies = [ "windows-sys 0.61.2", ] @@ -8604,9 +8611,9 @@ dependencies = [ [[package]] name = "shadow-rs" -version = "1.7.0" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d9967e7c3cd89d19cd533d8fceb3e5dcee28cf97fe76441481abe1d32723039" +checksum = "3c798acfc78a69c7b038adde44084d8df875555b091da42c90ae46257cdcc41a" dependencies = [ "cargo_metadata", "const_format", @@ -9092,9 +9099,9 @@ dependencies = [ [[package]] name = "sysinfo" -version = "0.38.3" +version = "0.38.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d03c61d2a49c649a15c407338afe7accafde9dac869995dccb73e5f7ef7d9034" +checksum = "92ab6a2f8bfe508deb3c6406578252e491d299cbbf3bc0529ecc3313aee4a52f" dependencies = [ "libc", "memchr", @@ -9143,9 +9150,9 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.26.0" +version = "3.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82a72c767771b47409d2345987fda8628641887d5466101319899796367354a0" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", "getrandom 0.4.2", @@ -10657,18 +10664,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.40" +version = "0.8.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a789c6e490b576db9f7e6b6d661bcc9799f7c0ac8352f56ea20193b2681532e5" +checksum = "f2578b716f8a7a858b7f02d5bd870c14bf4ddbbcf3a4c05414ba6503640505e3" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.40" +version = "0.8.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f65c489a7071a749c849713807783f70672b28094011623e200cb86dcb835953" +checksum = "7e6cc098ea4d3bd6246687de65af3f920c430e236bee1e3bf2e441463f08a02f" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index 01a244b7c..29c6ba032 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -144,7 +144,7 @@ flatbuffers = "25.12.19" form_urlencoded = "1.2.2" prost = "0.14.3" quick-xml = "0.39.2" -rmcp = { version = "1.1.0" } +rmcp = { version = "1.2.0" } rmp = { version = "0.8.15" } rmp-serde = { version = "1.3.1" } serde = { version = "1.0.228", features = ["derive"] } @@ -162,7 +162,7 @@ hmac = { version = "0.13.0-rc.5" } jsonwebtoken = { version = "10.3.0", features = ["aws_lc_rs"] } openidconnect = { version = "4.0", default-features = false } pbkdf2 = "0.13.0-rc.9" -rsa = { version = "0.10.0-rc.16" } +rsa = { version = "0.10.0-rc.17" } rustls = { version = "0.23.37", default-features = false, features = ["aws-lc-rs", "logging", "tls12", "prefer-post-quantum", "std"] } rustls-pki-types = "1.14.0" sha1 = "0.11.0-rc.5" @@ -184,7 +184,7 @@ atoi = "2.0.0" atomic_enum = "0.3.0" aws-config = { version = "1.8.15" } aws-credential-types = { version = "1.2.14" } -aws-sdk-s3 = { version = "1.125.0", default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] } +aws-sdk-s3 = { version = "1.126.0", default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] } aws-smithy-http-client = { version = "1.1.12", default-features = false, features = ["default-client", "rustls-aws-lc"] } aws-smithy-types = { version = "1.4.6" } backtrace = "0.3.76" @@ -192,19 +192,19 @@ base64 = "0.22.1" base64-simd = "0.8.0" brotli = "8.0.2" cfg-if = "1.0.4" -clap = { version = "4.5.60", features = ["derive", "env"] } +clap = { version = "4.6.0", features = ["derive", "env"] } const-str = { version = "1.1.0", features = ["std", "proc"] } convert_case = "0.11.0" criterion = { version = "0.8", features = ["html_reports"] } crossbeam-queue = "0.3.12" -datafusion = "52.2.0" +datafusion = "52.3.0" derive_builder = "0.20.2" enumset = "1.1.10" faster-hex = "0.10.0" flate2 = "1.1.9" glob = "0.3.3" -google-cloud-storage = "1.8.0" -google-cloud-auth = "1.6.0" +google-cloud-storage = "1.9.0" +google-cloud-auth = "1.7.0" hashbrown = { version = "0.16.1", features = ["serde", "rayon"] } hex = "0.4.3" hex-simd = "0.8.0" @@ -239,9 +239,9 @@ rumqttc = { version = "0.25.1" } rustix = { version = "1.1.4", features = ["fs"] } rust-embed = { version = "8.11.0" } rustc-hash = { version = "2.1.1" } -s3s = { git = "https://github.com/s3s-project/s3s", rev = "4189c86154ed1fa4b89d2eaf2acfe04b7dcd0990", features = ["minio"] } +s3s = { git = "https://github.com/s3s-project/s3s", rev = "c2dc7b16535659904d4efff52c558fc039be1ef3", features = ["minio"] } serial_test = "3.4.0" -shadow-rs = { version = "1.7.0", default-features = false } +shadow-rs = { version = "1.7.1", default-features = false } siphasher = "1.0.2" smallvec = { version = "1.15.1", features = ["serde"] } smartstring = "1.0.1" @@ -249,9 +249,9 @@ snafu = "0.9.0" snap = "1.1.1" starshard = { version = "1.1.0", features = ["rayon", "async", "serde"] } strum = { version = "0.28.0", features = ["derive"] } -sysinfo = "0.38.3" +sysinfo = "0.38.4" temp-env = "0.3.6" -tempfile = "3.26.0" +tempfile = "3.27.0" test-case = "3.3.1" thiserror = "2.0.18" tracing = { version = "0.1.44" } @@ -279,7 +279,7 @@ opentelemetry-otlp = { version = "0.31.0", features = ["gzip-http", "reqwest-rus opentelemetry_sdk = { version = "0.31.0" } opentelemetry-semantic-conventions = { version = "0.31.0", features = ["semconv_experimental"] } opentelemetry-stdout = { version = "0.31.0" } -pyroscope = { version = "1.0.2", features = ["backend-pprof-rs"] } +pyroscope = { version = "2.0.0", features = ["backend-pprof-rs"] } # FTP and SFTP libunftp = { version = "0.23.0", features = ["experimental"] } diff --git a/crates/obs/Cargo.toml b/crates/obs/Cargo.toml index b401eed99..dc4a7f7db 100644 --- a/crates/obs/Cargo.toml +++ b/crates/obs/Cargo.toml @@ -38,6 +38,7 @@ rustfs-config = { workspace = true, features = ["constants", "observability"] } rustfs-utils = { workspace = true, features = ["ip", "path"] } flate2 = { workspace = true } glob = { workspace = true } +jiff = { workspace = true } metrics = { workspace = true } nvml-wrapper = { workspace = true, optional = true } opentelemetry = { workspace = true } @@ -56,7 +57,6 @@ tracing-subscriber = { workspace = true, features = ["registry", "std", "fmt", " tokio = { workspace = true, features = ["sync", "fs", "rt-multi-thread", "rt", "time", "macros"] } sysinfo = { workspace = true } thiserror = { workspace = true } -walkdir = { workspace = true } [target.'cfg(unix)'.dependencies] pyroscope = { workspace = true, features = ["backend-pprof-rs"] } diff --git a/crates/obs/README.md b/crates/obs/README.md index a43cb7b09..5d38290e9 100644 --- a/crates/obs/README.md +++ b/crates/obs/README.md @@ -1,7 +1,7 @@ # rustfs-obs Observability library for [RustFS](https://github.com/rustfs/rustfs) providing structured JSON -logging, distributed tracing, and metrics via OpenTelemetry. +logging, distributed tracing, metrics via OpenTelemetry, and continuous profiling via Pyroscope. --- @@ -10,9 +10,10 @@ logging, distributed tracing, and metrics via OpenTelemetry. | Feature | Description | |---------|-------------| | **Structured logging** | JSON-formatted logs via `tracing-subscriber` | -| **Rolling-file logging** | Daily / hourly rotation with automatic cleanup | +| **Rolling-file logging** | Daily / hourly rotation with automatic cleanup and high-precision timestamps | | **Distributed tracing** | OTLP/HTTP export to Jaeger, Tempo, or any OTel collector | | **Metrics** | OTLP/HTTP export, bridged from the `metrics` crate facade | +| **Continuous Profiling** | CPU/Memory profiling export to Pyroscope | | **Log cleanup** | Background task: size limits, gzip compression, retention policies | | **GPU metrics** *(optional)* | Enable with the `gpu` feature flag | @@ -79,7 +80,7 @@ The library selects a backend automatically based on configuration: ``` 1. Any OTLP endpoint set? - └─ YES → Full OTLP/HTTP pipeline (traces + metrics + logs) + └─ YES → Full OTLP/HTTP pipeline (traces + metrics + logs + profiling) 2. RUSTFS_OBS_LOG_DIRECTORY set to a non-empty path? └─ YES → Rolling-file JSON logging @@ -110,9 +111,11 @@ All configuration is read from environment variables at startup. | `RUSTFS_OBS_TRACE_ENDPOINT` | _(empty)_ | Dedicated trace endpoint (overrides root + `/v1/traces`) | | `RUSTFS_OBS_METRIC_ENDPOINT` | _(empty)_ | Dedicated metrics endpoint | | `RUSTFS_OBS_LOG_ENDPOINT` | _(empty)_ | Dedicated log endpoint | +| `RUSTFS_OBS_PROFILING_ENDPOINT` | _(empty)_ | Dedicated profiling endpoint (e.g. Pyroscope) | | `RUSTFS_OBS_TRACES_EXPORT_ENABLED` | `true` | Toggle trace export | | `RUSTFS_OBS_METRICS_EXPORT_ENABLED` | `true` | Toggle metrics export | | `RUSTFS_OBS_LOGS_EXPORT_ENABLED` | `true` | Toggle OTLP log export | +| `RUSTFS_OBS_PROFILING_EXPORT_ENABLED` | `true` | Toggle profiling export | | `RUSTFS_OBS_USE_STDOUT` | `false` | Mirror all signals to stdout alongside OTLP | | `RUSTFS_OBS_SAMPLE_RATIO` | `0.1` | Trace sampling ratio `0.0`–`1.0` | | `RUSTFS_OBS_METER_INTERVAL` | `15` | Metrics export interval (seconds) | @@ -132,7 +135,7 @@ All configuration is read from environment variables at startup. | `RUSTFS_OBS_LOGGER_LEVEL` | `info` | Log level; `RUST_LOG` syntax supported | | `RUSTFS_OBS_LOG_STDOUT_ENABLED` | `false` | When file logging is active, also mirror to stdout | | `RUSTFS_OBS_LOG_DIRECTORY` | _(empty)_ | **Directory for rolling log files. When empty, logs go to stdout only** | -| `RUSTFS_OBS_LOG_FILENAME` | `rustfs.log` | Base filename for rolling logs (date suffix added automatically) | +| `RUSTFS_OBS_LOG_FILENAME` | `rustfs.log` | Base filename for rolling logs. Rotated archives include a high-precision timestamp and counter. With the default `RUSTFS_OBS_LOG_MATCH_MODE=suffix`, names look like `-.rustfs.log` (e.g., `20231027103001.123456-0.rustfs.log`); with `prefix`, they look like `rustfs.log.-` (e.g., `rustfs.log.20231027103001.123456-0`). | | `RUSTFS_OBS_LOG_ROTATION_TIME` | `hourly` | Rotation granularity: `minutely`, `hourly`, or `daily` | | `RUSTFS_OBS_LOG_KEEP_FILES` | `30` | Number of rolling files to keep (also used by cleaner) | | `RUSTFS_OBS_LOG_MATCH_MODE` | `suffix` | File matching mode: `prefix` or `suffix` | @@ -247,21 +250,23 @@ use std::path::PathBuf; use rustfs_obs::LogCleaner; use rustfs_obs::types::FileMatchMode; -let cleaner = LogCleaner::new( +let cleaner = LogCleaner::builder( PathBuf::from("/var/log/rustfs"), "rustfs.log.".to_string(), // file_pattern - FileMatchMode::Prefix, // match_mode - 10, // keep_count - 2 * 1024 * 1024 * 1024, // max_total_size_bytes (2 GiB) - 0, // max_single_file_size_bytes (unlimited) - true, // compress_old_files - 6, // gzip_compression_level - 30, // compressed_file_retention_days - vec!["current.log".to_string()], // exclude_patterns - true, // delete_empty_files - 3600, // min_file_age_seconds (1 hour) - false, // dry_run -); + "rustfs.log".to_string(), // active_filename +) +.match_mode(FileMatchMode::Prefix) +.keep_files(10) +.max_total_size_bytes(2 * 1024 * 1024 * 1024) // 2 GiB +.max_single_file_size_bytes(0) // unlimited +.compress_old_files(true) +.gzip_compression_level(6) +.compressed_file_retention_days(30) +.exclude_patterns(vec!["current.log".to_string()]) +.delete_empty_files(true) +.min_file_age_seconds(3600) // 1 hour +.dry_run(false) +.build(); let (deleted, freed_bytes) = cleaner.cleanup().expect("cleanup failed"); println!("Deleted {deleted} files, freed {freed_bytes} bytes"); diff --git a/crates/obs/src/cleaner/README.md b/crates/obs/src/cleaner/README.md new file mode 100644 index 000000000..7f59a2b75 --- /dev/null +++ b/crates/obs/src/cleaner/README.md @@ -0,0 +1,71 @@ +# Log Cleaner Subsystem + +The `cleaner` module provides a robust, background log-file lifecycle manager for RustFS. It is designed to run periodically to enforce retention policies, compress old logs, and prevent disk exhaustion. + +## Architecture + +The cleaner operates as a pipeline: + +1. **Discovery (`scanner.rs`)**: Scans the configured log directory for eligible files. + * **Non-recursive**: Only scans the top-level directory for safety. + * **Filtering**: Ignores the currently active log file, files matching exclude patterns, and files that do not match the configured prefix/suffix pattern. + * **Performance**: Uses `std::fs::read_dir` directly to minimize overhead and syscalls. + +2. **Selection (`core.rs`)**: Applies retention policies to select files for deletion. + * **Keep Count**: Ensures at least `N` recent files are kept. + * **Total Size**: Deletes oldest files if the total size exceeds the limit. + * **Single File Size**: Deletes individual files that exceed a size limit (e.g., runaway logs). + +3. **Action (`core.rs` / `compress.rs`)**: + * **Compression**: Optionally compresses selected files using Gzip (level 1-9) before deletion. + * **Deletion**: Removes the original file (and eventually the compressed archive based on retention days). + +## Configuration + +The cleaner is configured via `LogCleanerBuilder`. When initialized via `rustfs-obs::init_obs`, it reads from environment variables. + +| Parameter | Env Var | Description | +|-----------|---------|-------------| +| `log_dir` | `RUSTFS_OBS_LOG_DIRECTORY` | The directory to scan. | +| `file_pattern` | `RUSTFS_OBS_LOG_FILENAME` | The base filename pattern (e.g., `rustfs.log`). | +| `active_filename` | (Derived) | The exact name of the currently active log file, excluded from cleanup. | +| `match_mode` | `RUSTFS_OBS_LOG_MATCH_MODE` | `prefix` or `suffix`. Determines how `file_pattern` is matched against filenames. | +| `keep_files` | `RUSTFS_OBS_LOG_KEEP_FILES` | Minimum number of rolling log files to keep. | +| `max_total_size_bytes` | `RUSTFS_OBS_LOG_MAX_TOTAL_SIZE_BYTES` | Maximum aggregate size of all log files. Oldest files are deleted to satisfy this. | +| `compress_old_files` | `RUSTFS_OBS_LOG_COMPRESS_OLD_FILES` | If `true`, files selected for removal are first gzipped. | +| `compressed_file_retention_days` | `RUSTFS_OBS_LOG_COMPRESSED_FILE_RETENTION_DAYS` | Age in days after which `.gz` files are deleted. | + +## Timestamp Format & Rotation + +The cleaner works in tandem with the `RollingAppender` in `telemetry/rolling.rs`. + +* **Rotation**: Logs are rotated based on time (Daily/Hourly/Minutely) or Size. +* **Naming**: Archived logs use a high-precision timestamp format: `YYYYMMDDHHMMSS.uuuuuu` (microseconds), plus a unique counter to prevent collisions. + * **Suffix Mode**: `-.` (e.g., `20231027103001.123456-0.rustfs.log`) + * **Prefix Mode**: `.-` (e.g., `rustfs.log.20231027103001.123456-0`) + +This high-precision naming ensures that files sort chronologically by name, and collisions are virtually impossible even under high load. + +## Usage Example + +```rust +use rustfs_obs::LogCleaner; +use rustfs_obs::types::FileMatchMode; +use std::path::PathBuf; + +let cleaner = LogCleaner::builder( + PathBuf::from("/var/log/rustfs"), + "rustfs.log.".to_string(), + "rustfs.log".to_string(), +) +.match_mode(FileMatchMode::Prefix) +.keep_files(10) +.max_total_size_bytes(1024 * 1024 * 100) // 100 MB +.compress_old_files(true) +.build(); + +// Run cleanup (blocking operation, spawn in a background task) +if let Ok((deleted, freed)) = cleaner.cleanup() { + println!("Cleaned up {} files, freed {} bytes", deleted, freed); +} +``` diff --git a/crates/obs/src/cleaner/core.rs b/crates/obs/src/cleaner/core.rs index 3ec6b2e43..6c8dbc819 100644 --- a/crates/obs/src/cleaner/core.rs +++ b/crates/obs/src/cleaner/core.rs @@ -15,7 +15,7 @@ //! Core log-file cleanup orchestration. //! //! [`LogCleaner`] is the public entry point for the cleanup subsystem. -//! Construct it with [`LogCleaner::new`] and call [`LogCleaner::cleanup`] +//! Construct it with [`LogCleaner::builder`] and call [`LogCleaner::cleanup`] //! periodically (e.g. from a `tokio::spawn`-ed loop). //! //! Internally the cleaner delegates to: @@ -24,9 +24,11 @@ //! - [`LogCleaner::select_files_to_delete`] — to apply count / size limits. use super::compress::compress_file; -use super::scanner::{collect_expired_compressed_files, collect_log_files}; +use super::scanner::{LogScanResult, scan_log_directory}; use super::types::{FileInfo, FileMatchMode}; +use rustfs_config::DEFAULT_LOG_KEEP_FILES; use std::path::PathBuf; +use std::time::SystemTime; use tracing::{debug, error, info}; /// Log-file lifecycle manager. @@ -43,6 +45,8 @@ pub struct LogCleaner { pub(super) log_dir: PathBuf, /// Pattern string to match files (used as prefix or suffix). pub(super) file_pattern: String, + /// Exact name of the active log file (to exclude from cleanup). + pub(super) active_filename: String, /// Whether to match by prefix or suffix. pub(super) match_mode: FileMatchMode, /// The cleaner will never delete files if doing so would leave fewer than @@ -70,54 +74,19 @@ pub struct LogCleaner { } impl LogCleaner { - /// Build a new [`LogCleaner`] with the supplied policy parameters. - /// - /// `exclude_patterns` is a list of glob strings (e.g. `"*.lock"`). Invalid - /// glob patterns are silently ignored. - /// - /// `gzip_compression_level` is clamped to the range `[1, 9]`. - #[allow(clippy::too_many_arguments)] - pub fn new( - log_dir: PathBuf, - file_pattern: String, - match_mode: FileMatchMode, - keep_files: usize, - max_total_size_bytes: u64, - max_single_file_size_bytes: u64, - compress_old_files: bool, - gzip_compression_level: u32, - compressed_file_retention_days: u64, - exclude_patterns: Vec, - delete_empty_files: bool, - min_file_age_seconds: u64, - dry_run: bool, - ) -> Self { - let patterns = exclude_patterns - .into_iter() - .filter_map(|p| glob::Pattern::new(&p).ok()) - .collect(); - - Self { - log_dir, - file_pattern, - match_mode, - keep_files, - max_total_size_bytes, - max_single_file_size_bytes, - compress_old_files, - gzip_compression_level: gzip_compression_level.clamp(1, 9), - compressed_file_retention_days, - exclude_patterns: patterns, - delete_empty_files, - min_file_age_seconds, - dry_run, - } + /// Create a builder to construct a `LogCleaner`. + pub fn builder( + log_dir: impl Into, + file_pattern: impl Into, + active_filename: impl Into, + ) -> LogCleanerBuilder { + LogCleanerBuilder::new(log_dir, file_pattern, active_filename) } /// Perform one full cleanup pass. /// /// Steps: - /// 1. Scan the log directory for managed files. + /// 1. Scan the log directory for managed files (excluding the active file). /// 2. Apply count/size policies to select files for deletion. /// 3. Optionally compress selected files, then delete them. /// 4. Collect and delete expired compressed archives. @@ -137,10 +106,15 @@ impl LogCleaner { let mut total_deleted = 0usize; let mut total_freed = 0u64; - // ── 1. Discover active log files ────────────────────────────────────── - let mut files = collect_log_files( + // ── 1. Discover active log files (Archives only) ────────────────────── + // We explicitly pass `active_filename` to exclude it from the list. + let LogScanResult { + mut logs, + mut compressed_archives, + } = scan_log_directory( &self.log_dir, &self.file_pattern, + Some(&self.active_filename), self.match_mode, &self.exclude_patterns, self.min_file_age_seconds, @@ -148,27 +122,19 @@ impl LogCleaner { self.dry_run, )?; - if files.is_empty() { - debug!("No log files found in directory: {:?}", self.log_dir); - } else { - files.sort_by_key(|f| f.modified); - let total_size: u64 = files.iter().map(|f| f.size).sum(); + // ── 2. Select + compress + delete (Regular Logs) ────────────────────── + if !logs.is_empty() { + logs.sort_by_key(|f| f.modified); + let total_size: u64 = logs.iter().map(|f| f.size).sum(); + info!( - "Found {} log files, total size: {} bytes ({:.2} MB)", - files.len(), + "Found {} regular log files, total size: {} bytes ({:.2} MB)", + logs.len(), total_size, total_size as f64 / 1024.0 / 1024.0 ); - // ── 2. Select + compress + delete ───────────────────────────────── - let (to_delete, to_rotate) = self.select_files_to_process(&files, total_size); - - // Handle rotation for active file if needed - if let Some(active_file) = to_rotate - && let Err(e) = self.rotate_active_file(&active_file) - { - error!("Failed to rotate active file {:?}: {}", active_file.path, e); - } + let to_delete = self.select_files_to_process(&logs, total_size); if !to_delete.is_empty() { let (d, f) = self.compress_and_delete(&to_delete)?; @@ -178,16 +144,13 @@ impl LogCleaner { } // ── 3. Remove expired compressed archives ───────────────────────────── - let expired_gz = collect_expired_compressed_files( - &self.log_dir, - &self.file_pattern, - self.match_mode, - self.compressed_file_retention_days, - )?; - if !expired_gz.is_empty() { - let (d, f) = self.delete_files(&expired_gz)?; - total_deleted += d; - total_freed += f; + if !compressed_archives.is_empty() && self.compressed_file_retention_days > 0 { + let expired = self.select_expired_compressed(&mut compressed_archives); + if !expired.is_empty() { + let (d, f) = self.delete_files(&expired)?; + total_deleted += d; + total_freed += f; + } } if total_deleted > 0 || total_freed > 0 { @@ -204,28 +167,19 @@ impl LogCleaner { // ─── Selection ──────────────────────────────────────────────────────────── - /// Choose which files from `files` (sorted oldest-first) should be deleted or rotated. + /// Choose which files from `files` (sorted oldest-first) should be deleted. /// /// The algorithm respects three constraints in order: - /// 1. Always keep at least `keep_files` files. + /// 1. Always keep at least `keep_files` files (archives). /// 2. Delete old files while the total size exceeds `max_total_size_bytes`. /// 3. Delete any file whose individual size exceeds `max_single_file_size_bytes`. - /// - /// **Note**: The most recent file (assumed to be the active log) is exempt - /// from size-based deletion. If it exceeds the size limit, it is returned - /// as `to_rotate`. - pub(super) fn select_files_to_process(&self, files: &[FileInfo], total_size: u64) -> (Vec, Option) { + pub(super) fn select_files_to_process(&self, files: &[FileInfo], total_size: u64) -> Vec { let mut to_delete = Vec::new(); - let mut to_rotate = None; if files.is_empty() { - return (to_delete, to_rotate); + return to_delete; } - // Identify the index of the most recent file (last in the sorted list). - // We will protect this file from size-based deletion. - let active_file_idx = files.len() - 1; - // Calculate how many files we *must* delete to satisfy keep_files. let must_delete_count = files.len().saturating_sub(self.keep_files); @@ -244,142 +198,111 @@ impl LogCleaner { let over_total = self.max_total_size_bytes > 0 && current_size > self.max_total_size_bytes; // Condition 3: Enforce max_single_file_size_bytes. + // Note: Since active file is excluded, if an archive is > max_single, it means it + // was rotated out being too large (likely) or we lowered the limit. It should be deleted. let over_single = self.max_single_file_size_bytes > 0 && file.size > self.max_single_file_size_bytes; if over_total { - // If we are over total size, we delete unless it's the active file. - if idx == active_file_idx { - debug!( - "Active log file contributes to total size limit overflow, but skipping deletion to preserve current logs." - ); - } else { - current_size = current_size.saturating_sub(file.size); - to_delete.push(file.clone()); - } + current_size = current_size.saturating_sub(file.size); + to_delete.push(file.clone()); } else if over_single { - // For single file limits, we MUST NOT delete the active file. - if idx == active_file_idx { - // Mark active file for rotation instead of deletion - to_rotate = Some(file.clone()); - } else { - debug!( - "File exceeds single-file size limit: {:?} ({} > {} bytes)", - file.path, file.size, self.max_single_file_size_bytes - ); - current_size = current_size.saturating_sub(file.size); - to_delete.push(file.clone()); - } + debug!( + "Archive exceeds single-file size limit: {:?} ({} > {} bytes). Deleting.", + file.path, file.size, self.max_single_file_size_bytes + ); + current_size = current_size.saturating_sub(file.size); + to_delete.push(file.clone()); } } - (to_delete, to_rotate) + to_delete } - // ─── Rotation ───────────────────────────────────────────────────────────── + /// Select compressed files that have exceeded the retention period. + fn select_expired_compressed(&self, files: &mut [FileInfo]) -> Vec { + let retention = std::time::Duration::from_secs(self.compressed_file_retention_days * 24 * 3600); + let now = SystemTime::now(); + let mut expired = Vec::new(); - /// Rotate the active file by renaming it with a timestamp suffix. - /// The original filename will be recreated by the logging appender on next write. - fn rotate_active_file(&self, file: &FileInfo) -> Result<(), std::io::Error> { - if self.dry_run { - info!("[DRY RUN] Would rotate active file: {:?} ({} bytes)", file.path, file.size); - return Ok(()); - } - - // Generate timestamp: unix timestamp in seconds - let timestamp = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map_err(std::io::Error::other)? - .as_secs(); - - let file_name = file - .path - .file_name() - .and_then(|n| n.to_str()) - .ok_or_else(|| std::io::Error::new(std::io::ErrorKind::InvalidData, "Invalid filename"))?; - - // Construct the rotated filename. - // We must ensure the new filename still matches the file_pattern so it can be discovered - // by the scanner in future runs (and eventually deleted). - // - // Suffix mode: Insert timestamp BEFORE the suffix. - // Example: "2026-03-01.rustfs.log" (pattern="rustfs.log") - // -> "2026-03-01.1740810000.rustfs.log" - // - // Prefix mode: Append timestamp at the end. - // Example: "app.log" (pattern="app") - // -> "app.log.1740810000" - let rotated_name = match self.match_mode { - FileMatchMode::Suffix => { - if let Some(base) = file_name.strip_suffix(&self.file_pattern) { - let mut new_name = String::with_capacity(file_name.len() + 20); - new_name.push_str(base); - - // Ensure separator between base and timestamp - if !base.is_empty() && !base.ends_with('.') { - new_name.push('.'); - } - new_name.push_str(×tamp.to_string()); - - // Ensure separator between timestamp and suffix - if !self.file_pattern.starts_with('.') { - new_name.push('.'); - } - new_name.push_str(&self.file_pattern); - - new_name - } else { - // Should not happen if scanner works correctly, but fallback safely - format!("{}.{}", file_name, timestamp) - } + for file in files { + if let Ok(age) = now.duration_since(file.modified) + && age > retention + { + expired.push(file.clone()); } - FileMatchMode::Prefix => { - // For prefix matching, appending to the end preserves the prefix. - format!("{}.{}", file_name, timestamp) - } - }; - - let rotated_path = file.path.with_file_name(&rotated_name); - - // Check if target already exists to avoid overwriting (unlikely with timestamp but possible) - if rotated_path.exists() { - return Err(std::io::Error::new( - std::io::ErrorKind::AlreadyExists, - format!("Rotated file already exists: {:?}", rotated_path), - )); } - info!("Rotating active log file: {:?} -> {:?}", file.path, rotated_path); - - // Rename the current active file to the rotated name. - // The logging appender (tracing-appender) will automatically create a new file - // with the original name when it next attempts to write. - // Note: On Linux/Unix, this rename is atomic and safe even if the file is open. - if let Err(e) = std::fs::rename(&file.path, &rotated_path) { - // Add context to the error - return Err(std::io::Error::new( - e.kind(), - format!("Failed to rename {:?} to {:?}: {}", file.path, rotated_path, e), - )); - } - - Ok(()) + expired } // ─── Compression + deletion ─────────────────────────────────────────────── + /// Securely delete a file, preventing symlink attacks (TOCTOU). + /// + /// This function verifies that the path is not a symlink before attempting deletion. + /// While strictly speaking a race condition is still theoretically possible between + /// `symlink_metadata` and `remove_file`, this check covers the vast majority of + /// privilege escalation vectors where a user replaces a log file with a symlink + /// to a system file. + fn secure_delete(&self, path: &PathBuf) -> std::io::Result<()> { + // 1. Lstat (symlink_metadata) - do not follow links + let meta = std::fs::symlink_metadata(path)?; + + // 2. Symlink Check + // If it's a symlink, we NEVER delete it. It might point to /etc/passwd. + // In a log directory, symlinks are unexpected and dangerous. + if meta.file_type().is_symlink() { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!("Security: refusing to delete symlink: {:?}", path), + )); + } + + // 3. Perform Deletion + std::fs::remove_file(path) + } + /// Optionally compress and then delete the given files. /// /// This function is synchronous and blocking. It should be called within a /// `spawn_blocking` task if running in an async context. fn compress_and_delete(&self, files: &[FileInfo]) -> Result<(usize, u64), std::io::Error> { - if self.compress_old_files { - for f in files { - if let Err(e) = compress_file(&f.path, self.gzip_compression_level, self.dry_run) { - tracing::warn!("Failed to compress {:?}: {}", f.path, e); + let mut total_deleted = 0; + let mut total_freed = 0; + + for f in files { + let mut deleted_size = 0; + if self.compress_old_files { + match compress_file(&f.path, self.gzip_compression_level, self.dry_run) { + Ok(_) => {} + Err(e) => { + tracing::warn!("Failed to compress {:?}: {}", f.path, e); + } } } + + // Now delete + if self.dry_run { + info!("[DRY RUN] Would delete: {:?} ({} bytes)", f.path, f.size); + deleted_size = f.size; + } else { + match self.secure_delete(&f.path) { + Ok(()) => { + debug!("Deleted: {:?}", f.path); + deleted_size = f.size; + } + Err(e) => { + error!("Failed to delete {:?}: {}", f.path, e); + } + } + } + if deleted_size > 0 { + total_deleted += 1; + total_freed += deleted_size; + } } - self.delete_files(files) + + Ok((total_deleted, total_freed)) } /// Delete all files in `files`, logging each operation. @@ -398,7 +321,7 @@ impl LogCleaner { deleted += 1; freed += f.size; } else { - match std::fs::remove_file(&f.path) { + match self.secure_delete(&f.path) { Ok(()) => { debug!("Deleted: {:?}", f.path); deleted += 1; @@ -414,3 +337,125 @@ impl LogCleaner { Ok((deleted, freed)) } } + +/// Builder for [`LogCleaner`]. +pub struct LogCleanerBuilder { + log_dir: PathBuf, + file_pattern: String, + active_filename: String, + match_mode: FileMatchMode, + keep_files: usize, + max_total_size_bytes: u64, + max_single_file_size_bytes: u64, + compress_old_files: bool, + gzip_compression_level: u32, + compressed_file_retention_days: u64, + exclude_patterns: Vec, + delete_empty_files: bool, + min_file_age_seconds: u64, + dry_run: bool, +} + +impl LogCleanerBuilder { + pub fn new(log_dir: impl Into, file_pattern: impl Into, active_filename: impl Into) -> Self { + Self { + log_dir: log_dir.into(), + file_pattern: file_pattern.into(), + active_filename: active_filename.into(), + match_mode: FileMatchMode::Prefix, + // Default to a safe non-zero value so that a builder created + // without an explicit `keep_files()` call does not immediately + // delete all matching log files. + keep_files: DEFAULT_LOG_KEEP_FILES, + max_total_size_bytes: 0, + max_single_file_size_bytes: 0, + compress_old_files: false, + gzip_compression_level: 6, + compressed_file_retention_days: 0, + exclude_patterns: Vec::new(), + delete_empty_files: false, + min_file_age_seconds: 0, + dry_run: false, + } + } + + pub fn match_mode(mut self, match_mode: FileMatchMode) -> Self { + self.match_mode = match_mode; + self + } + + pub fn keep_files(mut self, keep_files: usize) -> Self { + self.keep_files = keep_files; + self + } + + pub fn max_total_size_bytes(mut self, max_total_size_bytes: u64) -> Self { + self.max_total_size_bytes = max_total_size_bytes; + self + } + + pub fn max_single_file_size_bytes(mut self, max_single_file_size_bytes: u64) -> Self { + self.max_single_file_size_bytes = max_single_file_size_bytes; + self + } + + pub fn compress_old_files(mut self, compress_old_files: bool) -> Self { + self.compress_old_files = compress_old_files; + self + } + + pub fn gzip_compression_level(mut self, gzip_compression_level: u32) -> Self { + self.gzip_compression_level = gzip_compression_level; + self + } + + pub fn compressed_file_retention_days(mut self, days: u64) -> Self { + self.compressed_file_retention_days = days; + self + } + + pub fn exclude_patterns(mut self, patterns: Vec) -> Self { + self.exclude_patterns = patterns; + self + } + + pub fn delete_empty_files(mut self, delete_empty_files: bool) -> Self { + self.delete_empty_files = delete_empty_files; + self + } + + pub fn min_file_age_seconds(mut self, seconds: u64) -> Self { + self.min_file_age_seconds = seconds; + self + } + + pub fn dry_run(mut self, dry_run: bool) -> Self { + self.dry_run = dry_run; + self + } + + pub fn build(self) -> LogCleaner { + let patterns = self + .exclude_patterns + .into_iter() + .filter_map(|p| glob::Pattern::new(&p).ok()) + .collect(); + + LogCleaner { + log_dir: self.log_dir, + file_pattern: self.file_pattern, + active_filename: self.active_filename, + match_mode: self.match_mode, + keep_files: self.keep_files, + max_total_size_bytes: self.max_total_size_bytes, + max_single_file_size_bytes: self.max_single_file_size_bytes, + compress_old_files: self.compress_old_files, + gzip_compression_level: self.gzip_compression_level.clamp(1, 9), + compressed_file_retention_days: self.compressed_file_retention_days, + exclude_patterns: patterns, + delete_empty_files: self.delete_empty_files, + min_file_age_seconds: self.min_file_age_seconds, + dry_run: self.dry_run, + } + } +} diff --git a/crates/obs/src/cleaner/mod.rs b/crates/obs/src/cleaner/mod.rs index 73545fcd5..4676bc59d 100644 --- a/crates/obs/src/cleaner/mod.rs +++ b/crates/obs/src/cleaner/mod.rs @@ -33,21 +33,23 @@ //! use rustfs_obs::LogCleaner; //! use rustfs_obs::types::FileMatchMode; //! -//! let cleaner = LogCleaner::new( +//! let cleaner = LogCleaner::builder( //! PathBuf::from("/var/log/rustfs"), //! "rustfs.log.".to_string(), -//! FileMatchMode::Prefix, -//! 10, // keep_files -//! 2 * 1024 * 1024 * 1024, // max_total_size_bytes (2 GiB) -//! 0, // max_single_file_size_bytes (unlimited) -//! true, // compress_old_files -//! 6, // gzip_compression_level -//! 30, // compressed_file_retention_days -//! vec![], // exclude_patterns -//! true, // delete_empty_files -//! 3600, // min_file_age_seconds (1 hour) -//! false, // dry_run -//! ); +//! "rustfs.log".to_string(), +//! ) +//! .match_mode(FileMatchMode::Prefix) +//! .keep_files(10) +//! .max_total_size_bytes(2 * 1024 * 1024 * 1024) // 2 GiB +//! .max_single_file_size_bytes(0) // unlimited +//! .compress_old_files(true) +//! .gzip_compression_level(6) +//! .compressed_file_retention_days(30) +//! .exclude_patterns(vec![]) +//! .delete_empty_files(true) +//! .min_file_age_seconds(3600) // 1 hour +//! .dry_run(false) +//! .build(); //! //! let (deleted, freed_bytes) = cleaner.cleanup().expect("cleanup failed"); //! println!("Deleted {deleted} files, freed {freed_bytes} bytes"); @@ -79,21 +81,13 @@ mod tests { /// Build a cleaner with sensible test defaults (no compression, no age gate). fn make_cleaner(dir: std::path::PathBuf, keep: usize, max_bytes: u64) -> LogCleaner { - LogCleaner::new( - dir, - "app.log.".to_string(), - FileMatchMode::Prefix, - keep, - max_bytes, - 0, // max_single_file_size_bytes - false, // compress_old_files - 6, // gzip_compression_level - 30, // compressed_file_retention_days - Vec::new(), // exclude_patterns - true, // delete_empty_files - 0, // min_file_age_seconds (0 = no age gate in tests) - false, // dry_run - ) + LogCleaner::builder(dir, "app.log.".to_string(), "app.log".to_string()) + .match_mode(FileMatchMode::Prefix) + .keep_files(keep) + .max_total_size_bytes(max_bytes) + .min_file_age_seconds(0) // 0 = no age gate in tests + .delete_empty_files(true) + .build() } #[test] @@ -141,11 +135,15 @@ mod tests { create_log_file(&dir, "app.log.2024-01-02", 1024)?; create_log_file(&dir, "other.log", 512)?; // different prefix - let cleaner = make_cleaner(dir.clone(), 1, 512); + // keep_files=1 and max_bytes=1500: deleting one managed file (1024 bytes) leaves + // a single managed file of 1024 bytes, which satisfies both the file-count and + // size limits. "other.log" (different prefix) must never be touched. + let cleaner = make_cleaner(dir.clone(), 1, 1500); let (deleted, _) = cleaner.cleanup()?; - // "other.log" must not be counted or deleted. + // "other.log" must not be counted or deleted; only 1 managed file removed. assert_eq!(deleted, 1, "only managed files should be deleted"); + assert!(dir.join("other.log").exists(), "unrelated file must not be deleted"); Ok(()) } @@ -158,8 +156,8 @@ mod tests { create_log_file(&dir, "app.log.2024-01-02", 2048)?; create_log_file(&dir, "other.log", 512)?; - let files = scanner::collect_log_files(&dir, "app.log.", FileMatchMode::Prefix, &[], 0, true, false)?; - assert_eq!(files.len(), 2, "scanner should find exactly 2 managed files"); + let result = scanner::scan_log_directory(&dir, "app.log.", Some("app.log"), FileMatchMode::Prefix, &[], 0, true, false)?; + assert_eq!(result.logs.len(), 2, "scanner should find exactly 2 managed files"); Ok(()) } @@ -172,21 +170,12 @@ mod tests { create_log_file(&dir, "app.log.2024-01-02", 1024)?; create_log_file(&dir, "app.log.2024-01-03", 1024)?; - let cleaner = LogCleaner::new( - dir.clone(), - "app.log.".to_string(), - FileMatchMode::Prefix, - 1, - 1024, - 0, - false, - 6, - 30, - vec![], - true, - 0, - true, - ); + let cleaner = LogCleaner::builder(dir.clone(), "app.log.".to_string(), "app.log".to_string()) + .match_mode(FileMatchMode::Prefix) + .keep_files(1) + .max_total_size_bytes(1024) + .dry_run(true) + .build(); let (deleted, _freed) = cleaner.cleanup()?; // dry_run=true reports deletions but doesn't actually remove files. @@ -204,21 +193,11 @@ mod tests { create_log_file(&dir, "2026-03-01-06-22.rustfs.log", 1024)?; create_log_file(&dir, "other.log", 1024)?; // not managed - let cleaner = LogCleaner::new( - dir.clone(), - "rustfs.log".to_string(), - FileMatchMode::Suffix, - 1, - 1024, - 0, - false, - 6, - 30, - vec![], - true, - 0, - false, - ); + let cleaner = LogCleaner::builder(dir.clone(), ".rustfs.log".to_string(), "current.log".to_string()) + .match_mode(FileMatchMode::Suffix) + .keep_files(1) + .max_total_size_bytes(1024) + .build(); let (deleted, freed) = cleaner.cleanup()?; assert_eq!(deleted, 1, "should delete exactly one file"); diff --git a/crates/obs/src/cleaner/scanner.rs b/crates/obs/src/cleaner/scanner.rs index b40ea24a5..f4331da31 100644 --- a/crates/obs/src/cleaner/scanner.rs +++ b/crates/obs/src/cleaner/scanner.rs @@ -14,55 +14,91 @@ //! Filesystem scanner for discovering log files eligible for cleanup. //! -//! This module is intentionally kept read-only: it does **not** delete or -//! compress any files — it only reports what it found. +//! This module is primarily read-only: it reports what files it found. +//! The one exception is zero-byte file removal — when `delete_empty_files` +//! is enabled, `scan_log_directory` removes empty regular files as part of +//! the scan so that they are not counted in retention calculations. use super::types::{FileInfo, FileMatchMode}; use rustfs_config::observability::DEFAULT_OBS_LOG_GZIP_COMPRESSION_ALL_EXTENSION; +use std::fs; use std::path::Path; -use std::time::{Duration, SystemTime}; +use std::time::SystemTime; use tracing::debug; -use walkdir::WalkDir; -/// Collect all log files in `log_dir` whose name matches `file_pattern` based on `match_mode`. +/// Result of a single pass directory scan. +pub(super) struct LogScanResult { + /// Regular log files eligible for deletion/compression. + pub logs: Vec, + /// Already compressed files eligible for expiry deletion. + pub compressed_archives: Vec, +} + +/// Perform a single-pass scan of the log directory. /// -/// Files that: -/// - are already compressed (`.gz` extension), -/// - are zero-byte and `delete_empty_files` is `true` (these are handled -/// immediately by the caller), or -/// - match one of the `exclude_patterns`, -/// - were modified more recently than `min_file_age_seconds` seconds ago, -/// -/// are skipped and not returned in the result list. +/// This function iterates over the directory entries once and categorizes them +/// into regular logs or compressed archives based on extensions and patterns. /// /// # Arguments /// * `log_dir` - Root directory to scan (depth 1 only, no recursion). /// * `file_pattern` - Pattern string to match filenames. +/// * `active_filename` - The name of the currently active log file (to be excluded). /// * `match_mode` - Whether to match by prefix or suffix. /// * `exclude_patterns` - Compiled glob patterns; matching files are skipped. -/// * `min_file_age_seconds` - Files younger than this threshold are skipped. -/// * `delete_empty_files` - When `true`, zero-byte files trigger an immediate -/// delete by the caller before the rest of cleanup runs. -pub(super) fn collect_log_files( +/// * `min_file_age_seconds` - Files younger than this threshold are skipped (for regular logs). +/// * `delete_empty_files` - When `true`, zero-byte regular files that match +/// the pattern are deleted immediately inside this function and excluded +/// from the returned [`LogScanResult`]. +#[allow(clippy::too_many_arguments)] +pub(super) fn scan_log_directory( log_dir: &Path, file_pattern: &str, + active_filename: Option<&str>, match_mode: FileMatchMode, exclude_patterns: &[glob::Pattern], min_file_age_seconds: u64, delete_empty_files: bool, dry_run: bool, -) -> Result, std::io::Error> { - let mut files = Vec::new(); +) -> Result { + let mut logs = Vec::new(); + let mut compressed_archives = Vec::new(); let now = SystemTime::now(); - for entry in WalkDir::new(log_dir) - .max_depth(1) - .follow_links(false) - .into_iter() - .filter_map(|e| e.ok()) - { + // Use read_dir for a lightweight, non-recursive scan. + let entries = match fs::read_dir(log_dir) { + Ok(entries) => entries, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => { + // If the log directory does not exist (or was removed), treat this + // as "no files found" instead of failing the whole cleanup pass. + return Ok(LogScanResult { + logs, + compressed_archives, + }); + } + Err(e) => return Err(e), + }; + + for entry in entries { + let entry = match entry { + Ok(e) => e, + Err(_) => continue, // Skip unreadable entries + }; + let path = entry.path(); - if !path.is_file() { + + // We only care about regular files inside the log directory. + // Use `fs::symlink_metadata` (which does *not* follow symlinks) for + // both the file-type check *and* size/mtime collection below. Using + // `entry.metadata()` or `Path::is_file()` (both of which follow + // symlinks) would allow a symlink placed in the log directory to reach + // files outside the tree, and would introduce a TOCTOU window between + // the type-check and the metadata read. + let metadata = match fs::symlink_metadata(&path) { + Ok(md) => md, + Err(_) => continue, + }; + let file_type = metadata.file_type(); + if !file_type.is_file() { continue; } @@ -71,42 +107,53 @@ pub(super) fn collect_log_files( None => continue, }; - // Match filename based on mode + // 1. Explicitly skip the active log file (if known). + if let Some(active) = active_filename + && filename == active + { + continue; + } + + // 2. Check exclusion patterns early. + if is_excluded(filename, exclude_patterns) { + debug!("Excluding file from cleanup: {:?}", filename); + continue; + } + + // 3. Classify file type and check pattern match. + let is_compressed = filename.ends_with(DEFAULT_OBS_LOG_GZIP_COMPRESSION_ALL_EXTENSION); + + // For matching, we need the "base" name. + // If compressed: "foo.log.gz" -> check "foo.log" + // If regular: "foo.log" -> check "foo.log" + let name_to_match = if is_compressed { + &filename[..filename.len() - DEFAULT_OBS_LOG_GZIP_COMPRESSION_ALL_EXTENSION.len()] + } else { + filename + }; + let matches = match match_mode { - FileMatchMode::Prefix => filename.starts_with(file_pattern), - FileMatchMode::Suffix => filename.ends_with(file_pattern), + FileMatchMode::Prefix => name_to_match.starts_with(file_pattern), + FileMatchMode::Suffix => name_to_match.ends_with(file_pattern), }; if !matches { continue; } - // Compressed files are handled by collect_compressed_files. - if filename.ends_with(DEFAULT_OBS_LOG_GZIP_COMPRESSION_ALL_EXTENSION) { - continue; - } - - // Honour exclusion patterns. - if is_excluded(filename, exclude_patterns) { - debug!("Excluding file from cleanup: {:?}", filename); - continue; - } - - let metadata = match entry.metadata() { - Ok(m) => m, - Err(_) => continue, - }; + // 4. Gather size and mtime from the already-fetched symlink_metadata + // (reuse; no second syscall, no symlink following). + let file_size = metadata.len(); let modified = match metadata.modified() { Ok(t) => t, - Err(_) => continue, + Err(_) => continue, // Skip files where we can't read modification time }; - let file_size = metadata.len(); - // Delete zero-byte files immediately (outside the normal selection - // logic) when the feature is enabled. - if file_size == 0 && delete_empty_files { + // 5. Handle zero-byte files (Regular logs only). + // We generally don't delete empty compressed files implicitly, but let's stick to regular files logic. + if !is_compressed && file_size == 0 && delete_empty_files { if !dry_run { - if let Err(e) = std::fs::remove_file(path) { + if let Err(e) = std::fs::remove_file(&path) { tracing::warn!("Failed to delete empty file {:?}: {}", path, e); } else { debug!("Deleted empty file: {:?}", path); @@ -117,99 +164,33 @@ pub(super) fn collect_log_files( continue; } - // Skip files that are too young. - if let Ok(age) = now.duration_since(modified) + // 6. Age Check (Regular logs only). + // Compressed files have their own retention check in the caller. + if !is_compressed + && let Ok(age) = now.duration_since(modified) && age.as_secs() < min_file_age_seconds { - debug!( - "Skipping file (too new): {:?}, age: {}s, min_age: {}s", - filename, - age.as_secs(), - min_file_age_seconds - ); + // Too young to be touched. continue; } - files.push(FileInfo { - path: path.to_path_buf(), + let info = FileInfo { + path, size: file_size, modified, - }); - } - - Ok(files) -} - -/// Collect compressed `.gz` log files whose age exceeds the retention period. -/// -/// When `compressed_file_retention_days` is `0` the function returns immediately -/// without collecting anything (files are kept indefinitely). -/// -/// # Arguments -/// * `log_dir` - Root directory to scan. -/// * `file_pattern` - Pattern string to match filenames. -/// * `match_mode` - Whether to match by prefix or suffix. -/// * `compressed_file_retention_days` - Files older than this are eligible for -/// deletion; `0` means never delete compressed files. -pub(super) fn collect_expired_compressed_files( - log_dir: &Path, - file_pattern: &str, - match_mode: FileMatchMode, - compressed_file_retention_days: u64, -) -> Result, std::io::Error> { - if compressed_file_retention_days == 0 { - return Ok(Vec::new()); - } - - let retention = Duration::from_secs(compressed_file_retention_days * 24 * 3600); - let now = SystemTime::now(); - let mut files = Vec::new(); - - for entry in WalkDir::new(log_dir) - .max_depth(1) - .follow_links(false) - .into_iter() - .filter_map(|e| e.ok()) - { - let path = entry.path(); - if !path.is_file() { - continue; - } - - let filename = match path.file_name().and_then(|n| n.to_str()) { - Some(f) => f, - None => continue, }; - if !filename.ends_with(DEFAULT_OBS_LOG_GZIP_COMPRESSION_ALL_EXTENSION) { - continue; - } - - // Check if the base filename (without .gz) matches the pattern - let base_filename = &filename[..filename.len() - 3]; - let matches = match match_mode { - FileMatchMode::Prefix => base_filename.starts_with(file_pattern), - FileMatchMode::Suffix => base_filename.ends_with(file_pattern), - }; - - if !matches { - continue; - } - - let Ok(metadata) = entry.metadata() else { continue }; - let Ok(modified) = metadata.modified() else { continue }; - let Ok(age) = now.duration_since(modified) else { continue }; - - if age > retention { - files.push(FileInfo { - path: path.to_path_buf(), - size: metadata.len(), - modified, - }); + if is_compressed { + compressed_archives.push(info); + } else { + logs.push(info); } } - Ok(files) + Ok(LogScanResult { + logs, + compressed_archives, + }) } /// Returns `true` if `filename` matches any of the compiled exclusion patterns. diff --git a/crates/obs/src/config.rs b/crates/obs/src/config.rs index 7788ef7fc..4f8a0e4b9 100644 --- a/crates/obs/src/config.rs +++ b/crates/obs/src/config.rs @@ -210,7 +210,11 @@ impl OtelConfig { // `log_keep_files` is the single source of truth for file retention count. // It defaults to `DEFAULT_LOG_KEEP_FILES` (30). - let log_keep_files = Some(get_env_usize(ENV_OBS_LOG_KEEP_FILES, DEFAULT_LOG_KEEP_FILES)); + let mut log_keep_files = get_env_usize(ENV_OBS_LOG_KEEP_FILES, DEFAULT_LOG_KEEP_FILES); + if log_keep_files == 0 { + log_keep_files = DEFAULT_LOG_KEEP_FILES; + } + let log_keep_files = Some(log_keep_files); // `log_rotation_time` drives the rolling-appender rotation period. let log_rotation_time = Some(get_env_str(ENV_OBS_LOG_ROTATION_TIME, DEFAULT_LOG_ROTATION_TIME)); diff --git a/crates/obs/src/telemetry/filter.rs b/crates/obs/src/telemetry/filter.rs index cc75cb0ba..3211baec2 100644 --- a/crates/obs/src/telemetry/filter.rs +++ b/crates/obs/src/telemetry/filter.rs @@ -75,12 +75,25 @@ pub(super) fn build_env_filter(logger_level: &str, default_level: Option<&str>) .map(EnvFilter::new) .unwrap_or_else(|| EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(level))); - // Suppress chatty infrastructure crates unless the operator explicitly - // requests trace/debug output. if should_suppress_noisy_crates(logger_level, default_level, rust_log.as_deref()) { - let directives: SmallVec<[&str; 5]> = smallvec::smallvec!["hyper", "tonic", "h2", "reqwest", "tower"]; - for directive in directives { - filter = filter.add_directive(format!("{directive}=off").parse().unwrap()); + let directives: SmallVec<[(&str, &str); 6]> = smallvec::smallvec![ + ("hyper", "off"), + ("tonic", "off"), + ("h2", "off"), + ("reqwest", "off"), + ("tower", "off"), + // HTTP request logs are demoted to WARN to reduce volume in production. + ("rustfs::server::http", "warn"), + ]; + for (crate_name, level) in directives { + match format!("{crate_name}={level}").parse() { + Ok(directive) => filter = filter.add_directive(directive), + Err(e) => { + // The directive strings are compile-time constants, so this + // branch should never be reached; emit a diagnostic just in case. + eprintln!("obs: invalid log filter directive '{crate_name}={level}': {e}"); + } + } } } diff --git a/crates/obs/src/telemetry/local.rs b/crates/obs/src/telemetry/local.rs index 6e8964106..dec0f4b19 100644 --- a/crates/obs/src/telemetry/local.rs +++ b/crates/obs/src/telemetry/local.rs @@ -32,6 +32,7 @@ use crate::cleaner::types::FileMatchMode; use crate::config::OtelConfig; use crate::global::OBSERVABILITY_METRIC_ENABLED; use crate::telemetry::filter::build_env_filter; +use crate::telemetry::rolling::{RollingAppender, Rotation}; use metrics::counter; use rustfs_config::observability::{ DEFAULT_OBS_LOG_CLEANUP_INTERVAL_SECONDS, DEFAULT_OBS_LOG_COMPRESS_OLD_FILES, DEFAULT_OBS_LOG_COMPRESSED_FILE_RETENTION_DAYS, @@ -175,7 +176,7 @@ fn init_file_logging_internal( // ── 3. Choose rotation strategy ────────────────────────────────────────── // `log_rotation_time` drives the rolling-appender rotation period. - let rotation = config + let rotation_str = config .log_rotation_time .as_deref() .unwrap_or(DEFAULT_LOG_ROTATION_TIME) @@ -187,28 +188,20 @@ fn init_file_logging_internal( _ => FileMatchMode::Suffix, }; - use tracing_appender::rolling::{RollingFileAppender, Rotation}; - let file_appender = { - let rotation = match rotation.as_str() { - "minutely" => Rotation::MINUTELY, - "hourly" => Rotation::HOURLY, - _ => Rotation::DAILY, - }; - - let mut builder = RollingFileAppender::builder() - .rotation(rotation) - .max_log_files(keep_files * 3); // Make sure there are some data files to archive to avoid premature deletion - - match match_mode { - FileMatchMode::Prefix => builder = builder.filename_prefix(log_filename), - FileMatchMode::Suffix => builder = builder.filename_suffix(log_filename), - } - - builder - .build(log_directory) - .map_err(|e| TelemetryError::Io(format!("failed to initialize rolling file appender: {e}")))? + let rotation = match rotation_str.as_str() { + "minutely" => Rotation::Minutely, + "hourly" => Rotation::Hourly, + "daily" => Rotation::Daily, + _ => Rotation::Daily, }; + let max_single_file_size = config + .log_max_single_file_size_bytes + .unwrap_or(DEFAULT_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES); + + let file_appender = + RollingAppender::new(log_directory, log_filename.to_string(), rotation, max_single_file_size, match_mode)?; + let (non_blocking, guard) = tracing_appender::non_blocking(file_appender); // ── 4. Build subscriber layers ──────────────────────────────────────────── @@ -270,7 +263,7 @@ fn init_file_logging_internal( info!( "Init file logging at '{}', rotation: {}, keep {} files", - log_directory, rotation, keep_files + log_directory, rotation_str, keep_files ); Ok(OtelGuard { @@ -352,6 +345,7 @@ fn spawn_cleanup_task( // Use suffix matching for log files like "2026-03-01-06-21.rustfs.log" // where "rustfs.log" is the suffix. let file_pattern = config.log_filename.as_deref().unwrap_or(log_filename).to_string(); + let active_filename = file_pattern.clone(); // Determine match mode from config, defaulting to Suffix let match_mode = match config.log_match_mode.as_deref().map(|s| s.to_lowercase()).as_deref() { @@ -386,21 +380,21 @@ fn spawn_cleanup_task( .log_cleanup_interval_seconds .unwrap_or(DEFAULT_OBS_LOG_CLEANUP_INTERVAL_SECONDS); - let cleaner = Arc::new(LogCleaner::new( - log_dir, - file_pattern, - match_mode, - keep_files, - max_total_size, - max_single_file_size, - compress, - gzip_level, - retention_days, - exclude_patterns, - delete_empty, - min_age, - dry_run, - )); + let cleaner = Arc::new( + LogCleaner::builder(log_dir, file_pattern, active_filename) + .match_mode(match_mode) + .keep_files(keep_files) + .max_total_size_bytes(max_total_size) + .max_single_file_size_bytes(max_single_file_size) + .compress_old_files(compress) + .gzip_compression_level(gzip_level) + .compressed_file_retention_days(retention_days) + .exclude_patterns(exclude_patterns) + .delete_empty_files(delete_empty) + .min_file_age_seconds(min_age) + .dry_run(dry_run) + .build(), + ); tokio::spawn(async move { let mut interval = tokio::time::interval(Duration::from_secs(cleanup_interval)); @@ -433,8 +427,13 @@ mod tests { ..OtelConfig::default() }; - let result = init_file_logging_internal(&config, temp_path, "info", true); - - assert!(result.is_err()); + // We must run within a Tokio runtime because init_file_logging_internal spawns a background task. + let rt = tokio::runtime::Runtime::new().unwrap(); + rt.block_on(async { + let result = init_file_logging_internal(&config, temp_path, "info", true); + // With eager file opening, an invalid filename (null byte) causes the OS to reject + // the open() call, so the function returns Err instead of panicking. + assert!(result.is_err(), "invalid filename must return Err, not panic"); + }); } } diff --git a/crates/obs/src/telemetry/mod.rs b/crates/obs/src/telemetry/mod.rs index 87ef6fe4f..7ac8aff05 100644 --- a/crates/obs/src/telemetry/mod.rs +++ b/crates/obs/src/telemetry/mod.rs @@ -45,6 +45,7 @@ mod local; mod otel; mod recorder; mod resource; +mod rolling; use crate::TelemetryError; use crate::config::OtelConfig; @@ -117,8 +118,9 @@ pub(crate) fn init_telemetry(config: &OtelConfig) -> Result bool { + match self { + Rotation::Minutely => now / 60 != last / 60, + Rotation::Hourly => now / 3600 != last / 3600, + Rotation::Daily => { + // Align daily rotation with the local day boundary rather than UTC midnight. + // We shift both timestamps by the current local offset before bucketing into days. + let offset_secs = Zoned::now().offset().seconds() as i64; + (now + offset_secs) / 86400 != (last + offset_secs) / 86400 + } + Rotation::Never => false, + } + } +} + +pub struct RollingAppender { + dir: PathBuf, + filename: String, + rotation: Rotation, + max_size_bytes: u64, + match_mode: FileMatchMode, + + file: Option, + size: u64, + // Store as seconds since Unix epoch + last_roll_ts: i64, +} + +impl RollingAppender { + /// Create and immediately validate a new `RollingAppender`. + /// + /// The log directory is created if it does not already exist, and the + /// active log file is opened (or created) eagerly so that configuration + /// errors — e.g. an invalid filename — surface at initialisation time + /// rather than on the first write. + /// + /// # Errors + /// Returns an [`io::Error`] if: + /// - `filename` is not a plain file name (absolute path, path separators, + /// or `..` components are rejected to prevent path traversal). + /// - The directory cannot be created. + /// - The active log file cannot be opened/created. + pub fn new( + dir: impl AsRef, + filename: String, + rotation: Rotation, + max_size_bytes: u64, + match_mode: FileMatchMode, + ) -> io::Result { + // Validate that `filename` is a plain file name: not absolute and no + // directory components (separators or `..`). If `file_name()` equals + // the entire path, there can be no parent-directory traversal. + { + let p = Path::new(&filename); + let is_plain_name = !p.is_absolute() && p.file_name().map(|n| n == p.as_os_str()).unwrap_or(false); + if !is_plain_name { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("log filename must be a plain file name with no path components, got: {filename:?}"), + )); + } + } + + let mut appender = Self { + dir: dir.as_ref().to_path_buf(), + filename, + rotation, + max_size_bytes, + match_mode, + file: None, + size: 0, + last_roll_ts: Zoned::now().timestamp().as_second(), + }; + // Eagerly open the file to validate the path and capture accurate + // initial size / last-roll timestamp. + appender.open_file()?; + Ok(appender) + } + + fn active_file_path(&self) -> PathBuf { + self.dir.join(&self.filename) + } + + fn open_file(&mut self) -> io::Result<()> { + if self.file.is_some() { + return Ok(()); + } + + let path = self.active_file_path(); + // Ensure directory exists + if let Some(parent) = path.parent() { + fs::create_dir_all(parent)?; + } + + // Open in append mode + let file = fs::OpenOptions::new().create(true).append(true).open(&path)?; + + let meta = file.metadata()?; + self.size = meta.len(); + + // Seed `last_roll_ts` from the file's modification time so that a + // process restart correctly triggers time-based rotation if the active + // file belongs to a previous period. + if let Ok(modified) = meta.modified() { + // Convert SystemTime to jiff::Timestamp + if let Ok(ts) = jiff::Timestamp::try_from(modified) { + self.last_roll_ts = ts.as_second(); + } + } + + self.file = Some(file); + Ok(()) + } + + fn should_roll(&self, write_len: u64) -> bool { + // 1. Size-based check (Cheap, check first) + // If max_size is set (non-zero) and writing would exceed it, roll immediately. + if self.max_size_bytes > 0 && (self.size + write_len) > self.max_size_bytes { + return true; + } + + // 2. Time-based check + // We check this after size check to avoid unnecessary time calls if size forces a roll. + let now = Zoned::now().timestamp().as_second(); + + self.rotation.check_should_roll(self.last_roll_ts, now) + } + + fn roll(&mut self) -> io::Result<()> { + // 1. Close current file first to ensure all buffers are flushed to OS (if any) + // and handle released. + self.file = None; + + let active_path = self.active_file_path(); + if !active_path.exists() { + return Ok(()); + } + + // 2. Generate archive name. + // Format: YYYYMMDDHHMMSS.uuuuuu (Microsecond/Nanosecond precision) + // We use jiff's strftime. "%Y%m%d%H%M%S%.6f" gives microsecond precision. + let now = Zoned::now(); + let timestamp_str = now.strftime("%Y%m%d%H%M%S%.6f").to_string(); + + // Add a unique counter to prevent collisions in high-concurrency/fast-rotation scenarios. + let counter = ROLL_UNIQUIFIER.fetch_add(1, Ordering::Relaxed); + + // Final suffix/prefix part: timestamp + counter + // Example: 20231027103001.123456-0 + let unique_part = format!("{}-{}", timestamp_str, counter); + + // Match naming strategy with LogCleaner expectations. + let archive_name = match self.match_mode { + FileMatchMode::Suffix => { + // Suffix mode: timestamp BEFORE filename. + // e.g. rustfs.log -> 20231027103001.123456-0.rustfs.log + format!("{}.{}", unique_part, self.filename) + } + FileMatchMode::Prefix => { + // Prefix mode: timestamp AFTER filename. + // e.g. rustfs -> rustfs.20231027103001.123456-0 + format!("{}.{}", self.filename, unique_part) + } + }; + + // 3. Rename the active file to the archive path. + let archive_path = self.dir.join(&archive_name); + + // Robust Rename Strategy: + // On Windows, file locking (e.g. by AV software or indexers) can cause `rename` to fail + // spuriously with PermissionDenied. We implement a short retry loop with backoff. + const MAX_RETRIES: u32 = 3; + let mut last_error = None; + + for i in 0..MAX_RETRIES { + match fs::rename(&active_path, &archive_path) { + Ok(_) => { + // Success! + // 4. Reset state + self.size = 0; + self.last_roll_ts = now.timestamp().as_second(); + + // 5. Re-open (creates new active file) + self.open_file()?; + return Ok(()); + } + Err(e) => { + // Decide if we should retry based on error kind + let should_retry = match e.kind() { + // Windows often returns PermissionDenied for locked files + io::ErrorKind::PermissionDenied => true, + io::ErrorKind::Interrupted => true, + _ => false, + }; + + last_error = Some(e); + + if !should_retry { + break; + } + + // Exponential backoff: 10ms, 20ms, 40ms... + thread::sleep(Duration::from_millis(10 * (1 << i))); + } + } + } + + // 6. Recovery Failure + // If we exhausted retries, we MUST NOT lose log data. + // We re-open the ACTIVE file (which is still there because rename failed). + // The file will grow beyond max_size, but availability > strict sizing. + eprintln!( + "RollingAppender: Failed to rotate log file after {} retries. Error: {:?}", + MAX_RETRIES, last_error + ); + + // Attempt to re-open existing active file to allow continued writing + self.open_file()?; + + // Return the error so it can be logged/handled, even though we recovered the handle. + Err(last_error.unwrap_or_else(|| io::Error::other("Unknown rename error"))) + } +} + +impl Write for RollingAppender { + fn write(&mut self, buf: &[u8]) -> io::Result { + // Ensure file is open + if self.file.is_none() { + self.open_file()?; + } + + // Check rotation + if self.should_roll(buf.len() as u64) + && let Err(e) = self.roll() + { + // If rotation fails, we log to stderr and try to continue writing to the active file + // to avoid losing logs if possible. + eprintln!("RollingAppender: failed to rotate log file: {}", e); + } + + // Ensure file is open (in case roll closed it and failed to open new one, or open_file failed above) + if self.file.is_none() { + self.open_file()?; + } + + if let Some(file) = &mut self.file { + let n = file.write(buf)?; + self.size += n as u64; + Ok(n) + } else { + Err(io::Error::other("Failed to open log file")) + } + } + + fn flush(&mut self) -> io::Result<()> { + if let Some(file) = &mut self.file { + file.flush() + } else { + Ok(()) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::fs; + use tempfile::TempDir; + + fn count_files(dir: &Path) -> usize { + fs::read_dir(dir) + .unwrap() + .filter(|e| e.as_ref().unwrap().path().is_file()) + .count() + } + + // ── Construction ────────────────────────────────────────────────────────── + + #[test] + fn test_new_creates_file_eagerly() { + let tmp = TempDir::new().unwrap(); + let _appender = RollingAppender::new(tmp.path(), "test.log".to_string(), Rotation::Daily, 0, FileMatchMode::Suffix) + .expect("should create appender without error"); + assert!(tmp.path().join("test.log").exists(), "active log file should be created on new()"); + } + + #[test] + fn test_new_invalid_filename_returns_error() { + let tmp = TempDir::new().unwrap(); + // Null byte is invalid on both Unix and Windows. + let result = RollingAppender::new(tmp.path(), "invalid\0name.log".to_string(), Rotation::Daily, 0, FileMatchMode::Suffix); + assert!(result.is_err(), "null byte in filename must produce an error"); + } + + #[test] + fn test_new_rejects_path_with_separators() { + let tmp = TempDir::new().unwrap(); + // A filename containing path separators could escape the log directory. + let result = RollingAppender::new(tmp.path(), "subdir/app.log".to_string(), Rotation::Daily, 0, FileMatchMode::Suffix); + assert!(result.is_err(), "filename with path separator must be rejected"); + } + + #[test] + fn test_new_rejects_parent_directory_traversal() { + let tmp = TempDir::new().unwrap(); + // "../secret.log" would write outside the log directory. + let result = RollingAppender::new(tmp.path(), "../secret.log".to_string(), Rotation::Daily, 0, FileMatchMode::Suffix); + assert!(result.is_err(), "parent-directory traversal in filename must be rejected"); + } + + #[test] + fn test_new_rejects_absolute_path_as_filename() { + let tmp = TempDir::new().unwrap(); + let result = RollingAppender::new(tmp.path(), "/etc/app.log".to_string(), Rotation::Daily, 0, FileMatchMode::Suffix); + assert!(result.is_err(), "absolute path as filename must be rejected"); + } + + /// On Windows, backslash is a path separator and must be rejected. + #[cfg(windows)] + #[test] + fn test_new_rejects_backslash_path_separator_on_windows() { + let tmp = TempDir::new().unwrap(); + let result = RollingAppender::new(tmp.path(), "subdir\\app.log".to_string(), Rotation::Daily, 0, FileMatchMode::Suffix); + assert!(result.is_err(), "backslash path separator in filename must be rejected on Windows"); + } + + // ── Basic writes ────────────────────────────────────────────────────────── + + #[test] + fn test_write_stores_content() { + let tmp = TempDir::new().unwrap(); + let mut appender = + RollingAppender::new(tmp.path(), "test.log".to_string(), Rotation::Daily, 0, FileMatchMode::Suffix).unwrap(); + + appender.write_all(b"hello world\n").expect("write should succeed"); + appender.flush().expect("flush should succeed"); + + let content = fs::read_to_string(tmp.path().join("test.log")).unwrap(); + assert_eq!(content, "hello world\n"); + } + + // ── Size-based rotation ──────────────────────────────────────────────────── + + #[test] + fn test_size_rotation_creates_archive() { + let tmp = TempDir::new().unwrap(); + // Allow only 5 bytes before rotating. + let mut appender = + RollingAppender::new(tmp.path(), "app.log".to_string(), Rotation::Never, 5, FileMatchMode::Suffix).unwrap(); + + // First write: 5 bytes exactly — no rotation yet. + appender.write_all(b"12345").expect("write should succeed"); + // Second write: would push past the limit — rotation should occur first. + appender.write_all(b"abcde").expect("write after rotation should succeed"); + appender.flush().unwrap(); + + // There should now be 2 files: the active log + 1 archive. + assert_eq!(count_files(tmp.path()), 2, "one rotation should have produced one archive"); + + // The active file should only contain the second write. + let content = fs::read_to_string(tmp.path().join("app.log")).unwrap(); + assert_eq!(content, "abcde"); + } + + #[test] + fn test_multiple_size_rotations_produce_unique_archives() { + let tmp = TempDir::new().unwrap(); + // Force a rotation on every write of 4+ bytes. + let mut appender = + RollingAppender::new(tmp.path(), "app.log".to_string(), Rotation::Never, 3, FileMatchMode::Suffix).unwrap(); + + for _ in 0..5 { + appender.write_all(b"abcd").expect("write should succeed"); + } + appender.flush().unwrap(); + + let file_count = count_files(tmp.path()); + // At least 5 archives (one per rotation) plus the active file. + assert!( + file_count >= 5, + "each burst write should produce a distinct archive; got {file_count} files" + ); + } + + // ── Archive filename format ──────────────────────────────────────────────── + + #[test] + fn test_suffix_mode_archive_name() { + let tmp = TempDir::new().unwrap(); + let mut appender = + RollingAppender::new(tmp.path(), "app.log".to_string(), Rotation::Never, 3, FileMatchMode::Suffix).unwrap(); + + appender.write_all(b"1234").expect("write should succeed"); + appender.flush().unwrap(); + + let archives: Vec<_> = fs::read_dir(tmp.path()) + .unwrap() + .filter_map(|e| e.ok()) + .map(|e| e.file_name().to_string_lossy().to_string()) + .filter(|n| n != "app.log") + .collect(); + + assert_eq!(archives.len(), 1); + // Suffix mode: "-.app.log" + // Since timestamp contains digits and we use high precision, checking strictly is hard, + // but it should definitely NOT be the old unix timestamp format (just digits). + // It should contain "-" before "app.log" due to our new format. + assert!( + archives[0].ends_with(".app.log"), + "archive should end with '.app.log' in Suffix mode; got '{}'", + archives[0] + ); + // Check for new format chars (YMD) + // 20xx... + assert!( + archives[0].starts_with("20"), + "archive should start with year (20xx); got '{}'", + archives[0] + ); + } + + #[test] + fn test_prefix_mode_archive_name() { + let tmp = TempDir::new().unwrap(); + let mut appender = + RollingAppender::new(tmp.path(), "app".to_string(), Rotation::Never, 3, FileMatchMode::Prefix).unwrap(); + + appender.write_all(b"1234").expect("write should succeed"); + appender.flush().unwrap(); + + let archives: Vec<_> = fs::read_dir(tmp.path()) + .unwrap() + .filter_map(|e| e.ok()) + .map(|e| e.file_name().to_string_lossy().to_string()) + .filter(|n| n != "app") + .collect(); + + assert_eq!(archives.len(), 1); + // Prefix mode: "app.-" + assert!( + archives[0].starts_with("app.20"), + "archive should start with 'app.20' in Prefix mode; got '{}'", + archives[0] + ); + } + + // ── Restart with existing file ───────────────────────────────────────────── + + #[test] + fn test_restart_with_existing_file_reads_size() { + let tmp = TempDir::new().unwrap(); + let log_path = tmp.path().join("app.log"); + fs::write(&log_path, b"existing content").unwrap(); + + let appender = + RollingAppender::new(tmp.path(), "app.log".to_string(), Rotation::Daily, 0, FileMatchMode::Suffix).unwrap(); + + assert_eq!( + appender.size, + b"existing content".len() as u64, + "size should reflect existing file content" + ); + } +}