diff --git a/Cargo.lock b/Cargo.lock index e19cd12fa..3c0f8fee6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -68,9 +68,9 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "1.1.3" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" dependencies = [ "memchr", ] @@ -176,6 +176,15 @@ version = "1.0.100" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" +[[package]] +name = "ar_archive_writer" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0c269894b6fe5e9d7ada0cf69b5bf847ff35bc25fc271f08e1d080fce80339a" +dependencies = [ + "object 0.32.2", +] + [[package]] name = "arbitrary" version = "1.4.2" @@ -655,9 +664,9 @@ dependencies = [ [[package]] name = "aws-sdk-s3" -version = "1.108.0" +version = "1.109.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "200be4aed61e3c0669f7268bacb768f283f1c32a7014ce57225e1160be2f6ccb" +checksum = "3c6d81b75f8ff78882e70c5909804b44553d56136899fb4015a0a68ecc870e0e" dependencies = [ "aws-credential-types", "aws-runtime", @@ -711,9 +720,9 @@ dependencies = [ [[package]] name = "aws-sdk-ssooidc" -version = "1.88.0" +version = "1.89.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a68d675582afea0e94d38b6ca9c5aaae4ca14f1d36faa6edb19b42e687e70d7" +checksum = "695dc67bb861ccb8426c9129b91c30e266a0e3d85650cafdf62fcca14c8fd338" dependencies = [ "aws-credential-types", "aws-runtime", @@ -1099,7 +1108,7 @@ dependencies = [ "cfg-if", "libc", "miniz_oxide", - "object", + "object 0.37.3", "rustc-demangle", "windows-link 0.2.1", ] @@ -1424,9 +1433,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.2.41" +version = "1.2.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac9fe6cdbb24b6ade63616c0a0688e45bb56732262c158df3c0c4bea4ca47cb7" +checksum = "739eb0f94557554b3ca9a86d2d37bebd49c5e6d0c1d2bda35ba5bdac830befc2" dependencies = [ "find-msvc-tools", "jobserver", @@ -2117,9 +2126,9 @@ dependencies = [ [[package]] name = "datafusion" -version = "50.2.0" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc6759cf9ef57c5c469e4027ac4b4cfa746e06a0f5472c2b922b6a403c2a64c4" +checksum = "2af15bb3c6ffa33011ef579f6b0bcbe7c26584688bd6c994f548e44df67f011a" dependencies = [ "arrow", "arrow-ipc", @@ -2172,9 +2181,9 @@ dependencies = [ [[package]] name = "datafusion-catalog" -version = "50.2.0" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a1c48fc7e6d62590d45f7be7c531980b8ff091d1ab113a9ddf465bef41e4093" +checksum = "187622262ad8f7d16d3be9202b4c1e0116f1c9aa387e5074245538b755261621" dependencies = [ "arrow", "async-trait", @@ -2198,9 +2207,9 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" -version = "50.2.0" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3db1266da115de3ab0b2669fc027d96cf0ff777deb3216d52c74b528446ccdd6" +checksum = "9657314f0a32efd0382b9a46fdeb2d233273ece64baa68a7c45f5a192daf0f83" dependencies = [ "arrow", "async-trait", @@ -2221,9 +2230,9 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "50.2.0" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad4eb2a48ca10fa1e1a487a28a5bf080e31efac2d4bf12bb7e92c2d9ea4f35e5" +checksum = "5a83760d9a13122d025fbdb1d5d5aaf93dd9ada5e90ea229add92aa30898b2d1" dependencies = [ "ahash", "arrow", @@ -2246,9 +2255,9 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" -version = "50.2.0" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0422ee64d5791599c46b786063e695f7699fadd3a12ad25038cb3094d05886a" +checksum = "5b6234a6c7173fe5db1c6c35c01a12b2aa0f803a3007feee53483218817f8b1e" dependencies = [ "futures", "log", @@ -2257,9 +2266,9 @@ dependencies = [ [[package]] name = "datafusion-datasource" -version = "50.2.0" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "904c2e1089b3ccf10786f2dae12bc560fda278e4194a8917c5844d2e8c212818" +checksum = "7256c9cb27a78709dd42d0c80f0178494637209cac6e29d5c93edd09b6721b86" dependencies = [ "arrow", "async-compression", @@ -2294,9 +2303,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" -version = "50.2.0" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8336a805c42ef4e359daaad142ddc53649f23c7e934c117d8516816afe6b7a3d" +checksum = "64533a90f78e1684bfb113d200b540f18f268134622d7c96bbebc91354d04825" dependencies = [ "arrow", "async-trait", @@ -2319,9 +2328,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" -version = "50.2.0" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c691b1565e245ea369bc8418b472a75ea84c2ad2deb61b1521cfa38319a9cd47" +checksum = "8d7ebeb12c77df0aacad26f21b0d033aeede423a64b2b352f53048a75bf1d6e6" dependencies = [ "arrow", "async-trait", @@ -2344,9 +2353,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-parquet" -version = "50.2.0" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9f7576ceb5974c5f6874d7f2a5ebfeb58960a920da64017def849e0352fe2d8" +checksum = "09e783c4c7d7faa1199af2df4761c68530634521b176a8d1331ddbc5a5c75133" dependencies = [ "arrow", "async-trait", @@ -2377,15 +2386,15 @@ dependencies = [ [[package]] name = "datafusion-doc" -version = "50.2.0" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dde7c10244f3657fc01eef8247c0b2b20eae4cf6439a0ebb27322f32026d6b8" +checksum = "99ee6b1d9a80d13f9deb2291f45c07044b8e62fb540dbde2453a18be17a36429" [[package]] name = "datafusion-execution" -version = "50.2.0" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5143fc795cef959b6d5271b2e8f1120382fe929fc4bd027c7d7b993f5352ef7e" +checksum = "a4cec0a57653bec7b933fb248d3ffa3fa3ab3bd33bd140dc917f714ac036f531" dependencies = [ "arrow", "async-trait", @@ -2403,9 +2412,9 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "50.2.0" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63e826296bc5f5d0af3e39c1af473d4091ac6a152a5be2f80c256f0182938428" +checksum = "ef76910bdca909722586389156d0aa4da4020e1631994d50fadd8ad4b1aa05fe" dependencies = [ "arrow", "async-trait", @@ -2425,9 +2434,9 @@ dependencies = [ [[package]] name = "datafusion-expr-common" -version = "50.2.0" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9096732d0d8862d1950ca70324fe91f9dee3799eeb0db53ef452bdb573484db6" +checksum = "6d155ccbda29591ca71a1344dd6bed26c65a4438072b400df9db59447f590bb6" dependencies = [ "arrow", "datafusion-common", @@ -2438,9 +2447,9 @@ dependencies = [ [[package]] name = "datafusion-functions" -version = "50.2.0" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f362c78ac283e64fd3976e060c1a8a57d5f4dcf844a6b6bd2eb320640a1572e" +checksum = "7de2782136bd6014670fd84fe3b0ca3b3e4106c96403c3ae05c0598577139977" dependencies = [ "arrow", "arrow-buffer", @@ -2467,9 +2476,9 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" -version = "50.2.0" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22e2a80a80145a796ae3f02eb724ac516178556aec864fe89f6ab3741a4cd249" +checksum = "07331fc13603a9da97b74fd8a273f4238222943dffdbbed1c4c6f862a30105bf" dependencies = [ "ahash", "arrow", @@ -2488,9 +2497,9 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" -version = "50.2.0" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7dcca2fe7c33409e9ab3f950366aa4cba5db6175a09599fdb658ad9f2cc4296" +checksum = "b5951e572a8610b89968a09b5420515a121fbc305c0258651f318dc07c97ab17" dependencies = [ "ahash", "arrow", @@ -2501,9 +2510,9 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" -version = "50.2.0" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1b298733377f3ec8c2868c75b5555b15396d9c13e36c5fda28e80feee34e3ed" +checksum = "fdacca9302c3d8fc03f3e94f338767e786a88a33f5ebad6ffc0e7b50364b9ea3" dependencies = [ "arrow", "arrow-ord", @@ -2523,9 +2532,9 @@ dependencies = [ [[package]] name = "datafusion-functions-table" -version = "50.2.0" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fa4a380ca362eb0fbd33093e8ca6b7a31057616c7e6ee999b87a4ad3c7c0b3f" +checksum = "8c37ff8a99434fbbad604a7e0669717c58c7c4f14c472d45067c4b016621d981" dependencies = [ "arrow", "async-trait", @@ -2539,9 +2548,9 @@ dependencies = [ [[package]] name = "datafusion-functions-window" -version = "50.2.0" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9068fc85b8e187c706427794d79bb7ee91132b6b192cb7b18e650a5f7c5c1340" +checksum = "48e2aea7c79c926cffabb13dc27309d4eaeb130f4a21c8ba91cdd241c813652b" dependencies = [ "arrow", "datafusion-common", @@ -2557,9 +2566,9 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" -version = "50.2.0" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2f80ec56e177d166269556649be817a382a374642872df4ca48cf9be3d09b3a" +checksum = "0fead257ab5fd2ffc3b40fda64da307e20de0040fe43d49197241d9de82a487f" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -2567,9 +2576,9 @@ dependencies = [ [[package]] name = "datafusion-macros" -version = "50.2.0" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4868fe261ba01e462033eff141e90453b7630722cad6420fddd81ebb786f6e2" +checksum = "ec6f637bce95efac05cdfb9b6c19579ed4aa5f6b94d951cfa5bb054b7bb4f730" dependencies = [ "datafusion-expr", "quote", @@ -2578,9 +2587,9 @@ dependencies = [ [[package]] name = "datafusion-optimizer" -version = "50.2.0" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40ed8c51b5c37c057e5c7d5945ed807f1cecfba003bdb1a4c3036595dda287c7" +checksum = "c6583ef666ae000a613a837e69e456681a9faa96347bf3877661e9e89e141d8a" dependencies = [ "arrow", "chrono", @@ -2598,9 +2607,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "50.2.0" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f678f5734147446e1adbee63be4b244c8f0e9cbd5c41525004ace3730190d03e" +checksum = "c8668103361a272cbbe3a61f72eca60c9b7c706e87cc3565bcf21e2b277b84f6" dependencies = [ "ahash", "arrow", @@ -2621,9 +2630,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-adapter" -version = "50.2.0" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "086877d4eca538e9cd1f28b917db0036efe0ad8b4fb7c702f520510672032c8d" +checksum = "815acced725d30601b397e39958e0e55630e0a10d66ef7769c14ae6597298bb0" dependencies = [ "arrow", "datafusion-common", @@ -2636,9 +2645,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" -version = "50.2.0" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5c5d17f6a4f28f9849ee3449bb9b83406a718e4275c218bf37ca247ee123779" +checksum = "6652fe7b5bf87e85ed175f571745305565da2c0b599d98e697bcbedc7baa47c3" dependencies = [ "ahash", "arrow", @@ -2650,9 +2659,9 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" -version = "50.2.0" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab9fb8b3fba2634d444e0177862797dc1231e0e20bc4db291a15d39c0d4136c3" +checksum = "49b7d623eb6162a3332b564a0907ba00895c505d101b99af78345f1acf929b5c" dependencies = [ "arrow", "datafusion-common", @@ -2670,9 +2679,9 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" -version = "50.2.0" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5086cb2e579270173ff0eb38d60ba2a081f1d422a743fa673f6096920950eb5" +checksum = "e2f7f778a1a838dec124efb96eae6144237d546945587557c9e6936b3414558c" dependencies = [ "ahash", "arrow", @@ -2701,9 +2710,9 @@ dependencies = [ [[package]] name = "datafusion-pruning" -version = "50.2.0" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f84b866d906118c320459f30385048aeedbe36ac06973d3e4fa0cc5d60d722c" +checksum = "cd1e59e2ca14fe3c30f141600b10ad8815e2856caa59ebbd0e3e07cd3d127a65" dependencies = [ "arrow", "arrow-schema", @@ -2719,9 +2728,9 @@ dependencies = [ [[package]] name = "datafusion-session" -version = "50.2.0" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3820062b9dd2846954eeb844ff9fe3662977b7d2d74947647c779fabfa502508" +checksum = "21ef8e2745583619bd7a49474e8f45fbe98ebb31a133f27802217125a7b3d58d" dependencies = [ "arrow", "async-trait", @@ -2743,9 +2752,9 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "50.2.0" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "375232baa851b2e9d09fcbe8906141a0ec6e0e058addc5565e0d3d790bb9d51d" +checksum = "89abd9868770386fede29e5a4b14f49c0bf48d652c3b9d7a8a0332329b87d50b" dependencies = [ "arrow", "bigdecimal", @@ -2796,9 +2805,9 @@ dependencies = [ [[package]] name = "deranged" -version = "0.5.4" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a41953f86f8a05768a6cda24def994fd2f424b04ec5c719cf89989779f199071" +checksum = "ececcb659e7ba858fb4f10388c250a7252eb0a27373f1a72b8748afdd248e587" dependencies = [ "powerfmt", "serde_core", @@ -3300,9 +3309,9 @@ dependencies = [ [[package]] name = "flate2" -version = "1.1.4" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc5a4e564e38c699f2880d3fda590bedc2e69f3f84cd48b457bd892ce61d0aa9" +checksum = "bfe33edd8e85a12a67454e37f8c75e730830d83e313556ab9ebf9ee7fbeb3bfb" dependencies = [ "crc32fast", "libz-rs-sys", @@ -3545,9 +3554,9 @@ checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" [[package]] name = "google-cloud-auth" -version = "1.0.1" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5a0f0ef58bc79d636e95db264939a6f3fd80951f77743f2b7ec55e22171150d" +checksum = "f084abe65c3bdde490bf4a4eebc7e103637fb3ac4b101d09508dd5bf12ce82b1" dependencies = [ "async-trait", "base64 0.22.1", @@ -3567,9 +3576,9 @@ dependencies = [ [[package]] name = "google-cloud-gax" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58bc95deae841e35758fa5caba317092f26940135c7184570feb691a1844db08" +checksum = "a36f8701dac18c2cbedf588d15ee4ce5c018da531c2e74e9ad043cbd32b0fccb" dependencies = [ "base64 0.22.1", "bytes", @@ -3587,9 +3596,9 @@ dependencies = [ [[package]] name = "google-cloud-gax-internal" -version = "0.7.2" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7963ef5d9a7e1c2c20138b6c6cbe32dc14ded18d51ba4e8c781c7f5de414dfd1" +checksum = "152076418386463ff650deabbef392672626d62e97639bb5194e11f8984b4835" dependencies = [ "bytes", "google-cloud-auth", @@ -3598,7 +3607,7 @@ dependencies = [ "google-cloud-wkt", "http 1.3.1", "http-body-util", - "opentelemetry-semantic-conventions 0.30.0", + "opentelemetry-semantic-conventions", "percent-encoding", "prost", "prost-types", @@ -3635,9 +3644,9 @@ dependencies = [ [[package]] name = "google-cloud-longrunning" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69debfcc085fc9588e8d90ed27a2ae500f636f6d7d08ee7e8cd62992ca164d68" +checksum = "398201f50d0dd0180105628c370ba5dae77a3f5e842eebce494f451caee96371" dependencies = [ "async-trait", "bytes", @@ -3655,9 +3664,9 @@ dependencies = [ [[package]] name = "google-cloud-lro" -version = "1.0.0" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9d7432f793f49c9d556da45995f39539dcecdc2361de113b05b0f7bbed73072" +checksum = "a93fd80965a47da86e1d21c90dcd10858b859e6c702c5c6e1b383686692ae73e" dependencies = [ "google-cloud-gax", "google-cloud-longrunning", @@ -3682,9 +3691,9 @@ dependencies = [ [[package]] name = "google-cloud-storage" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d326d5fa84cc77e4af8d070f46f040abfd043eb72b4ba312756102da802d2fc" +checksum = "166471b8f807470024f4533d7229bc98696161b5e1f254d1a485b60c6c42baf3" dependencies = [ "async-trait", "base64 0.22.1", @@ -3705,6 +3714,7 @@ dependencies = [ "hyper 1.7.0", "lazy_static", "md5", + "mime", "percent-encoding", "pin-project", "prost", @@ -3963,11 +3973,11 @@ dependencies = [ [[package]] name = "home" -version = "0.5.11" +version = "0.5.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "589533453244b0995c858700322199b2becb13b627df2851f64a2775d024abcf" +checksum = "cc627f471c528ff0c4a49e1d5e60450c8f6461dd6d10ba9dcd3a61d3dff7728d" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -4200,9 +4210,9 @@ dependencies = [ [[package]] name = "icu_collections" -version = "2.0.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "200072f5d0e3614556f94a9930d5dc3e0662a652823904c3a75dc3b0af7fee47" +checksum = "4c6b649701667bbe825c3b7e6388cb521c23d88644678e83c0c4d0a621a34b43" dependencies = [ "displaydoc", "potential_utf", @@ -4213,9 +4223,9 @@ dependencies = [ [[package]] name = "icu_locale_core" -version = "2.0.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cde2700ccaed3872079a65fb1a78f6c0a36c91570f28755dda67bc8f7d9f00a" +checksum = "edba7861004dd3714265b4db54a3c390e880ab658fec5f7db895fae2046b5bb6" dependencies = [ "displaydoc", "litemap", @@ -4226,11 +4236,10 @@ dependencies = [ [[package]] name = "icu_normalizer" -version = "2.0.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "436880e8e18df4d7bbc06d58432329d6458cc84531f7ac5f024e93deadb37979" +checksum = "5f6c8828b67bf8908d82127b2054ea1b4427ff0230ee9141c54251934ab1b599" dependencies = [ - "displaydoc", "icu_collections", "icu_normalizer_data", "icu_properties", @@ -4241,42 +4250,38 @@ dependencies = [ [[package]] name = "icu_normalizer_data" -version = "2.0.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00210d6893afc98edb752b664b8890f0ef174c8adbb8d0be9710fa66fbbf72d3" +checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a" [[package]] name = "icu_properties" -version = "2.0.1" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "016c619c1eeb94efb86809b015c58f479963de65bdb6253345c1a1276f22e32b" +checksum = "e93fcd3157766c0c8da2f8cff6ce651a31f0810eaa1c51ec363ef790bbb5fb99" dependencies = [ - "displaydoc", "icu_collections", "icu_locale_core", "icu_properties_data", "icu_provider", - "potential_utf", "zerotrie", "zerovec", ] [[package]] name = "icu_properties_data" -version = "2.0.1" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "298459143998310acd25ffe6810ed544932242d3f07083eee1084d83a71bd632" +checksum = "02845b3647bb045f1100ecd6480ff52f34c35f82d9880e029d329c21d1054899" [[package]] name = "icu_provider" -version = "2.0.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03c80da27b5f4187909049ee2d72f276f0d9f99a42c306bd0131ecfe04d8e5af" +checksum = "85962cf0ce02e1e0a629cc34e7ca3e373ce20dda4c4d7294bbd0bf1fdb59e614" dependencies = [ "displaydoc", "icu_locale_core", - "stable_deref_trait", - "tinystr", "writeable", "yoke", "zerofrom", @@ -4414,13 +4419,13 @@ dependencies = [ [[package]] name = "is-terminal" -version = "0.4.16" +version = "0.4.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e04d7f318608d35d4b61ddd75cbdaee86b023ebe2bd5a66ee0915f0bf93095a9" +checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -4471,9 +4476,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.81" +version = "0.3.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec48937a97411dcb524a265206ccd4c90bb711fca92b2792c407f268825b9305" +checksum = "b011eec8cc36da2aab2d5cff675ec18454fad408585853910a202391cf9f8e65" dependencies = [ "once_cell", "wasm-bindgen", @@ -4678,9 +4683,9 @@ checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" [[package]] name = "litemap" -version = "0.8.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "241eaef5fd12c88705a01fc1066c48c4b36e0dd4377dcdc7ec3942cea7a69956" +checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77" [[package]] name = "lmdb-master-sys" @@ -5263,6 +5268,15 @@ dependencies = [ "objc2-core-foundation", ] +[[package]] +name = "object" +version = "0.32.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" +dependencies = [ + "memchr", +] + [[package]] name = "object" version = "0.37.3" @@ -5411,12 +5425,6 @@ dependencies = [ "tonic-prost", ] -[[package]] -name = "opentelemetry-semantic-conventions" -version = "0.30.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83d059a296a47436748557a353c5e6c5705b9470ef6c95cfc52c21a8814ddac2" - [[package]] name = "opentelemetry-semantic-conventions" version = "0.31.0" @@ -5882,9 +5890,9 @@ checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483" [[package]] name = "potential_utf" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84df19adbe5b5a0782edcab45899906947ab039ccf4573713735ee7de1e6b08a" +checksum = "b73949432f5e2a09657003c25bca5e19a0e9c84f8058ca374f49e0ebe605af77" dependencies = [ "zerovec", ] @@ -5965,9 +5973,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.102" +version = "1.0.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e0f6df8eaa422d97d72edcd152e1451618fed47fabbdbd5a8864167b1d4aff7" +checksum = "5ee95bc4ef87b8d5ba32e8b7714ccc834865276eab0aed5c9958d00ec45f49e8" dependencies = [ "unicode-ident", ] @@ -6079,10 +6087,11 @@ dependencies = [ [[package]] name = "psm" -version = "0.1.27" +version = "0.1.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e66fcd288453b748497d8fb18bccc83a16b0518e3906d4b8df0a8d42d93dbb1c" +checksum = "d11f2fedc3b7dafdc2851bc52f277377c5473d378859be234bc7ebb593144d01" dependencies = [ + "ar_archive_writer", "cc", ] @@ -6708,6 +6717,7 @@ dependencies = [ "shadow-rs", "socket2 0.6.1", "sysctl", + "sysinfo", "thiserror 2.0.17", "tikv-jemallocator", "time", @@ -6864,7 +6874,6 @@ dependencies = [ "glob", "google-cloud-auth", "google-cloud-storage", - "heed", "hex-simd", "hmac 0.12.1", "http 1.3.1", @@ -7092,7 +7101,7 @@ dependencies = [ "opentelemetry 0.31.0", "opentelemetry-appender-tracing", "opentelemetry-otlp", - "opentelemetry-semantic-conventions 0.31.0", + "opentelemetry-semantic-conventions", "opentelemetry-stdout", "opentelemetry_sdk 0.31.0", "rustfs-config", @@ -7398,7 +7407,7 @@ dependencies = [ "once_cell", "ring", "rustls-pki-types", - "rustls-webpki 0.103.7", + "rustls-webpki 0.103.8", "subtle", "zeroize", ] @@ -7447,9 +7456,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.12.0" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "229a4a4c221013e7e1f1a043678c5cc39fe5171437c88fb47151a21e6f5b5c79" +checksum = "94182ad936a0c91c324cd46c6511b9510ed16af436d7b5bab34beab0afd55f7a" dependencies = [ "web-time", "zeroize", @@ -7478,9 +7487,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.7" +version = "0.103.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e10b3f4191e8a80e6b43eebabfac91e5dcecebb27a71f04e820c47ec41d314bf" +checksum = "2ffdfa2f5286e2247234e03f680868ac2815974dc39e00ea15adc445d0aafe52" dependencies = [ "aws-lc-rs", "ring", @@ -8468,6 +8477,7 @@ dependencies = [ "ntapi", "objc2-core-foundation", "objc2-io-kit", + "rayon", "windows", ] @@ -8677,9 +8687,9 @@ dependencies = [ [[package]] name = "tinystr" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d4f6d1145dcb577acf783d4e601bc1d76a13337bb54e6233add580b07344c8b" +checksum = "42d3e9c45c09de15d06dd8acf5f4e0e399e85927b7f00711024eb7ae10fa4869" dependencies = [ "displaydoc", "zerovec", @@ -9338,9 +9348,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.104" +version = "0.2.105" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1da10c01ae9f1ae40cbfac0bac3b1e724b320abfcf52229f80b547c0d250e2d" +checksum = "da95793dfc411fbbd93f5be7715b0578ec61fe87cb1a42b12eb625caa5c5ea60" dependencies = [ "cfg-if", "once_cell", @@ -9349,25 +9359,11 @@ dependencies = [ "wasm-bindgen-shared", ] -[[package]] -name = "wasm-bindgen-backend" -version = "0.2.104" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "671c9a5a66f49d8a47345ab942e2cb93c7d1d0339065d4f8139c486121b43b19" -dependencies = [ - "bumpalo", - "log", - "proc-macro2", - "quote", - "syn 2.0.108", - "wasm-bindgen-shared", -] - [[package]] name = "wasm-bindgen-futures" -version = "0.4.54" +version = "0.4.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e038d41e478cc73bae0ff9b36c60cff1c98b8f38f8d7e8061e79ee63608ac5c" +checksum = "551f88106c6d5e7ccc7cd9a16f312dd3b5d36ea8b4954304657d5dfba115d4a0" dependencies = [ "cfg-if", "js-sys", @@ -9378,9 +9374,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.104" +version = "0.2.105" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ca60477e4c59f5f2986c50191cd972e3a50d8a95603bc9434501cf156a9a119" +checksum = "04264334509e04a7bf8690f2384ef5265f05143a4bff3889ab7a3269adab59c2" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -9388,22 +9384,22 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.104" +version = "0.2.105" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f07d2f20d4da7b26400c9f4a0511e6e0345b040694e8a75bd41d578fa4421d7" +checksum = "420bc339d9f322e562942d52e115d57e950d12d88983a14c79b86859ee6c7ebc" dependencies = [ + "bumpalo", "proc-macro2", "quote", "syn 2.0.108", - "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.104" +version = "0.2.105" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bad67dc8b2a1a6e5448428adec4c3e84c43e561d8c9ee8a9e5aabeb193ec41d1" +checksum = "76f218a38c84bcb33c25ec7059b07847d465ce0e0a76b995e134a45adcb6af76" dependencies = [ "unicode-ident", ] @@ -9423,9 +9419,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.81" +version = "0.3.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9367c417a924a74cae129e6a2ae3b47fabb1f8995595ab474029da749a8be120" +checksum = "3a1f95c0d03a47f4ae1f7a64643a6bb97465d9b740f0fa8f90ea33915c99a9a1" dependencies = [ "js-sys", "wasm-bindgen", @@ -9855,9 +9851,9 @@ dependencies = [ [[package]] name = "writeable" -version = "0.6.1" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea2f10b9bb0928dfb1b42b65e1f9e36f7f54dbdf08457afefb38afcdec4fa2bb" +checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" [[package]] name = "xattr" @@ -9898,11 +9894,10 @@ checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" [[package]] name = "yoke" -version = "0.8.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f41bb01b8226ef4bfd589436a297c53d118f65921786300e427be8d487695cc" +checksum = "72d6e5c6afb84d73944e5cedb052c4680d5657337201555f9f2a16b7406d4954" dependencies = [ - "serde", "stable_deref_trait", "yoke-derive", "zerofrom", @@ -9910,9 +9905,9 @@ dependencies = [ [[package]] name = "yoke-derive" -version = "0.8.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38da3c9736e16c5d3c8c597a9aaa5d1fa565d0532ae05e27c24aa62fb32c0ab6" +checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" dependencies = [ "proc-macro2", "quote", @@ -9983,9 +9978,9 @@ dependencies = [ [[package]] name = "zerotrie" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36f0bbd478583f79edad978b407914f61b2972f5af6fa089686016be8f9af595" +checksum = "2a59c17a5562d507e4b54960e8569ebee33bee890c70aa3fe7b97e85a9fd7851" dependencies = [ "displaydoc", "yoke", @@ -9994,9 +9989,9 @@ dependencies = [ [[package]] name = "zerovec" -version = "0.11.4" +version = "0.11.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7aa2bd55086f1ab526693ecbe444205da57e25f4489879da80635a46d90e73b" +checksum = "6c28719294829477f525be0186d13efa9a3c602f7ec202ca9e353d310fb9a002" dependencies = [ "yoke", "zerofrom", @@ -10005,9 +10000,9 @@ dependencies = [ [[package]] name = "zerovec-derive" -version = "0.11.1" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b96237efa0c878c64bd89c436f661be4e46b2f3eff1ebb976f7ef2321d2f58f" +checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index bbe8dc0a2..e11c7c09b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -103,7 +103,7 @@ axum-server = { version = "0.7.2", features = ["tls-rustls-no-provider"], defaul futures = "0.3.31" futures-core = "0.3.31" futures-util = "0.3.31" -hyper = "1.7.0" +hyper = { version = "1.7.0", features = ["http2", "http1", "server"] } hyper-rustls = { version = "0.27.7", default-features = false, features = ["native-tokio", "http1", "tls12", "logging", "http2", "ring", "webpki-roots"] } hyper-util = { version = "0.1.17", features = ["tokio", "server-auto", "server-graceful"] } http = "1.3.1" @@ -152,7 +152,7 @@ pbkdf2 = "0.12.2" rsa = { version = "0.9.8" } rustls = { version = "0.23.34", features = ["ring", "logging", "std", "tls12"], default-features = false } rustls-pemfile = "2.2.0" -rustls-pki-types = "1.12.0" +rustls-pki-types = "1.13.0" sha1 = "0.10.6" sha2 = "0.10.9" zeroize = { version = "1.8.2", features = ["derive"] } @@ -170,7 +170,7 @@ atoi = "2.0.0" atomic_enum = "0.3.0" aws-config = { version = "1.8.8" } aws-credential-types = { version = "1.2.8" } -aws-sdk-s3 = { version = "1.108.0", default-features = false, features = ["sigv4a", "rustls", "rt-tokio"] } +aws-sdk-s3 = { version = "1.109.0", default-features = false, features = ["sigv4a", "rustls", "rt-tokio"] } aws-smithy-types = { version = "1.3.3" } base64 = "0.22.1" base64-simd = "0.8.0" @@ -181,12 +181,14 @@ const-str = { version = "0.7.0", features = ["std", "proc"] } convert_case = "0.8.0" criterion = { version = "0.7", features = ["html_reports"] } crossbeam-queue = "0.3.12" -datafusion = "50.2.0" +datafusion = "50.3.0" derive_builder = "0.20.2" enumset = "1.1.10" -flate2 = "1.1.4" +flate2 = "1.1.5" flexi_logger = { version = "0.31.7", features = ["trc", "dont_minimize_extra_stacks", "compress", "kv"] } glob = "0.3.3" +google-cloud-storage = "1.2.0" +google-cloud-auth = "1.1.0" hashbrown = { version = "0.16.0", features = ["serde", "rayon"] } hex-simd = "0.8.0" highway = { version = "1.3.0" } @@ -233,7 +235,7 @@ snap = "1.1.1" starshard = { version = "0.5.0", features = ["rayon", "async", "serde"] } strum = { version = "0.27.2", features = ["derive"] } sysctl = "0.7.1" -sysinfo = "0.37.1" +sysinfo = "0.37.2" temp-env = "0.3.6" tempfile = "3.23.0" test-case = "3.3.1" diff --git a/crates/config/src/constants/app.rs b/crates/config/src/constants/app.rs index 30b128819..255bfc388 100644 --- a/crates/config/src/constants/app.rs +++ b/crates/config/src/constants/app.rs @@ -21,12 +21,12 @@ pub const APP_NAME: &str = "RustFS"; /// Application version /// Default value: 1.0.0 /// Environment variable: RUSTFS_VERSION -pub const VERSION: &str = "0.0.1"; +pub const VERSION: &str = "1.0.0"; /// Default configuration logger level -/// Default value: info +/// Default value: error /// Environment variable: RUSTFS_LOG_LEVEL -pub const DEFAULT_LOG_LEVEL: &str = "info"; +pub const DEFAULT_LOG_LEVEL: &str = "error"; /// Default configuration use stdout /// Default value: false @@ -40,22 +40,15 @@ pub const SAMPLE_RATIO: f64 = 1.0; pub const METER_INTERVAL: u64 = 30; /// Default configuration service version -/// Default value: 0.0.1 -pub const SERVICE_VERSION: &str = "0.0.1"; +/// Default value: 1.0.0 +/// Environment variable: RUSTFS_OBS_SERVICE_VERSION +/// Uses the same value as VERSION constant +pub const SERVICE_VERSION: &str = "1.0.0"; /// Default configuration environment /// Default value: production pub const ENVIRONMENT: &str = "production"; -/// maximum number of connections -/// This is the maximum number of connections that the server will accept. -/// This is used to limit the number of connections to the server. -pub const MAX_CONNECTIONS: usize = 100; -/// timeout for connections -/// This is the timeout for connections to the server. -/// This is used to limit the time that a connection can be open. -pub const DEFAULT_TIMEOUT_MS: u64 = 3000; - /// Default Access Key /// Default value: rustfsadmin /// Environment variable: RUSTFS_ACCESS_KEY @@ -154,9 +147,18 @@ pub const DEFAULT_LOG_ROTATION_TIME: &str = "day"; /// Environment variable: RUSTFS_OBS_LOG_KEEP_FILES pub const DEFAULT_LOG_KEEP_FILES: u16 = 30; -/// 1 KiB +/// Default log local logging enabled for rustfs +/// This is the default log local logging enabled for rustfs. +/// It is used to enable or disable local logging of the application. +/// Default value: false +/// Environment variable: RUSTFS_OBS_LOCAL_LOGGING_ENABLED +pub const DEFAULT_LOG_LOCAL_LOGGING_ENABLED: bool = false; + +/// Constant representing 1 Kibibyte (1024 bytes) +/// Default value: 1024 pub const KI_B: usize = 1024; -/// 1 MiB +/// Constant representing 1 Mebibyte (1024 * 1024 bytes) +/// Default value: 1048576 pub const MI_B: usize = 1024 * 1024; #[cfg(test)] @@ -169,16 +171,16 @@ mod tests { assert_eq!(APP_NAME, "RustFS"); assert!(!APP_NAME.contains(' '), "App name should not contain spaces"); - assert_eq!(VERSION, "0.0.1"); + assert_eq!(VERSION, "1.0.0"); - assert_eq!(SERVICE_VERSION, "0.0.1"); + assert_eq!(SERVICE_VERSION, "1.0.0"); assert_eq!(VERSION, SERVICE_VERSION, "Version and service version should be consistent"); } #[test] fn test_logging_constants() { // Test logging related constants - assert_eq!(DEFAULT_LOG_LEVEL, "info"); + assert_eq!(DEFAULT_LOG_LEVEL, "error"); assert!( ["trace", "debug", "info", "warn", "error"].contains(&DEFAULT_LOG_LEVEL), "Log level should be a valid tracing level" @@ -199,14 +201,6 @@ mod tests { ); } - #[test] - fn test_connection_constants() { - // Test connection related constants - assert_eq!(MAX_CONNECTIONS, 100); - - assert_eq!(DEFAULT_TIMEOUT_MS, 3000); - } - #[test] fn test_security_constants() { // Test security related constants @@ -309,8 +303,8 @@ mod tests { // assert!(DEFAULT_TIMEOUT_MS < u64::MAX, "Timeout should be reasonable"); // These are const non-zero values, so zero checks are redundant - // assert!(DEFAULT_PORT != 0, "Default port should not be zero"); - // assert!(DEFAULT_CONSOLE_PORT != 0, "Console port should not be zero"); + assert_ne!(DEFAULT_PORT, 0, "Default port should not be zero"); + assert_ne!(DEFAULT_CONSOLE_PORT, 0, "Console port should not be zero"); } #[test] diff --git a/crates/config/src/constants/runtime.rs b/crates/config/src/constants/runtime.rs index 044dff963..b9fa58629 100644 --- a/crates/config/src/constants/runtime.rs +++ b/crates/config/src/constants/runtime.rs @@ -22,7 +22,10 @@ pub const ENV_THREAD_STACK_SIZE: &str = "RUSTFS_RUNTIME_THREAD_STACK_SIZE"; pub const ENV_THREAD_KEEP_ALIVE: &str = "RUSTFS_RUNTIME_THREAD_KEEP_ALIVE"; pub const ENV_GLOBAL_QUEUE_INTERVAL: &str = "RUSTFS_RUNTIME_GLOBAL_QUEUE_INTERVAL"; pub const ENV_THREAD_NAME: &str = "RUSTFS_RUNTIME_THREAD_NAME"; +pub const ENV_MAX_IO_EVENTS_PER_TICK: &str = "RUSTFS_RUNTIME_MAX_IO_EVENTS_PER_TICK"; pub const ENV_RNG_SEED: &str = "RUSTFS_RUNTIME_RNG_SEED"; +/// Event polling interval +pub const ENV_EVENT_INTERVAL: &str = "RUSTFS_RUNTIME_EVENT_INTERVAL"; // Default values for Tokio runtime pub const DEFAULT_WORKER_THREADS: usize = 16; @@ -32,4 +35,7 @@ pub const DEFAULT_THREAD_STACK_SIZE: usize = MI_B; // 1 MiB pub const DEFAULT_THREAD_KEEP_ALIVE: u64 = 60; // seconds pub const DEFAULT_GLOBAL_QUEUE_INTERVAL: u32 = 31; pub const DEFAULT_THREAD_NAME: &str = "rustfs-worker"; +pub const DEFAULT_MAX_IO_EVENTS_PER_TICK: usize = 1024; +/// Event polling default (Tokio default 61) +pub const DEFAULT_EVENT_INTERVAL: u32 = 61; pub const DEFAULT_RNG_SEED: Option = None; // None means random diff --git a/crates/ecstore/Cargo.toml b/crates/ecstore/Cargo.toml index ecc2dd84a..3ea356715 100644 --- a/crates/ecstore/Cargo.toml +++ b/crates/ecstore/Cargo.toml @@ -103,8 +103,8 @@ parking_lot = { workspace = true } moka = { workspace = true } base64-simd.workspace = true serde_urlencoded.workspace = true -google-cloud-storage = "1.1.0" -google-cloud-auth = "1.0.1" +google-cloud-storage = { workspace = true } +google-cloud-auth = { workspace = true } aws-config = { workspace = true } [target.'cfg(not(windows))'.dependencies] @@ -118,7 +118,6 @@ winapi = { workspace = true } tokio = { workspace = true, features = ["rt-multi-thread", "macros"] } criterion = { workspace = true, features = ["html_reports"] } temp-env = { workspace = true } -heed = "0.22.0" [build-dependencies] shadow-rs = { workspace = true, features = ["build", "metadata"] } diff --git a/crates/obs/Cargo.toml b/crates/obs/Cargo.toml index a181b37b2..507455e7a 100644 --- a/crates/obs/Cargo.toml +++ b/crates/obs/Cargo.toml @@ -31,6 +31,7 @@ workspace = true [features] default = [] gpu = ["dep:nvml-wrapper"] +full = ["gpu"] [dependencies] rustfs-config = { workspace = true, features = ["constants", "observability"] } diff --git a/crates/obs/examples/server.rs b/crates/obs/examples/server.rs index 5741d4040..b0121caf0 100644 --- a/crates/obs/examples/server.rs +++ b/crates/obs/examples/server.rs @@ -43,7 +43,7 @@ async fn run(service_name: String) { &[opentelemetry::KeyValue::new("operation", "run")], ); - match SystemObserver::init_process_observer(meter).await { + match SystemObserver::init_process_observer().await { Ok(_) => info!("Process observer initialized successfully"), Err(e) => error!("Failed to initialize process observer: {:?}", e), } diff --git a/crates/obs/src/config.rs b/crates/obs/src/config.rs index d15fc6771..c1d46dd84 100644 --- a/crates/obs/src/config.rs +++ b/crates/obs/src/config.rs @@ -18,21 +18,39 @@ use rustfs_config::observability::{ ENV_OBS_METER_INTERVAL, ENV_OBS_SAMPLE_RATIO, ENV_OBS_SERVICE_NAME, ENV_OBS_SERVICE_VERSION, ENV_OBS_USE_STDOUT, }; use rustfs_config::{ - APP_NAME, DEFAULT_LOG_KEEP_FILES, DEFAULT_LOG_LEVEL, DEFAULT_LOG_ROTATION_SIZE_MB, DEFAULT_LOG_ROTATION_TIME, - DEFAULT_OBS_LOG_FILENAME, ENVIRONMENT, METER_INTERVAL, SAMPLE_RATIO, SERVICE_VERSION, USE_STDOUT, + APP_NAME, DEFAULT_LOG_KEEP_FILES, DEFAULT_LOG_LEVEL, DEFAULT_LOG_LOCAL_LOGGING_ENABLED, DEFAULT_LOG_ROTATION_SIZE_MB, + DEFAULT_LOG_ROTATION_TIME, DEFAULT_OBS_LOG_FILENAME, ENVIRONMENT, METER_INTERVAL, SAMPLE_RATIO, SERVICE_VERSION, USE_STDOUT, }; use rustfs_utils::dirs::get_log_directory_to_string; use serde::{Deserialize, Serialize}; use std::env; -/// OpenTelemetry Configuration -/// Add service name, service version, environment -/// Add interval time for metric collection -/// Add sample ratio for trace sampling -/// Add endpoint for metric collection -/// Add use_stdout for output to stdout -/// Add logger level for log level -/// Add local_logging_enabled for local logging enabled +/// Observability: OpenTelemetry configuration +/// # Fields +/// * `endpoint`: Endpoint for metric collection +/// * `use_stdout`: Output to stdout +/// * `sample_ratio`: Trace sampling ratio +/// * `meter_interval`: Metric collection interval +/// * `service_name`: Service name +/// * `service_version`: Service version +/// * `environment`: Environment +/// * `logger_level`: Logger level +/// * `local_logging_enabled`: Local logging enabled +/// # Added flexi_logger related configurations +/// * `log_directory`: Log file directory +/// * `log_filename`: The name of the log file +/// * `log_rotation_size_mb`: Log file size cut threshold (MB) +/// * `log_rotation_time`: Logs are cut by time (Hour,Day,Minute,Second) +/// * `log_keep_files`: Number of log files to be retained +/// # Returns +/// A new instance of OtelConfig +/// +/// # Example +/// ```no_run +/// use rustfs_obs::OtelConfig; +/// +/// let config = OtelConfig::new(); +/// ``` #[derive(Debug, Deserialize, Serialize, Clone)] pub struct OtelConfig { pub endpoint: String, // Endpoint for metric collection @@ -102,7 +120,7 @@ impl OtelConfig { local_logging_enabled: env::var(ENV_OBS_LOCAL_LOGGING_ENABLED) .ok() .and_then(|v| v.parse().ok()) - .or(Some(false)), + .or(Some(DEFAULT_LOG_LOCAL_LOGGING_ENABLED)), log_directory: Some(get_log_directory_to_string(ENV_OBS_LOG_DIRECTORY)), log_filename: env::var(ENV_OBS_LOG_FILENAME) .ok() @@ -127,11 +145,28 @@ impl OtelConfig { /// /// # Returns /// A new instance of OtelConfig + /// + /// # Example + /// ```no_run + /// use rustfs_obs::OtelConfig; + /// + /// let config = OtelConfig::new(); + /// ``` pub fn new() -> Self { Self::extract_otel_config_from_env(None) } } +/// Implement Default trait for OtelConfig +/// This allows creating a default instance of OtelConfig using OtelConfig::default() +/// which internally calls OtelConfig::new() +/// +/// # Example +/// ```no_run +/// use rustfs_obs::OtelConfig; +/// +/// let config = OtelConfig::default(); +/// ``` impl Default for OtelConfig { fn default() -> Self { Self::new() @@ -165,6 +200,20 @@ impl AppConfig { } } + /// Create a new instance of AppConfig with specified endpoint + /// + /// # Arguments + /// * `endpoint` - An optional string representing the endpoint for metric collection + /// + /// # Returns + /// A new instance of AppConfig + /// + /// # Example + /// ```no_run + /// use rustfs_obs::AppConfig; + /// + /// let config = AppConfig::new_with_endpoint(Some("http://localhost:4317".to_string())); + /// ``` pub fn new_with_endpoint(endpoint: Option) -> Self { Self { observability: OtelConfig::extract_otel_config_from_env(endpoint), @@ -172,7 +221,16 @@ impl AppConfig { } } -// implement default for AppConfig +/// Implement Default trait for AppConfig +/// This allows creating a default instance of AppConfig using AppConfig::default() +/// which internally calls AppConfig::new() +/// +/// # Example +/// ```no_run +/// use rustfs_obs::AppConfig; +/// +/// let config = AppConfig::default(); +/// ``` impl Default for AppConfig { fn default() -> Self { Self::new() diff --git a/crates/obs/src/global.rs b/crates/obs/src/global.rs index 5b373f331..04791982f 100644 --- a/crates/obs/src/global.rs +++ b/crates/obs/src/global.rs @@ -12,10 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::AppConfig; use crate::telemetry::{OtelGuard, init_telemetry}; -use opentelemetry::metrics::Meter; -use rustfs_config::APP_NAME; +use crate::{AppConfig, SystemObserver}; use std::sync::{Arc, Mutex}; use tokio::sync::{OnceCell, SetError}; use tracing::{error, info}; @@ -26,20 +24,11 @@ static GLOBAL_GUARD: OnceCell>> = OnceCell::const_new(); /// Flag indicating if observability is enabled pub(crate) static IS_OBSERVABILITY_ENABLED: OnceCell = OnceCell::const_new(); -/// Name of the observability meter -pub(crate) static OBSERVABILITY_METER_NAME: OnceCell = OnceCell::const_new(); - /// Check whether Observability is enabled pub fn is_observability_enabled() -> bool { IS_OBSERVABILITY_ENABLED.get().copied().unwrap_or(false) } -/// Get the global meter for observability -pub fn global_meter() -> Meter { - let meter_name = OBSERVABILITY_METER_NAME.get().map(|s| s.as_str()).unwrap_or(APP_NAME); - opentelemetry::global::meter(meter_name) -} - /// Error type for global guard operations #[derive(Debug, thiserror::Error)] pub enum GlobalError { @@ -75,18 +64,33 @@ pub enum GlobalError { /// /// # Example /// ```no_run -/// use rustfs_obs::init_obs; +/// # use rustfs_obs::init_obs; /// /// # #[tokio::main] /// # async fn main() { -/// let guard = init_obs(None).await; +/// # let guard = init_obs(None).await; /// # } /// ``` pub async fn init_obs(endpoint: Option) -> OtelGuard { // Load the configuration file let config = AppConfig::new_with_endpoint(endpoint); - init_telemetry(&config.observability) + let otel_guard = init_telemetry(&config.observability); + // Server will be created per connection - this ensures isolation + tokio::spawn(async move { + // Record the PID-related metrics of the current process + let obs_result = SystemObserver::init_process_observer().await; + match obs_result { + Ok(_) => { + info!(target: "rustfs::obs::system::metrics","Process observer initialized successfully"); + } + Err(e) => { + error!(target: "rustfs::obs::system::metrics","Failed to initialize process observer: {}", e); + } + } + }); + + otel_guard } /// Set the global guard for OpenTelemetry @@ -99,14 +103,14 @@ pub async fn init_obs(endpoint: Option) -> OtelGuard { /// * `Err(GuardError)` if setting fails /// /// # Example -/// ```rust -/// use rustfs_obs::{ init_obs, set_global_guard}; +/// ```no_run +/// # use rustfs_obs::{ init_obs, set_global_guard}; /// -/// async fn init() -> Result<(), Box> { -/// let guard = init_obs(None).await; -/// set_global_guard(guard)?; -/// Ok(()) -/// } +/// # async fn init() -> Result<(), Box> { +/// # let guard = init_obs(None).await; +/// # set_global_guard(guard)?; +/// # Ok(()) +/// # } /// ``` pub fn set_global_guard(guard: OtelGuard) -> Result<(), GlobalError> { info!("Initializing global OpenTelemetry guard"); @@ -120,29 +124,20 @@ pub fn set_global_guard(guard: OtelGuard) -> Result<(), GlobalError> { /// * `Err(GuardError)` if guard not initialized /// /// # Example -/// ```rust -/// use rustfs_obs::get_global_guard; +/// ```no_run +/// # use rustfs_obs::get_global_guard; /// -/// async fn trace_operation() -> Result<(), Box> { -/// let guard = get_global_guard()?; -/// let _lock = guard.lock().unwrap(); -/// // Perform traced operation -/// Ok(()) -/// } +/// # async fn trace_operation() -> Result<(), Box> { +/// # let guard = get_global_guard()?; +/// # let _lock = guard.lock().unwrap(); +/// # // Perform traced operation +/// # Ok(()) +/// # } /// ``` pub fn get_global_guard() -> Result>, GlobalError> { GLOBAL_GUARD.get().cloned().ok_or(GlobalError::NotInitialized) } -/// Try to get the global guard for OpenTelemetry -/// -/// # Returns -/// * `Some(Arc>)` if guard exists -/// * `None` if guard not initialized -pub fn try_get_global_guard() -> Option>> { - GLOBAL_GUARD.get().cloned() -} - #[cfg(test)] mod tests { use super::*; diff --git a/crates/obs/src/lib.rs b/crates/obs/src/lib.rs index 43bfec38d..43bc0542b 100644 --- a/crates/obs/src/lib.rs +++ b/crates/obs/src/lib.rs @@ -17,7 +17,7 @@ //! provides tools for system and service monitoring //! //! ## feature mark -//! +//! - `default`: default monitoring function //! - `gpu`: gpu monitoring function //! - `full`: includes all functions //! @@ -47,6 +47,6 @@ mod metrics; mod system; mod telemetry; -pub use config::AppConfig; +pub use config::{AppConfig, OtelConfig}; pub use global::*; pub use system::SystemObserver; diff --git a/crates/obs/src/system/attributes.rs b/crates/obs/src/system/attributes.rs index 9cd34fcec..86c553cec 100644 --- a/crates/obs/src/system/attributes.rs +++ b/crates/obs/src/system/attributes.rs @@ -16,10 +16,10 @@ use crate::GlobalError; use opentelemetry::KeyValue; use sysinfo::{Pid, System}; -pub const PROCESS_PID: opentelemetry::Key = opentelemetry::Key::from_static_str("process.pid"); -pub const PROCESS_EXECUTABLE_NAME: opentelemetry::Key = opentelemetry::Key::from_static_str("process.executable.name"); -pub const PROCESS_EXECUTABLE_PATH: opentelemetry::Key = opentelemetry::Key::from_static_str("process.executable.path"); -pub const PROCESS_COMMAND: opentelemetry::Key = opentelemetry::Key::from_static_str("process.command"); +pub(crate) const PROCESS_PID: opentelemetry::Key = opentelemetry::Key::from_static_str("process.pid"); +pub(crate) const PROCESS_EXECUTABLE_NAME: opentelemetry::Key = opentelemetry::Key::from_static_str("process.executable.name"); +pub(crate) const PROCESS_EXECUTABLE_PATH: opentelemetry::Key = opentelemetry::Key::from_static_str("process.executable.path"); +pub(crate) const PROCESS_COMMAND: opentelemetry::Key = opentelemetry::Key::from_static_str("process.command"); /// Struct to hold process attributes pub struct ProcessAttributes { diff --git a/crates/obs/src/system/collector.rs b/crates/obs/src/system/collector.rs index ecaa05101..45e84d23e 100644 --- a/crates/obs/src/system/collector.rs +++ b/crates/obs/src/system/collector.rs @@ -14,7 +14,6 @@ use crate::GlobalError; use crate::system::attributes::ProcessAttributes; -use crate::system::gpu::GpuCollector; use crate::system::metrics::{DIRECTION, INTERFACE, Metrics, STATUS}; use opentelemetry::KeyValue; use std::time::SystemTime; @@ -27,7 +26,8 @@ use tokio::time::{Duration, sleep}; pub struct Collector { metrics: Metrics, attributes: ProcessAttributes, - gpu_collector: GpuCollector, + #[cfg(feature = "gpu")] + gpu_collector: crate::system::gpu::GpuCollector, pid: Pid, system: System, networks: Networks, @@ -41,12 +41,14 @@ impl Collector { let attributes = ProcessAttributes::new(pid, &mut system)?; let core_count = System::physical_core_count().ok_or(GlobalError::CoreCountError)?; let metrics = Metrics::new(&meter); - let gpu_collector = GpuCollector::new(pid)?; + #[cfg(feature = "gpu")] + let gpu_collector = crate::system::gpu::GpuCollector::new(pid)?; let networks = Networks::new_with_refreshed_list(); Ok(Collector { metrics, attributes, + #[cfg(feature = "gpu")] gpu_collector, pid, system, @@ -163,6 +165,7 @@ impl Collector { ); // GPU Metrics (Optional) Non-MacOS + #[cfg(feature = "gpu")] self.gpu_collector.collect(&self.metrics, &self.attributes)?; Ok(()) diff --git a/crates/obs/src/system/gpu.rs b/crates/obs/src/system/gpu.rs index edb33a2f0..403c30ae8 100644 --- a/crates/obs/src/system/gpu.rs +++ b/crates/obs/src/system/gpu.rs @@ -12,29 +12,20 @@ // See the License for the specific language governing permissions and // limitations under the License. -#[cfg(feature = "gpu")] use crate::GlobalError; -#[cfg(feature = "gpu")] use crate::system::attributes::ProcessAttributes; -#[cfg(feature = "gpu")] use crate::system::metrics::Metrics; -#[cfg(feature = "gpu")] use nvml_wrapper::Nvml; -#[cfg(feature = "gpu")] use nvml_wrapper::enums::device::UsedGpuMemory; -#[cfg(feature = "gpu")] use sysinfo::Pid; -#[cfg(feature = "gpu")] use tracing::warn; /// `GpuCollector` is responsible for collecting GPU memory usage metrics. -#[cfg(feature = "gpu")] pub struct GpuCollector { nvml: Nvml, pid: Pid, } -#[cfg(feature = "gpu")] impl GpuCollector { pub fn new(pid: Pid) -> Result { let nvml = Nvml::init().map_err(|e| GlobalError::GpuInitError(e.to_string()))?; @@ -64,21 +55,3 @@ impl GpuCollector { Ok(()) } } - -#[cfg(not(feature = "gpu"))] -pub struct GpuCollector; - -#[cfg(not(feature = "gpu"))] -impl GpuCollector { - pub fn new(_pid: sysinfo::Pid) -> Result { - Ok(GpuCollector) - } - - pub fn collect( - &self, - _metrics: &crate::system::metrics::Metrics, - _attributes: &crate::system::attributes::ProcessAttributes, - ) -> Result<(), crate::GlobalError> { - Ok(()) - } -} diff --git a/crates/obs/src/system/metrics.rs b/crates/obs/src/system/metrics.rs index 3e1e86093..43883956b 100644 --- a/crates/obs/src/system/metrics.rs +++ b/crates/obs/src/system/metrics.rs @@ -12,19 +12,21 @@ // See the License for the specific language governing permissions and // limitations under the License. -pub const PROCESS_CPU_USAGE: &str = "process.cpu.usage"; -pub const PROCESS_CPU_UTILIZATION: &str = "process.cpu.utilization"; -pub const PROCESS_MEMORY_USAGE: &str = "process.memory.usage"; -pub const PROCESS_MEMORY_VIRTUAL: &str = "process.memory.virtual"; -pub const PROCESS_DISK_IO: &str = "process.disk.io"; -pub const PROCESS_NETWORK_IO: &str = "process.network.io"; -pub const PROCESS_NETWORK_IO_PER_INTERFACE: &str = "process.network.io.per_interface"; -pub const PROCESS_STATUS: &str = "process.status"; +use opentelemetry::metrics::{Gauge, Meter}; + +pub(crate) const PROCESS_CPU_USAGE: &str = "process.cpu.usage"; +pub(crate) const PROCESS_CPU_UTILIZATION: &str = "process.cpu.utilization"; +pub(crate) const PROCESS_MEMORY_USAGE: &str = "process.memory.usage"; +pub(crate) const PROCESS_MEMORY_VIRTUAL: &str = "process.memory.virtual"; +pub(crate) const PROCESS_DISK_IO: &str = "process.disk.io"; +pub(crate) const PROCESS_NETWORK_IO: &str = "process.network.io"; +pub(crate) const PROCESS_NETWORK_IO_PER_INTERFACE: &str = "process.network.io.per_interface"; +pub(crate) const PROCESS_STATUS: &str = "process.status"; #[cfg(feature = "gpu")] pub const PROCESS_GPU_MEMORY_USAGE: &str = "process.gpu.memory.usage"; -pub const DIRECTION: opentelemetry::Key = opentelemetry::Key::from_static_str("direction"); -pub const STATUS: opentelemetry::Key = opentelemetry::Key::from_static_str("status"); -pub const INTERFACE: opentelemetry::Key = opentelemetry::Key::from_static_str("interface"); +pub(crate) const DIRECTION: opentelemetry::Key = opentelemetry::Key::from_static_str("direction"); +pub(crate) const STATUS: opentelemetry::Key = opentelemetry::Key::from_static_str("status"); +pub(crate) const INTERFACE: opentelemetry::Key = opentelemetry::Key::from_static_str("interface"); /// `Metrics` struct holds the OpenTelemetry metrics for process monitoring. /// It contains various metrics such as CPU usage, memory usage, @@ -36,20 +38,20 @@ pub const INTERFACE: opentelemetry::Key = opentelemetry::Key::from_static_str("i /// The `new` method initializes the metrics using the provided /// `opentelemetry::metrics::Meter`. pub struct Metrics { - pub cpu_usage: opentelemetry::metrics::Gauge, - pub cpu_utilization: opentelemetry::metrics::Gauge, - pub memory_usage: opentelemetry::metrics::Gauge, - pub memory_virtual: opentelemetry::metrics::Gauge, - pub disk_io: opentelemetry::metrics::Gauge, - pub network_io: opentelemetry::metrics::Gauge, - pub network_io_per_interface: opentelemetry::metrics::Gauge, - pub process_status: opentelemetry::metrics::Gauge, + pub cpu_usage: Gauge, + pub cpu_utilization: Gauge, + pub memory_usage: Gauge, + pub memory_virtual: Gauge, + pub disk_io: Gauge, + pub network_io: Gauge, + pub network_io_per_interface: Gauge, + pub process_status: Gauge, #[cfg(feature = "gpu")] - pub gpu_memory_usage: opentelemetry::metrics::Gauge, + pub gpu_memory_usage: Gauge, } impl Metrics { - pub fn new(meter: &opentelemetry::metrics::Meter) -> Self { + pub fn new(meter: &Meter) -> Self { let cpu_usage = meter .f64_gauge(PROCESS_CPU_USAGE) .with_description("The percentage of CPU in use.") diff --git a/crates/obs/src/system/mod.rs b/crates/obs/src/system/mod.rs index 4e71ba603..4fdf6fcd6 100644 --- a/crates/obs/src/system/mod.rs +++ b/crates/obs/src/system/mod.rs @@ -12,12 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::GlobalError; +use crate::{GlobalError, is_observability_enabled}; +use opentelemetry::global::meter; -pub(crate) mod attributes; +mod attributes; mod collector; -pub(crate) mod gpu; -pub(crate) mod metrics; +#[cfg(feature = "gpu")] +mod gpu; +mod metrics; pub struct SystemObserver {} @@ -25,10 +27,12 @@ impl SystemObserver { /// Initialize the indicator collector for the current process /// This function will create a new `Collector` instance and start collecting metrics. /// It will run indefinitely until the process is terminated. - pub async fn init_process_observer(meter: opentelemetry::metrics::Meter) -> Result<(), GlobalError> { - let pid = sysinfo::get_current_pid().map_err(|e| GlobalError::PidError(e.to_string()))?; - let mut collector = collector::Collector::new(pid, meter, 30000)?; - collector.run().await + pub async fn init_process_observer() -> Result<(), GlobalError> { + if is_observability_enabled() { + let meter = meter("system"); + return SystemObserver::init_process_observer_for_pid(meter, 30000).await; + } + Ok(()) } /// Initialize the metric collector for the specified PID process diff --git a/crates/obs/src/telemetry.rs b/crates/obs/src/telemetry.rs index 9b0f7a36c..9507fd4b5 100644 --- a/crates/obs/src/telemetry.rs +++ b/crates/obs/src/telemetry.rs @@ -13,7 +13,7 @@ // limitations under the License. use crate::config::OtelConfig; -use crate::global::{IS_OBSERVABILITY_ENABLED, OBSERVABILITY_METER_NAME}; +use crate::global::IS_OBSERVABILITY_ENABLED; use flexi_logger::{ Age, Cleanup, Criterion, DeferredNow, FileSpec, LogSpecification, Naming, Record, WriteMode, WriteMode::{AsyncWith, BufferAndFlush}, @@ -36,7 +36,8 @@ use opentelemetry_semantic_conventions::{ attribute::{DEPLOYMENT_ENVIRONMENT_NAME, NETWORK_LOCAL_ADDRESS, SERVICE_VERSION as OTEL_SERVICE_VERSION}, }; use rustfs_config::{ - APP_NAME, DEFAULT_LOG_KEEP_FILES, DEFAULT_LOG_LEVEL, ENVIRONMENT, METER_INTERVAL, SAMPLE_RATIO, SERVICE_VERSION, USE_STDOUT, + APP_NAME, DEFAULT_LOG_KEEP_FILES, DEFAULT_LOG_LEVEL, DEFAULT_LOG_LOCAL_LOGGING_ENABLED, ENVIRONMENT, METER_INTERVAL, + SAMPLE_RATIO, SERVICE_VERSION, USE_STDOUT, observability::{ DEFAULT_OBS_ENVIRONMENT_PRODUCTION, DEFAULT_OBS_LOG_FLUSH_MS, DEFAULT_OBS_LOG_MESSAGE_CAPA, DEFAULT_OBS_LOG_POOL_CAPA, ENV_OBS_LOG_DIRECTORY, @@ -294,7 +295,7 @@ pub(crate) fn init_telemetry(config: &OtelConfig) -> OtelGuard { tracing_subscriber::registry() .with(filter) .with(ErrorLayer::default()) - .with(if config.local_logging_enabled.unwrap_or(false) { + .with(if config.local_logging_enabled.unwrap_or(DEFAULT_LOG_LOCAL_LOGGING_ENABLED) { Some(fmt_layer) } else { None @@ -312,7 +313,6 @@ pub(crate) fn init_telemetry(config: &OtelConfig) -> OtelGuard { env::var("RUST_LOG").unwrap_or_else(|_| "Not set".to_string()) ); IS_OBSERVABILITY_ENABLED.set(true).ok(); - OBSERVABILITY_METER_NAME.set(service_name.to_string()).ok(); } } counter!("rustfs.start.total").increment(1); diff --git a/rustfs/Cargo.toml b/rustfs/Cargo.toml index 6e6296661..a2fac5ff1 100644 --- a/rustfs/Cargo.toml +++ b/rustfs/Cargo.toml @@ -73,7 +73,7 @@ http.workspace = true http-body.workspace = true reqwest = { workspace = true } socket2 = { workspace = true } -tokio = { workspace = true, features = ["rt-multi-thread", "macros", "net", "signal"] } +tokio = { workspace = true, features = ["rt-multi-thread", "macros", "net", "signal", "process"] } tokio-rustls = { workspace = true } tokio-stream.workspace = true tokio-util.workspace = true @@ -113,6 +113,7 @@ pin-project-lite.workspace = true rust-embed = { workspace = true, features = ["interpolate-folder-path"] } s3s.workspace = true shadow-rs = { workspace = true, features = ["build", "metadata"] } +sysinfo = { workspace = true, features = ["multithread"] } thiserror = { workspace = true } tracing.workspace = true url = { workspace = true } diff --git a/rustfs/src/main.rs b/rustfs/src/main.rs index 64598436d..1302499a7 100644 --- a/rustfs/src/main.rs +++ b/rustfs/src/main.rs @@ -97,25 +97,9 @@ fn print_server_info() { } fn main() -> Result<()> { - let mut builder = server::get_tokio_runtime_builder(); - builder.enable_all(); - if server::print_tokio_thread_enable() { - builder.on_thread_start(|| { - println!( - "RustFS Worker Thread running - initializing resources time: {:?}, thread id: {:?}", - chrono::Utc::now().to_rfc3339(), - std::thread::current().id() - ); - }); - builder.on_thread_stop(|| { - println!( - "RustFS Worker Thread stopping - cleaning up resources time: {:?}, thread id: {:?}", - chrono::Utc::now().to_rfc3339(), - std::thread::current().id() - ) - }); - } - let runtime = builder.build().expect("Failed to build Tokio runtime"); + let runtime = server::get_tokio_runtime_builder() + .build() + .expect("Failed to build Tokio runtime"); runtime.block_on(async_main()) } async fn async_main() -> Result<()> { diff --git a/rustfs/src/server/http.rs b/rustfs/src/server/http.rs index ace6d8981..da1d4ad2b 100644 --- a/rustfs/src/server/http.rs +++ b/rustfs/src/server/http.rs @@ -29,7 +29,6 @@ use hyper_util::{ }; use metrics::{counter, histogram}; use rustfs_config::{DEFAULT_ACCESS_KEY, DEFAULT_SECRET_KEY, MI_B, RUSTFS_TLS_CERT, RUSTFS_TLS_KEY}; -use rustfs_obs::SystemObserver; use rustfs_protos::proto_gen::node_service::node_service_server::NodeServiceServer; use rustfs_utils::net::parse_and_resolve_address; use rustls::ServerConfig; @@ -221,21 +220,6 @@ pub async fn start_http_server( b.build() }; - // Server will be created per connection - this ensures isolation - tokio::spawn(async move { - // Record the PID-related metrics of the current process - let meter = opentelemetry::global::meter("system"); - let obs_result = SystemObserver::init_process_observer(meter).await; - match obs_result { - Ok(_) => { - info!("Process observer initialized successfully"); - } - Err(e) => { - error!("Failed to initialize process observer: {}", e); - } - } - }); - // Create shutdown channel let (shutdown_tx, mut shutdown_rx) = tokio::sync::broadcast::channel(1); let shutdown_tx_clone = shutdown_tx.clone(); @@ -651,50 +635,3 @@ fn get_listen_backlog() -> i32 { DEFAULT_BACKLOG } } - -/// Customize the Tokio runtime configuration -/// These configurations can be adjusted by environment variables -/// to optimize performance based on the deployment environment -pub(crate) fn get_tokio_runtime_builder() -> tokio::runtime::Builder { - let mut builder = tokio::runtime::Builder::new_multi_thread(); - - // Worker threads - let worker_threads = rustfs_utils::get_env_usize(rustfs_config::ENV_WORKER_THREADS, rustfs_config::DEFAULT_WORKER_THREADS); - builder.worker_threads(worker_threads); - - // Max blocking threads - let max_blocking_threads = - rustfs_utils::get_env_usize(rustfs_config::ENV_MAX_BLOCKING_THREADS, rustfs_config::DEFAULT_MAX_BLOCKING_THREADS); - builder.max_blocking_threads(max_blocking_threads); - - // Thread stack size - let thread_stack_size = - rustfs_utils::get_env_usize(rustfs_config::ENV_THREAD_STACK_SIZE, rustfs_config::DEFAULT_THREAD_STACK_SIZE); - builder.thread_stack_size(thread_stack_size); - - // Thread keep alive - let thread_keep_alive = - rustfs_utils::get_env_u64(rustfs_config::ENV_THREAD_KEEP_ALIVE, rustfs_config::DEFAULT_THREAD_KEEP_ALIVE); - builder.thread_keep_alive(Duration::from_secs(thread_keep_alive)); - - // Global queue interval - let global_queue_interval = - rustfs_utils::get_env_u32(rustfs_config::ENV_GLOBAL_QUEUE_INTERVAL, rustfs_config::DEFAULT_GLOBAL_QUEUE_INTERVAL); - builder.global_queue_interval(global_queue_interval); - - // Thread name - let thread_name = rustfs_utils::get_env_str(rustfs_config::ENV_THREAD_NAME, rustfs_config::DEFAULT_THREAD_NAME); - builder.thread_name(thread_name.clone()); - println!( - "Starting Tokio runtime with configured parameters:\n\ - worker_threads: {worker_threads}, max_blocking_threads: {max_blocking_threads}, thread_stack_size: {thread_stack_size}, thread_keep_alive: {thread_keep_alive}, \ - global_queue_interval: {global_queue_interval}, thread_name: {thread_name}" - ); - builder -} - -/// Whether to print tokio threads -/// This can be useful for debugging purposes -pub(crate) fn print_tokio_thread_enable() -> bool { - rustfs_utils::get_env_bool(rustfs_config::ENV_THREAD_PRINT_ENABLED, rustfs_config::DEFAULT_THREAD_PRINT_ENABLED) -} diff --git a/rustfs/src/server/mod.rs b/rustfs/src/server/mod.rs index 9df696e73..5aee97e39 100644 --- a/rustfs/src/server/mod.rs +++ b/rustfs/src/server/mod.rs @@ -19,10 +19,12 @@ mod layer; mod service_state; mod event; +mod runtime; pub(crate) use audit::{start_audit_system, stop_audit_system}; pub(crate) use event::{init_event_notifier, shutdown_event_notifier}; -pub(crate) use http::{get_tokio_runtime_builder, print_tokio_thread_enable, start_http_server}; +pub(crate) use http::start_http_server; +pub(crate) use runtime::get_tokio_runtime_builder; pub(crate) use service_state::SHUTDOWN_TIMEOUT; pub(crate) use service_state::ServiceState; pub(crate) use service_state::ServiceStateManager; diff --git a/rustfs/src/server/runtime.rs b/rustfs/src/server/runtime.rs new file mode 100644 index 000000000..b10349d7e --- /dev/null +++ b/rustfs/src/server/runtime.rs @@ -0,0 +1,164 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::time::Duration; +use sysinfo::{RefreshKind, System}; + +#[inline] +fn compute_default_thread_stack_size() -> usize { + // Baseline: Release 1 MiB,Debug 2 MiB;macOS at least 2 MiB + // macOS is more conservative: many system libraries and backtracking are more "stack-eating" + if cfg!(debug_assertions) || cfg!(target_os = "macos") { + 2 * rustfs_config::DEFAULT_THREAD_STACK_SIZE + } else { + rustfs_config::DEFAULT_THREAD_STACK_SIZE + } +} + +#[inline] +fn detect_cores() -> usize { + // Priority physical cores, fallback logic cores, minimum 1 + let mut sys = System::new_with_specifics(RefreshKind::everything().without_memory().without_processes()); + sys.refresh_cpu_all(); + sys.cpus().len().max(1) +} + +#[inline] +fn compute_default_worker_threads() -> usize { + // Physical cores are used by default (closer to CPU compute resources and cache topology) + detect_cores() +} + +/// Default max_blocking_threads calculations based on sysinfo: +/// 16 cores -> 1024; more than 16 cores are doubled by multiples: +/// 1..=16 -> 1024, 17..=32 -> 2048, 33..=64 -> 4096, and so on. +fn compute_default_max_blocking_threads() -> usize { + const BASE_CORES: usize = rustfs_config::DEFAULT_WORKER_THREADS; + const BASE_THREADS: usize = rustfs_config::DEFAULT_MAX_BLOCKING_THREADS; + + let cores = detect_cores(); + + let mut threads = BASE_THREADS; + let mut threshold = BASE_CORES; + + // When the number of cores exceeds the threshold, the number of threads is doubled for each doubling threshold + while cores > threshold { + threads = threads.saturating_mul(2); + threshold = threshold.saturating_mul(2); + } + + threads +} + +/// Customize the Tokio runtime configuration +/// These configurations can be adjusted by environment variables +/// to optimize performance based on the deployment environment +/// Custom Tokio runtime builder (can be fully overridden by ENV) +/// - RUSTFS_RUNTIME_WORKER_THREADS +/// - RUSTFS_RUNTIME_MAX_BLOCKING_THREADS +/// - RUSTFS_RUNTIME_THREAD_STACK_SIZE +/// - RUSTFS_RUNTIME_THREAD_KEEP_ALIVE +/// - RUSTFS_RUNTIME_GLOBAL_QUEUE_INTERVAL +/// - RUSTFS_RUNTIME_EVENT_INTERVAL +/// - RUSTFS_RUNTIME_THREAD_NAME +/// - RUSTFS_RUNTIME_MAX_IO_EVENTS_PER_TICK +/// - RUSTFS_RUNTIME_THREAD_PRINT_ENABLED +/// +/// Returns: Configured Tokio runtime builder +/// # Panics +/// Panics if environment variable values are invalid +/// # Examples +/// ```no_run +/// use rustfs_server::get_tokio_runtime_builder; +/// let builder = get_tokio_runtime_builder(); +/// let runtime = builder.build().unwrap(); +/// ``` +pub(crate) fn get_tokio_runtime_builder() -> tokio::runtime::Builder { + let mut builder = tokio::runtime::Builder::new_multi_thread(); + + // Worker threads(Default physical cores) + let default_worker_threads = compute_default_worker_threads(); + let worker_threads = rustfs_utils::get_env_usize(rustfs_config::ENV_WORKER_THREADS, default_worker_threads); + builder.worker_threads(worker_threads); + + // Max blocking threads: Prioritize environment variables, otherwise the default value is dynamically calculated based on sysinfo + let default_max_blocking_threads = compute_default_max_blocking_threads(); + let max_blocking_threads = rustfs_utils::get_env_usize(rustfs_config::ENV_MAX_BLOCKING_THREADS, default_max_blocking_threads); + builder.max_blocking_threads(max_blocking_threads); + + // Thread stack size (environment variables first, followed by dynamic default by platform/build type) + let default_stack = compute_default_thread_stack_size(); + let thread_stack_size = rustfs_utils::get_env_usize(rustfs_config::ENV_THREAD_STACK_SIZE, default_stack); + builder.thread_stack_size(thread_stack_size); + + // Thread keep alive(Blocking the thread pool is kept alive) + let thread_keep_alive = + rustfs_utils::get_env_u64(rustfs_config::ENV_THREAD_KEEP_ALIVE, rustfs_config::DEFAULT_THREAD_KEEP_ALIVE); + builder.thread_keep_alive(Duration::from_secs(thread_keep_alive)); + + // Global queue interval(Task Fairness/Throughput Tradeoff) + let global_queue_interval = + rustfs_utils::get_env_u32(rustfs_config::ENV_GLOBAL_QUEUE_INTERVAL, rustfs_config::DEFAULT_GLOBAL_QUEUE_INTERVAL); + builder.global_queue_interval(global_queue_interval); + + // Event interval(View the interval of I/O/timer events) + let event_interval = rustfs_utils::get_env_u32(rustfs_config::ENV_EVENT_INTERVAL, rustfs_config::DEFAULT_EVENT_INTERVAL); + builder.event_interval(event_interval); + + // Thread name + let thread_name = rustfs_utils::get_env_str(rustfs_config::ENV_THREAD_NAME, rustfs_config::DEFAULT_THREAD_NAME); + builder.thread_name(thread_name.clone()); + + // Enable I/O driver and set the maximum number of I/O events per tick (nevents) + let max_io_events_per_tick = + rustfs_utils::get_env_usize(rustfs_config::ENV_MAX_IO_EVENTS_PER_TICK, rustfs_config::DEFAULT_MAX_IO_EVENTS_PER_TICK); + builder.enable_all().max_io_events_per_tick(max_io_events_per_tick); + + // Optional: Simple log of thread start/stop + if print_tokio_thread_enable() { + builder + .on_thread_start(|| { + let id = std::thread::current().id(); + println!( + "RustFS Worker Thread running - initializing resources time: {:?}, thread id: {:?}", + chrono::Utc::now().to_rfc3339(), + id + ); + }) + .on_thread_stop(|| { + let id = std::thread::current().id(); + println!( + "RustFS Worker Thread stopping - cleaning up resources time: {:?}, thread id: {:?}", + chrono::Utc::now().to_rfc3339(), + id + ) + }); + } + + println!( + "Starting Tokio runtime with configured parameters:\n\ + worker_threads: {worker_threads}, max_blocking_threads: {max_blocking_threads}, \ + thread_stack_size: {thread_stack_size}, thread_keep_alive: {thread_keep_alive}, \ + global_queue_interval: {global_queue_interval}, event_interval: {event_interval}, \ + max_io_events_per_tick: {max_io_events_per_tick}, thread_name: {thread_name}" + ); + + builder +} + +/// Whether to print tokio threads +/// This can be useful for debugging purposes +fn print_tokio_thread_enable() -> bool { + rustfs_utils::get_env_bool(rustfs_config::ENV_THREAD_PRINT_ENABLED, rustfs_config::DEFAULT_THREAD_PRINT_ENABLED) +}