diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 6b9cd9fc7..2e0c38e9a 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -86,7 +86,7 @@ Crates are organized in a dependency DAG with 9 depth levels (0 = leaf, 8 = top) ``` Depth 0 — LEAF (no internal deps): appauth, checksums, config, credentials, crypto, io-metrics, - madmin, mcp, s3-common, workers, zip + madmin, s3-common, workers, zip Depth 1: io-core (→ io-metrics) @@ -195,7 +195,6 @@ Depth 8 — TOP: | `trusted-proxies` | 4.0K | Trusted proxy / IP forwarding | | `zip` | 986 | ZIP archive support for bulk downloads | | `workers` | 136 | Simple worker abstraction | -| `mcp` | 2.0K | Model Context Protocol server (AI tooling) | ## Architecture Invariants @@ -224,7 +223,7 @@ Depth 8 — TOP: 6. **Error types use `thiserror` with descriptive names** (e.g., `StorageError`, not bare `Error`). - ⚠️ VIOLATED: 6 crates use `pub enum Error`; 2 crates use `snafu`; - `mcp` and `heal` use `anyhow` in library code. + `heal` use `anyhow` in library code. ## Known Structural Issues diff --git a/Cargo.lock b/Cargo.lock index b0d00484b..eaeea7ab6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -248,7 +248,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7af50940b73bf4e16c15c448a2b121c63f2d68e3e54b6a8731673cb4aa0cdff5" dependencies = [ "base64ct", - "blake2 0.11.0-rc.5", + "blake2 0.11.0-rc.6", "cpufeatures 0.3.0", "password-hash", ] @@ -696,9 +696,9 @@ dependencies = [ [[package]] name = "aws-lc-rs" -version = "1.16.2" +version = "1.16.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a054912289d18629dc78375ba2c3726a3afe3ff71b4edba9dedfca0e3446d1fc" +checksum = "0ec6fb3fe69024a75fa7e1bfb48aa6cf59706a101658ea01bfd33b2b248a038f" dependencies = [ "aws-lc-sys", "untrusted 0.7.1", @@ -707,9 +707,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.39.1" +version = "0.40.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83a25cf98105baa966497416dbd42565ce3a8cf8dbfd59803ec9ad46f3126399" +checksum = "f50037ee5e1e41e7b8f9d161680a725bd1626cb6f8c7e901f91f942850852fe7" dependencies = [ "cc", "cmake", @@ -894,9 +894,9 @@ dependencies = [ [[package]] name = "aws-smithy-checksums" -version = "0.64.6" +version = "0.64.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6750f3dd509b0694a4377f0293ed2f9630d710b1cebe281fa8bac8f099f88bc6" +checksum = "10efbbcec1e044b81600e2fc562a391951d291152d95b482d5b7e7132299d762" dependencies = [ "aws-smithy-http", "aws-smithy-types", @@ -906,10 +906,10 @@ dependencies = [ "http 1.4.0", "http-body 1.0.1", "http-body-util", - "md-5 0.10.6", + "md-5 0.11.0", "pin-project-lite", - "sha1 0.10.6", - "sha2 0.10.9", + "sha1 0.11.0", + "sha2 0.11.0", "tracing", ] @@ -1000,9 +1000,9 @@ dependencies = [ [[package]] name = "aws-smithy-runtime" -version = "1.10.3" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "028999056d2d2fd58a697232f9eec4a643cf73a71cf327690a7edad1d2af2110" +checksum = "0504b1ab12debb5959e5165ee5fe97dd387e7aa7ea6a477bfd7635dfe769a4f5" dependencies = [ "aws-smithy-async", "aws-smithy-http", @@ -1025,11 +1025,12 @@ dependencies = [ [[package]] name = "aws-smithy-runtime-api" -version = "1.11.6" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "876ab3c9c29791ba4ba02b780a3049e21ec63dabda09268b175272c3733a79e6" +checksum = "b71a13df6ada0aafbf21a73bdfcdf9324cfa9df77d96b8446045be3cde61b42e" dependencies = [ "aws-smithy-async", + "aws-smithy-runtime-api-macros", "aws-smithy-types", "bytes", "http 0.2.12", @@ -1040,6 +1041,17 @@ dependencies = [ "zeroize", ] +[[package]] +name = "aws-smithy-runtime-api-macros" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d7396fd9500589e62e460e987ecb671bad374934e55ec3b5f498cc7a8a8a7b7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "aws-smithy-types" version = "1.4.7" @@ -1238,9 +1250,9 @@ dependencies = [ [[package]] name = "blake2" -version = "0.11.0-rc.5" +version = "0.11.0-rc.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d52965399b470437fc7f4d4b51134668dbc96573fea6f1b83318a420e4605745" +checksum = "061f1a09225e328e1ffbb378d2d49923c0ca5fee19fb5ac1cc9c1e9d52b93690" dependencies = [ "digest 0.11.2", ] @@ -1551,9 +1563,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.6.0" +version = "4.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b193af5b67834b676abd72466a96c1024e6a6ad978a1f484bd90b85c94041351" +checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" dependencies = [ "clap_builder", "clap_derive", @@ -1573,9 +1585,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.6.0" +version = "4.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1110bd8a634a1ab8cb04345d8d878267d57c3cf1b38d91b71af6686408bbca6a" +checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" dependencies = [ "heck", "proc-macro2", @@ -1726,11 +1738,12 @@ dependencies = [ [[package]] name = "const_format" -version = "0.2.35" +version = "0.2.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7faa7469a93a566e9ccc1c73fe783b4a65c274c5ace346038dca9c39fe0030ad" +checksum = "4481a617ad9a412be3b97c5d403fef8ed023103368908b9c50af598ff467cc1e" dependencies = [ "const_format_proc_macros", + "konst", ] [[package]] @@ -1794,15 +1807,6 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" -[[package]] -name = "core2" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b49ba7ef1ad6107f8824dbe97de947cbaac53c44e7f9756a1fba0d37c1eec505" -dependencies = [ - "memchr", -] - [[package]] name = "cpp_demangle" version = "0.5.1" @@ -2206,9 +2210,9 @@ dependencies = [ [[package]] name = "dary_heap" -version = "0.3.8" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06d2e3287df1c007e74221c49ca10a95d557349e54b3a75dc2fb14712c751f04" +checksum = "8b1e3a325bc115f096c8b77bbf027a7c2592230e70be2d985be950d3d5e60ebe" [[package]] name = "dashmap" @@ -2232,9 +2236,9 @@ checksum = "d7a1e2f27636f116493b8b860f5546edb47c8d8f8ea73e1d2a20be88e28d1fea" [[package]] name = "datafusion" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de9f8117889ba9503440f1dd79ebab32ba52ccf1720bb83cd718a29d4edc0d16" +checksum = "93db0e623840612f7f2cd757f7e8a8922064192363732c88692e0870016e141b" dependencies = [ "arrow", "arrow-schema", @@ -2287,9 +2291,9 @@ dependencies = [ [[package]] name = "datafusion-catalog" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be893b73a13671f310ffcc8da2c546b81efcc54c22e0382c0a28aa3537017137" +checksum = "37cefde60b26a7f4ff61e9d2ff2833322f91df2b568d7238afe67bde5bdffb66" dependencies = [ "arrow", "async-trait", @@ -2312,9 +2316,9 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830487b51ed83807d6b32d6325f349c3144ae0c9bf772cf2a712db180c31d5e6" +checksum = "17e112307715d6a7a331111a4c2330ff54bc237183511c319e3708a4cff431fb" dependencies = [ "arrow", "async-trait", @@ -2335,9 +2339,9 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d7663f3af955292f8004e74bcaf8f7ea3d66cc38438749615bb84815b61a293" +checksum = "d72a11ca44a95e1081870d3abb80c717496e8a7acb467a1d3e932bb636af5cc2" dependencies = [ "ahash 0.8.12", "arrow", @@ -2360,9 +2364,9 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f590205c7e32fe1fea48dd53ffb406e56ae0e7a062213a3ac848db8771641bd" +checksum = "89f4afaed29670ec4fd6053643adc749fe3f4bc9d1ce1b8c5679b22c67d12def" dependencies = [ "futures", "log", @@ -2371,9 +2375,9 @@ dependencies = [ [[package]] name = "datafusion-datasource" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fde1e030a9dc87b743c806fbd631f5ecfa2ccaa4ffb61fa19144a07fea406b79" +checksum = "e9fb386e1691355355a96419978a0022b7947b44d4a24a6ea99f00b6b485cbb6" dependencies = [ "arrow", "async-compression", @@ -2406,9 +2410,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-arrow" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "331ebae7055dc108f9b54994b93dff91f3a17445539efe5b74e89264f7b36e15" +checksum = "ffa6c52cfed0734c5f93754d1c0175f558175248bf686c944fb05c373e5fc096" dependencies = [ "arrow", "arrow-ipc", @@ -2430,9 +2434,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e0d475088325e2986876aa27bb30d0574f72a22955a527d202f454681d55c5c" +checksum = "503f29e0582c1fc189578d665ff57d9300da1f80c282777d7eb67bb79fb8cdca" dependencies = [ "arrow", "async-trait", @@ -2453,9 +2457,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea1520d81f31770f3ad6ee98b391e75e87a68a5bb90de70064ace5e0a7182fe8" +checksum = "e33804749abc8d0c8cb7473228483cb8070e524c6f6086ee1b85a64debe2b3d2" dependencies = [ "arrow", "async-trait", @@ -2477,9 +2481,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-parquet" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95be805d0742ab129720f4c51ad9242cd872599cdb076098b03f061fcdc7f946" +checksum = "32a8e0365e0e08e8ff94d912f0ababcf9065a1a304018ba90b1fc83c855b4997" dependencies = [ "arrow", "async-trait", @@ -2507,15 +2511,15 @@ dependencies = [ [[package]] name = "datafusion-doc" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c93ad9e37730d2c7196e68616f3f2dd3b04c892e03acd3a8eeca6e177f3c06a" +checksum = "8de6ac0df1662b9148ad3c987978b32cbec7c772f199b1d53520c8fa764a87ee" [[package]] name = "datafusion-execution" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9437d3cd5d363f9319f8122182d4d233427de79c7eb748f23054c9aaa0fdd8df" +checksum = "c03c7fbdaefcca4ef6ffe425a5fc2325763bfb426599bb0bf4536466efabe709" dependencies = [ "arrow", "arrow-buffer", @@ -2536,9 +2540,9 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67164333342b86521d6d93fa54081ee39839894fb10f7a700c099af96d7552cf" +checksum = "574b9b6977fedbd2a611cbff12e5caf90f31640ad9dc5870f152836d94bad0dd" dependencies = [ "arrow", "async-trait", @@ -2559,9 +2563,9 @@ dependencies = [ [[package]] name = "datafusion-expr-common" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab05fdd00e05d5a6ee362882546d29d6d3df43a6c55355164a7fbee12d163bc9" +checksum = "7d7c3adf3db8bf61e92eb90cb659c8e8b734593a8f7c8e12a843c7ddba24b87e" dependencies = [ "arrow", "datafusion-common", @@ -2572,9 +2576,9 @@ dependencies = [ [[package]] name = "datafusion-functions" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04fb863482d987cf938db2079e07ab0d3bb64595f28907a6c2f8671ad71cca7e" +checksum = "f28aa4e10384e782774b10e72aca4d93ef7b31aa653095d9d4536b0a3dbc51b6" dependencies = [ "arrow", "arrow-buffer", @@ -2604,9 +2608,9 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "829856f4e14275fb376c104f27cbf3c3b57a9cfe24885d98677525f5e43ce8d6" +checksum = "00aa6217e56098ba84e0a338176fe52f0a84cca398021512c6c8c5eff806d0ad" dependencies = [ "ahash 0.8.12", "arrow", @@ -2626,9 +2630,9 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08af79cc3d2aa874a362fb97decfcbd73d687190cb096f16a6c85a7780cce311" +checksum = "b511250349407db7c43832ab2de63f5557b19a20dfd236b39ca2c04468b50d47" dependencies = [ "ahash 0.8.12", "arrow", @@ -2639,9 +2643,9 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "465ae3368146d49c2eda3e2c0ef114424c87e8a6b509ab34c1026ace6497e790" +checksum = "ef13a858e20d50f0a9bb5e96e7ac82b4e7597f247515bccca4fdd2992df0212a" dependencies = [ "arrow", "arrow-ord", @@ -2664,9 +2668,9 @@ dependencies = [ [[package]] name = "datafusion-functions-table" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6156e6b22fcf1784112fc0173f3ae6e78c8fdb4d3ed0eace9543873b437e2af6" +checksum = "72b40d3f5bbb3905f9ccb1ce9485a9595c77b69758a7c24d3ba79e334ff51e7e" dependencies = [ "arrow", "async-trait", @@ -2680,9 +2684,9 @@ dependencies = [ [[package]] name = "datafusion-functions-window" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca7baec14f866729012efb89011a6973f3a346dc8090c567bfcd328deff551c1" +checksum = "d4e88ec9d57c9b685d02f58bfee7be62d72610430ddcedb82a08e5d9925dbfb6" dependencies = [ "arrow", "datafusion-common", @@ -2698,9 +2702,9 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "159228c3280d342658466bb556dc24de30047fe1d7e559dc5d16ccc5324166f9" +checksum = "8307bb93519b1a91913723a1130cfafeee3f72200d870d88e91a6fc5470ede5c" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -2708,9 +2712,9 @@ dependencies = [ [[package]] name = "datafusion-macros" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5427e5da5edca4d21ea1c7f50e1c9421775fe33d7d5726e5641a833566e7578" +checksum = "2e367e6a71051d0ebdd29b2f85d12059b38b1d1f172c6906e80016da662226bd" dependencies = [ "datafusion-doc", "quote", @@ -2719,9 +2723,9 @@ dependencies = [ [[package]] name = "datafusion-optimizer" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89099eefcd5b223ec685c36a41d35c69239236310d71d339f2af0fa4383f3f46" +checksum = "e929015451a67f77d9d8b727b2bf3a40c4445fdef6cdc53281d7d97c76888ace" dependencies = [ "arrow", "chrono", @@ -2739,9 +2743,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f222df5195d605d79098ef37bdd5323bff0131c9d877a24da6ec98dfca9fe36" +checksum = "4b1e68aba7a4b350401cfdf25a3d6f989ad898a7410164afe9ca52080244cb59" dependencies = [ "ahash 0.8.12", "arrow", @@ -2763,9 +2767,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-adapter" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40838625d63d9c12549d81979db3dd675d159055eb9135009ba272ab0e8d0f64" +checksum = "ea22315f33cf2e0adc104e8ec42e285f6ed93998d565c65e82fec6a9ee9f9db4" dependencies = [ "arrow", "datafusion-common", @@ -2778,9 +2782,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eacbcc4cfd502558184ed58fa3c72e775ec65bf077eef5fd2b3453db676f893c" +checksum = "b04b45ea8ad3ac2d78f2ea2a76053e06591c9629c7a603eda16c10649ecf4362" dependencies = [ "ahash 0.8.12", "arrow", @@ -2795,9 +2799,9 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d501d0e1d0910f015677121601ac177ec59272ef5c9324d1147b394988f40941" +checksum = "7cb13397809a425918f608dfe8653f332015a3e330004ab191b4404187238b95" dependencies = [ "arrow", "datafusion-common", @@ -2814,9 +2818,9 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "463c88ad6f1ecab1810f4c9f046898bee035b370137eb79b2b2db925e270631d" +checksum = "5edc023675791af9d5fb4cc4c24abf5f7bd3bd4dcf9e5bd90ea1eff6976dcc79" dependencies = [ "ahash 0.8.12", "arrow", @@ -2846,9 +2850,9 @@ dependencies = [ [[package]] name = "datafusion-pruning" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2857618a0ecbd8cd0cf29826889edd3a25774ec26b2995fc3862095c95d88fc6" +checksum = "ac8c76860e355616555081cab5968cec1af7a80701ff374510860bcd567e365a" dependencies = [ "arrow", "datafusion-common", @@ -2863,9 +2867,9 @@ dependencies = [ [[package]] name = "datafusion-session" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef8637e35022c5c775003b3ab1debc6b4a8f0eb41b069bdd5475dd3aa93f6eba" +checksum = "5412111aa48e2424ba926112e192f7a6b7e4ccb450145d25ce5ede9f19dc491e" dependencies = [ "async-trait", "datafusion-common", @@ -2877,9 +2881,9 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12d9e9f16a1692a11c94bcc418191fa15fd2b4d72a0c1a0c607db93c0b84dd81" +checksum = "fa0d133ddf8b9b3b872acac900157f783e7b879fe9a6bccf389abebbfac45ec1" dependencies = [ "arrow", "bigdecimal", @@ -4902,6 +4906,21 @@ dependencies = [ "simple_asn1", ] +[[package]] +name = "konst" +version = "0.2.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "128133ed7824fcd73d6e7b17957c5eb7bacb885649bd8c69708b2331a10bcefb" +dependencies = [ + "konst_macro_rules", +] + +[[package]] +name = "konst_macro_rules" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4933f3f57a8e9d9da04db23fb153356ecaf00cbd14aee46279c33dc80925c37" + [[package]] name = "lazy-regex" version = "3.6.0" @@ -5011,25 +5030,25 @@ checksum = "52ff2c0fe9bc6cb6b14a0592c2ff4fa9ceb83eea9db979b0487cd054946a2b8f" [[package]] name = "libflate" -version = "2.2.1" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3248b8d211bd23a104a42d81b4fa8bb8ac4a3b75e7a43d85d2c9ccb6179cd74" +checksum = "cd96e993e5f3368b0cb8497dae6c860c22af8ff18388c61c6c0b86c58d86b5df" dependencies = [ "adler32", - "core2", "crc32fast", "dary_heap", "libflate_lz77", + "no_std_io2", ] [[package]] name = "libflate_lz77" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a599cb10a9cd92b1300debcef28da8f70b935ec937f44fcd1b70a7c986a11c5c" +checksum = "ff7a10e427698aef6eef269482776debfef63384d30f13aad39a1a95e0e098fd" dependencies = [ - "core2", "hashbrown 0.16.1", + "no_std_io2", "rle-decode-fast", ] @@ -5559,9 +5578,9 @@ dependencies = [ [[package]] name = "mqttbytes-core-next" -version = "0.29.0" +version = "0.30.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f2068d9ea771ce85a7b8caa23d9e542b10b118e99b2b16dbcac626211c9903c" +checksum = "78a866db83825732641da480c56d8a01ea7f853a639c7cbf5cc82a265ef0ba34" dependencies = [ "bytes", "thiserror 2.0.18", @@ -5579,7 +5598,7 @@ version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7bddcd3bf5144b6392de80e04c347cd7fab2508f6df16a85fc496ecd5cec39bc" dependencies = [ - "rand 0.8.5", + "rand 0.8.6", ] [[package]] @@ -5666,6 +5685,15 @@ dependencies = [ "libc", ] +[[package]] +name = "no_std_io2" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b51ed7824b6e07d354605f4abb3d9d300350701299da96642ee084f5ce631550" +dependencies = [ + "memchr", +] + [[package]] name = "nom" version = "7.1.3" @@ -5738,7 +5766,7 @@ dependencies = [ "num-integer", "num-iter", "num-traits", - "rand 0.8.5", + "rand 0.8.6", "smallvec", "zeroize", ] @@ -5867,7 +5895,7 @@ dependencies = [ "chrono", "getrandom 0.2.17", "http 1.4.0", - "rand 0.8.5", + "rand 0.8.6", "serde", "serde_json", "serde_path_to_error", @@ -5984,7 +6012,7 @@ dependencies = [ "oauth2", "p256 0.13.2", "p384", - "rand 0.8.5", + "rand 0.8.6", "rsa 0.9.10", "serde", "serde-value", @@ -6291,12 +6319,6 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" -[[package]] -name = "pastey" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b867cad97c0791bbd3aaa6472142568c6c9e8f71937e98379f584cfb0cf35bec" - [[package]] name = "path-absolutize" version = "3.1.1" @@ -6587,9 +6609,9 @@ checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" [[package]] name = "portable-atomic-util" -version = "0.2.6" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "091397be61a01d4be58e7841595bd4bfedb15f1cd54977d79b8271e94ed799a3" +checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618" dependencies = [ "portable-atomic", ] @@ -7039,9 +7061,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" dependencies = [ "libc", "rand_chacha 0.3.1", @@ -7471,41 +7493,6 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3582f63211428f83597b51b2ddb88e2a91a9d52d12831f9d08f5e624e8977422" -[[package]] -name = "rmcp" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f542f74cf247da16f19bbc87e298cd201e912314f4083e88cdd671f44f5fcb53" -dependencies = [ - "async-trait", - "base64 0.22.1", - "chrono", - "futures", - "pastey", - "pin-project-lite", - "rmcp-macros", - "schemars 1.2.1", - "serde", - "serde_json", - "thiserror 2.0.18", - "tokio", - "tokio-util", - "tracing", -] - -[[package]] -name = "rmcp-macros" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2391e4ae47f314e70eaafb6c7bd82e495e770b935448864446302143019151f" -dependencies = [ - "darling 0.23.0", - "proc-macro2", - "quote", - "serde_json", - "syn 2.0.117", -] - [[package]] name = "rmp" version = "0.8.15" @@ -7565,9 +7552,9 @@ dependencies = [ [[package]] name = "rumqttc-core-next" -version = "0.29.0" +version = "0.30.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bad77a37d1cf72bf95973879c3b6542530889a27ced6861d98de80145ea0e871" +checksum = "ce991585c5d7af7690647aa87820543b67be40ed88f79addcf36e86a45f911e9" dependencies = [ "async-tungstenite", "futures-io", @@ -7583,18 +7570,18 @@ dependencies = [ [[package]] name = "rumqttc-next" -version = "0.29.0" +version = "0.30.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70844faf7766027294c088fdc456096cb0877eb4a346cc8119b7fc444eeb6afa" +checksum = "0413b02f3961d1059081360990fa5544a88f28884a6fb297ba3eea36a361bfc2" dependencies = [ "rumqttc-v5-next", ] [[package]] name = "rumqttc-v5-next" -version = "0.29.0" +version = "0.30.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8418a883899469a505b30076738ddf344f8c625690fb10602ea965786a094b0" +checksum = "76c0d2a9ac828bcc8ad70d45bed14170d883523cb972cb4064050491f4bd2fe1" dependencies = [ "async-tungstenite", "bytes", @@ -8186,24 +8173,6 @@ dependencies = [ "time", ] -[[package]] -name = "rustfs-mcp" -version = "0.0.5" -dependencies = [ - "anyhow", - "aws-sdk-s3", - "aws-smithy-http-client", - "clap", - "mime_guess", - "rmcp", - "schemars 1.2.1", - "serde", - "serde_json", - "tokio", - "tracing", - "tracing-subscriber", -] - [[package]] name = "rustfs-metrics" version = "0.0.5" @@ -8583,7 +8552,7 @@ name = "rustfs-utils" version = "0.0.5" dependencies = [ "base64-simd", - "blake2 0.11.0-rc.5", + "blake2 0.11.0-rc.6", "brotli", "bytes", "convert_case 0.11.0", @@ -8601,7 +8570,6 @@ dependencies = [ "lz4", "md-5 0.11.0", "netif", - "rand 0.10.1", "regex", "rustix 1.1.4", "rustls", @@ -8895,26 +8863,12 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2b42f36aa1cd011945615b92222f6bf73c599a102a300334cd7f8dbeec726cc" dependencies = [ - "chrono", "dyn-clone", "ref-cast", - "schemars_derive", "serde", "serde_json", ] -[[package]] -name = "schemars_derive" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d115b50f4aaeea07e79c1912f645c7513d81715d0420f8bc77a18c6260b307f" -dependencies = [ - "proc-macro2", - "quote", - "serde_derive_internals", - "syn 2.0.117", -] - [[package]] name = "scopeguard" version = "1.2.0" @@ -9034,17 +8988,6 @@ dependencies = [ "syn 2.0.117", ] -[[package]] -name = "serde_derive_internals" -version = "0.29.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", -] - [[package]] name = "serde_json" version = "1.0.149" @@ -9968,9 +9911,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.52.0" +version = "1.52.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a91135f59b1cbf38c91e73cf3386fca9bb77915c45ce2771460c9d92f0f3d776" +checksum = "b67dee974fe86fd92cc45b7a95fdd2f99a36a6d7b0d431a231178d3d670bbcc6" dependencies = [ "bytes", "io-uring", @@ -10462,9 +10405,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.23.0" +version = "1.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ac8b6f42ead25368cf5b098aeb3dc8a1a2c05a3eee8a9a1a68c640edbfc79d9" +checksum = "ddd74a9687298c6858e9b88ec8935ec45d22e8fd5e6394fa1bd4e99a87789c76" dependencies = [ "getrandom 0.4.2", "js-sys", @@ -10677,18 +10620,18 @@ dependencies = [ [[package]] name = "webpki-root-certs" -version = "1.0.6" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "804f18a4ac2676ffb4e8b5b5fa9ae38af06df08162314f96a68d2a363e21a8ca" +checksum = "f31141ce3fc3e300ae89b78c0dd67f9708061d1d2eda54b8209346fd6be9a92c" dependencies = [ "rustls-pki-types", ] [[package]] name = "webpki-roots" -version = "1.0.6" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22cfaf3c063993ff62e73cb4311efde4db1efb31ab78a3e5c457939ad5cc0bed" +checksum = "52f5ee44c96cf55f1b349600768e3ece3a8f26010c05265ab73f945bb1a2eb9d" dependencies = [ "rustls-pki-types", ] diff --git a/Cargo.toml b/Cargo.toml index e43780f4d..36c414f56 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,7 +31,6 @@ members = [ "crates/kms", # Key Management Service "crates/lock", # Distributed locking implementation "crates/madmin", # Management dashboard and admin API interface - "crates/mcp", # MCP server for S3 operations "crates/metrics", # Metrics collection and reporting "crates/notify", # Notification system for events "crates/obs", # Observability utilities @@ -95,7 +94,6 @@ rustfs-keystone = { path = "crates/keystone", version = "0.0.5" } rustfs-kms = { path = "crates/kms", version = "0.0.5" } rustfs-lock = { path = "crates/lock", version = "0.0.5" } rustfs-madmin = { path = "crates/madmin", version = "0.0.5" } -rustfs-mcp = { path = "crates/mcp", version = "0.0.5" } rustfs-metrics = { path = "crates/metrics", version = "0.0.5" } rustfs-notify = { path = "crates/notify", version = "0.0.5" } rustfs-io-metrics = { path = "crates/io-metrics", version = "0.0.5" } @@ -104,6 +102,7 @@ rustfs-object-capacity = { path = "crates/object-capacity", version = "0.0.5" } rustfs-obs = { path = "crates/obs", version = "0.0.5" } rustfs-policy = { path = "crates/policy", version = "0.0.5" } rustfs-protos = { path = "crates/protos", version = "0.0.5" } +rustfs-protocols = { path = "crates/protocols", version = "0.0.5" } rustfs-rio = { path = "crates/rio", version = "0.0.5" } rustfs-s3-common = { path = "crates/s3-common", version = "0.0.5" } rustfs-s3select-api = { path = "crates/s3select-api", version = "0.0.5" } @@ -115,7 +114,6 @@ rustfs-targets = { path = "crates/targets", version = "0.0.5" } rustfs-utils = { path = "crates/utils", version = "0.0.5" } rustfs-workers = { path = "crates/workers", version = "0.0.5" } rustfs-zip = { path = "./crates/zip", version = "0.0.5" } -rustfs-protocols = { path = "crates/protocols", version = "0.0.5" } # Async Runtime and Networking async-channel = "2.5.0" @@ -135,7 +133,7 @@ http-body = "1.0.1" http-body-util = "0.1.3" reqwest = { version = "0.13.2", default-features = false, features = ["rustls", "charset", "http2", "system-proxy", "stream", "json", "blocking", "query", "form"] } socket2 = { version = "0.6.3", features = ["all"] } -tokio = { version = "1.52.0", features = ["fs", "rt-multi-thread", "io-uring"] } +tokio = { version = "1.52.1", features = ["fs", "rt-multi-thread"] } tokio-rustls = { version = "0.26.4", default-features = false, features = ["logging", "tls12", "aws-lc-rs"] } tokio-stream = { version = "0.1.18" } tokio-test = "0.4.5" @@ -154,18 +152,16 @@ flatbuffers = "25.12.19" form_urlencoded = "1.2.2" prost = "0.14.3" quick-xml = "0.39.2" -rmcp = { version = "1.4.0" } rmp = { version = "0.8.15" } rmp-serde = { version = "1.3.1" } serde = { version = "1.0.228", features = ["derive"] } serde_json = { version = "1.0.149", features = ["raw_value"] } serde_urlencoded = "0.7.1" -schemars = "1.2.1" # Cryptography and Security aes-gcm = { version = "0.11.0-rc.3", features = ["rand_core"] } argon2 = { version = "0.6.0-rc.8" } -blake2 = "0.11.0-rc.5" +blake2 = "0.11.0-rc.6" chacha20poly1305 = { version = "0.11.0-rc.3" } crc-fast = "1.9.0" hmac = { version = "0.13.0" } @@ -202,7 +198,7 @@ base64 = "0.22.1" base64-simd = "0.8.0" brotli = "8.0.2" cfg-if = "1.0.4" -clap = { version = "4.6.0", features = ["derive", "env"] } +clap = { version = "4.6.1", features = ["derive", "env"] } const-str = { version = "1.1.0", features = ["std", "proc"] } convert_case = "0.11.0" criterion = { version = "0.8", features = ["html_reports"] } @@ -210,7 +206,7 @@ crossbeam-queue = "0.3.12" crossbeam-channel = "0.5.15" crossbeam-deque = "0.8.6" crossbeam-utils = "0.8.21" -datafusion = "53.0.0" +datafusion = "53.1.0" derive_builder = "0.20.2" enumset = "1.1.10" faster-hex = "0.10.0" @@ -250,7 +246,7 @@ rayon = "1.12.0" reed-solomon-erasure = { version = "6.0", default-features = false, features = ["std", "simd-accel"] } reed-solomon-simd = "3.1.0" regex = { version = "1.12.3" } -rumqttc = { package = "rumqttc-next", version = "0.29.0", features = ["websocket"] } +rumqttc = { package = "rumqttc-next", version = "0.30.0", features = ["websocket"] } rustix = { version = "1.1.4", features = ["fs"] } rust-embed = { version = "8.11.0" } rustc-hash = { version = "2.1.2" } @@ -277,7 +273,7 @@ tracing-subscriber = { version = "0.3.23", features = ["env-filter", "time"] } transform-stream = "0.3.1" url = "2.5.8" urlencoding = "2.1.3" -uuid = { version = "1.23.0", features = ["v4", "fast-rng", "macro-diagnostics"] } +uuid = { version = "1.23.1", features = ["v4", "fast-rng", "macro-diagnostics"] } vaultrs = { version = "0.8.0" } walkdir = "2.5.0" wildmatch = { version = "2.6.1", features = ["serde"] } @@ -320,7 +316,7 @@ jemalloc_pprof = { version = "0.8.2", features = ["symbolize", "flamegraph"] } pprof = { package = "pprof-pyroscope-fork", version = "0.1500.3", features = ["flamegraph", "protobuf-codec"] } [workspace.metadata.cargo-shear] -ignored = ["rustfs", "rustfs-mcp"] +ignored = ["rustfs"] [profile.release] opt-level = 3 diff --git a/crates/concurrency/Cargo.toml b/crates/concurrency/Cargo.toml index 7ae9eebe8..2acdc6f75 100644 --- a/crates/concurrency/Cargo.toml +++ b/crates/concurrency/Cargo.toml @@ -26,7 +26,7 @@ thiserror = { workspace = true } tracing = { workspace = true } [dev-dependencies] -tokio = { workspace = true, features = ["full"] } +tokio = { workspace = true, features = ["test-util","macros","rt-multi-thread"] } [features] default = ["timeout", "lock", "deadlock", "backpressure", "scheduler"] diff --git a/crates/ecstore/Cargo.toml b/crates/ecstore/Cargo.toml index 5b4888dfc..76d3e38c0 100644 --- a/crates/ecstore/Cargo.toml +++ b/crates/ecstore/Cargo.toml @@ -86,7 +86,7 @@ hyper.workspace = true hyper-util.workspace = true hyper-rustls.workspace = true rustls.workspace = true -tokio = { workspace = true, features = ["io-util", "sync", "signal"] } +tokio = { workspace = true, features = ["io-util", "sync", "signal","io-uring"] } tonic.workspace = true xxhash-rust = { workspace = true, features = ["xxh64", "xxh3"] } tower.workspace = true diff --git a/crates/heal/Cargo.toml b/crates/heal/Cargo.toml index d22bb9eb4..21b85c7cb 100644 --- a/crates/heal/Cargo.toml +++ b/crates/heal/Cargo.toml @@ -32,7 +32,7 @@ rustfs-ecstore = { workspace = true } rustfs-common = { workspace = true } rustfs-madmin = { workspace = true } rustfs-utils = { workspace = true } -tokio = { workspace = true, features = ["full"] } +tokio = { workspace = true, features = ["sync","io-util","time","macros"] } tokio-util = { workspace = true } tracing = { workspace = true } serde = { workspace = true, features = ["derive"] } @@ -52,6 +52,7 @@ tempfile = { workspace = true } walkdir = { workspace = true } http = { workspace = true } temp-env = { workspace = true } +tokio = { workspace = true, features = ["test-util","fs"] } [lib] doctest = false diff --git a/crates/io-core/Cargo.toml b/crates/io-core/Cargo.toml index 81f932532..e85c7e92e 100644 --- a/crates/io-core/Cargo.toml +++ b/crates/io-core/Cargo.toml @@ -30,7 +30,7 @@ workspace = true [dependencies] bytes = { workspace = true } thiserror = { workspace = true } -tokio = { workspace = true, features = ["io-util", "fs", "rt", "sync"] } +tokio = { workspace = true, features = ["io-util", "fs", "rt", "sync","io-uring"] } memmap2 = { workspace = true } rustfs-io-metrics = { workspace = true } diff --git a/crates/io-metrics/Cargo.toml b/crates/io-metrics/Cargo.toml index 470d321a1..8173fb999 100644 --- a/crates/io-metrics/Cargo.toml +++ b/crates/io-metrics/Cargo.toml @@ -28,8 +28,11 @@ categories = ["development-tools", "filesystem"] metrics = { workspace = true } num_cpus = { workspace = true } thiserror = { workspace = true } -tokio = { workspace = true, features = ["sync", "full"] } +tokio = { workspace = true, features = ["sync","rt"] } tracing = { workspace = true } +[dev-dependencies] +tokio = { workspace = true, features = ["test-util","rt","macros"] } + [lints] workspace = true diff --git a/crates/kms/Cargo.toml b/crates/kms/Cargo.toml index f3afe9f33..b786ff6b0 100644 --- a/crates/kms/Cargo.toml +++ b/crates/kms/Cargo.toml @@ -30,7 +30,7 @@ workspace = true [dependencies] # Core dependencies async-trait = { workspace = true } -tokio = { workspace = true, features = ["full"] } +tokio = { workspace = true, features = ["full","io-uring"] } uuid = { workspace = true, features = ["serde"] } jiff = { workspace = true } serde = { workspace = true, features = ["derive"] } diff --git a/crates/mcp/Cargo.toml b/crates/mcp/Cargo.toml deleted file mode 100644 index 676309762..000000000 --- a/crates/mcp/Cargo.toml +++ /dev/null @@ -1,63 +0,0 @@ -# Copyright 2024 RustFS Team -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -[package] -name = "rustfs-mcp" -version.workspace = true -edition.workspace = true -license.workspace = true -repository.workspace = true -rust-version.workspace = true -homepage.workspace = true -description = "RustFS MCP (Model Context Protocol) Server" -keywords = ["mcp", "s3", "aws", "rustfs", "server"] -categories = ["development-tools", "web-programming"] -documentation = "https://docs.rs/rustfs-mcp/latest/rustfs_mcp/" - -[[bin]] -name = "rustfs-mcp" -path = "src/main.rs" - -[dependencies] -# AWS SDK for S3 operations -aws-sdk-s3.workspace = true -aws-smithy-http-client.workspace = true - -# Async runtime and utilities -tokio = { workspace = true, features = ["io-std", "io-util", "macros", "signal"] } - -# MCP SDK with macros support -rmcp = { workspace = true, features = ["server", "transport-io", "macros"] } - -# Command line argument parsing -clap = { workspace = true, features = ["derive", "env"] } - -# Serialization (still needed for S3 data structures) -serde.workspace = true -serde_json.workspace = true -schemars = { workspace = true } - -# Error handling -anyhow.workspace = true - -# Logging -tracing.workspace = true -tracing-subscriber.workspace = true - -# File handling and MIME type detection -mime_guess = { workspace = true } - -[lib] -doctest = false -# Testing framework and utilities diff --git a/crates/mcp/Dockerfile b/crates/mcp/Dockerfile deleted file mode 100644 index d9c95e947..000000000 --- a/crates/mcp/Dockerfile +++ /dev/null @@ -1,17 +0,0 @@ -FROM rust:1.88 AS builder - -WORKDIR /build - -COPY . . - -RUN cargo build --release -p rustfs-mcp - -FROM debian:bookworm-slim - -WORKDIR /app - -COPY --from=builder /build/target/release/rustfs-mcp /app/ - -RUN apt-get update && apt-get install -y ca-certificates && update-ca-certificates - -ENTRYPOINT ["/app/rustfs-mcp"] diff --git a/crates/mcp/README.md b/crates/mcp/README.md deleted file mode 100644 index b32182375..000000000 --- a/crates/mcp/README.md +++ /dev/null @@ -1,261 +0,0 @@ -[![RustFS](https://rustfs.com/images/rustfs-github.png)](https://rustfs.com) - -# RustFS MCP Server - Model Context Protocol - -

- High-performance MCP server providing S3-compatible object storage operations for AI/LLM integration -

- -

- CI - 📖 Documentation - 🐛 Bug Reports - 💬 Discussions -

- ---- - -## 📖 Overview - -**RustFS MCP Server** is a high-performance [Model Context Protocol (MCP)](https://spec.modelcontextprotocol.org) server that provides AI/LLM tools with seamless access to S3-compatible object storage operations. Built with Rust for maximum performance and safety, it enables AI assistants like Claude Desktop to interact with cloud storage through a standardized protocol. - -### What is MCP? - -The Model Context Protocol is an open standard that enables secure, controlled connections between AI applications and external systems. This server acts as a bridge between AI tools and S3-compatible storage services, providing structured access to file operations while maintaining security and observability. - -## ✨ Features - -### Supported S3 Operations - -- **List Buckets**: List all accessible S3 buckets -- **List Objects**: Browse bucket contents with optional prefix filtering -- **Upload Files**: Upload local files with automatic MIME type detection and cache control -- **Get Objects**: Retrieve objects from S3 storage with read or download modes - -## 🔧 Installation - -### Prerequisites - -- Rust 1.70+ (for building from source) -- AWS credentials configured (via environment variables, AWS CLI, or IAM roles) -- Access to S3-compatible storage service - -### Build from Source - -```bash -# Clone the repository -git clone https://github.com/rustfs/rustfs.git -cd rustfs - -# Build the MCP server -cargo build --release -p rustfs-mcp - -# The binary will be available at -./target/release/rustfs-mcp -``` - -## ⚙️ Configuration - -### Environment Variables - -```bash -# AWS Credentials (required) -export AWS_ACCESS_KEY_ID=your_access_key -export AWS_SECRET_ACCESS_KEY=your_secret_key -export AWS_REGION=us-east-1 # Optional, defaults to us-east-1 - -# Optional: Custom S3 endpoint (for MinIO, etc.) -export AWS_ENDPOINT_URL=http://localhost:9000 - -# Logging level (optional) -export RUST_LOG=info -``` - -### Command Line Options - -```bash -rustfs-mcp --help -``` - -The server supports various command-line options for customizing behavior: - -- `--access-key-id`: AWS Access Key ID for S3 authentication -- `--secret-access-key`: AWS Secret Access Key for S3 authentication -- `--region`: AWS region to use for S3 operations (default: us-east-1) -- `--endpoint-url`: Custom S3 endpoint URL (for MinIO, LocalStack, etc.) -- `--log-level`: Log level configuration (default: rustfs_mcp_server=info) - -## 🚀 Usage - -### Starting the Server - -```bash -# Start the MCP server -rustfs-mcp - -# Or with custom options -rustfs-mcp --log-level debug --region us-west-2 -``` - -### Integration with chat client - -#### Option 1: Using Command Line Arguments - -```json -{ - "mcpServers": { - "rustfs-mcp": { - "command": "/path/to/rustfs-mcp", - "args": [ - "--access-key-id", "your_access_key", - "--secret-access-key", "your_secret_key", - "--region", "us-west-2", - "--log-level", "info" - ] - } - } -} -``` - -#### Option 2: Using Environment Variables - -```json -{ - "mcpServers": { - "rustfs-mcp": { - "command": "/path/to/rustfs-mcp", - "env": { - "AWS_ACCESS_KEY_ID": "your_access_key", - "AWS_SECRET_ACCESS_KEY": "your_secret_key", - "AWS_REGION": "us-east-1" - } - } - } -} -``` - -### Using MCP with Docker - -#### Docker image build - -Using MCP with docker will simply the usage of rustfs mcp. Building the docker image with below command: - -``` -docker build -f Dockerfile -t rustfs/rustfs-mcp ../../ -``` - -Alternatively, if you want to build the image from the rustfs codebase root directory,run the command: - -``` -docker build -f crates/mcp/Dockerfile -t rustfs/rustfs-mcp . -``` - -#### IDE Configuration - -Adding the following content in IDE MCP settings: - -``` -{ - "mcpServers": { - "rustfs-mcp": { - "command": "docker", - "args": [ - "run", - "--rm", - "-i", - "-e", - "AWS_ACCESS_KEY_ID", - "-e", - "AWS_SECRET_ACCESS_KEY", - "-e", - "AWS_REGION", - "-e", - "AWS_ENDPOINT_URL", - "rustfs/rustfs-mcp" - ], - "env": { - "AWS_ACCESS_KEY_ID": "rustfs_access_key", - "AWS_SECRET_ACCESS_KEY": "rustfs_secret_key", - "AWS_REGION": "cn-east-1", - "AWS_ENDPOINT_URL": "rustfs_instance_url" - } - } - } -} -``` - -If success, MCP configure page will show the [available tools](#️-available-tools). - -## 🛠️ Available Tools - -The MCP server exposes the following tools that AI assistants can use: - -### `list_buckets` - -List all S3 buckets accessible with the configured credentials. - -**Parameters:** None - -### `list_objects` - -List objects in an S3 bucket with optional prefix filtering. - -**Parameters:** - -- `bucket_name` (string): Name of the S3 bucket -- `prefix` (string, optional): Prefix to filter objects - -### `upload_file` - -Upload a local file to S3 with automatic MIME type detection. - -**Parameters:** - -- `local_file_path` (string): Path to the local file -- `bucket_name` (string): Target S3 bucket -- `object_key` (string): S3 object key (destination path) -- `content_type` (string, optional): Content type (auto-detected if not provided) -- `storage_class` (string, optional): S3 storage class -- `cache_control` (string, optional): Cache control header - -### `get_object` - -Retrieve an object from S3 with two operation modes: read content directly or download to a file. - -**Parameters:** - -- `bucket_name` (string): Source S3 bucket -- `object_key` (string): S3 object key -- `version_id` (string, optional): Version ID for versioned objects -- `mode` (string, optional): Operation mode - "read" (default) returns content directly, "download" saves to local file -- `local_path` (string, optional): Local file path (required when mode is "download") -- `max_content_size` (number, optional): Maximum content size in bytes for read mode (default: 1MB) - -### `create_bucket` - -Create a new S3 bucket with the specified name. - -**Parameters:** - -- `bucket_name` (string): Source S3 bucket. - -### `delete_bucket` - -Delete the specified S3 bucket. If the bucket is not empty, the deletion will fail. You should delete all objects and objects inside them before calling this method.**WARNING: This operation will permanently delete the bucket and all objects within it!** - -- `bucket_name` (string): Source S3 bucket. - -## Architecture - -The MCP server is built with a modular architecture: - -``` -rustfs-mcp/ -├── src/ -│ ├── main.rs # Entry point, CLI parsing, and server initialization -│ ├── server.rs # MCP server implementation and tool handlers -│ ├── s3_client.rs # S3 client wrapper with async operations -│ ├── config.rs # Configuration management and CLI options -│ └── lib.rs # Library exports and public API -└── Cargo.toml # Dependencies, metadata, and binary configuration -``` diff --git a/crates/mcp/src/config.rs b/crates/mcp/src/config.rs deleted file mode 100644 index 8ea795ffb..000000000 --- a/crates/mcp/src/config.rs +++ /dev/null @@ -1,224 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use anyhow::Result; -use clap::Parser; -use tracing::info; - -/// Configuration for RustFS MCP Server -#[derive(Parser, Debug, Clone)] -#[command( - name = "rustfs-mcp-server", - about = "RustFS MCP (Model Context Protocol) Server for S3 operations", - version, - long_about = r#" -RustFS MCP Server - Model Context Protocol server for S3 operations - -This server provides S3 operations through the Model Context Protocol (MCP), -allowing AI assistants to interact with S3-compatible storage systems. - -ENVIRONMENT VARIABLES: - All command-line options can also be set via environment variables. - Command-line arguments take precedence over environment variables. - -EXAMPLES: - # Using command-line arguments - rustfs-mcp-server --access-key-id your_key --secret-access-key your_secret - - # Using environment variables - export AWS_ACCESS_KEY_ID=your_key - export AWS_SECRET_ACCESS_KEY=your_secret - rustfs-mcp-server - - # Mixed usage (command-line overrides environment) - export AWS_REGION=us-east-1 - rustfs-mcp-server --access-key-id mykey --secret-access-key mysecret --endpoint-url http://localhost:9000 -"# -)] -pub struct Config { - /// AWS Access Key ID - #[arg( - long = "access-key-id", - env = "AWS_ACCESS_KEY_ID", - help = "AWS Access Key ID for S3 authentication" - )] - pub access_key_id: Option, - - /// AWS Secret Access Key - #[arg( - long = "secret-access-key", - env = "AWS_SECRET_ACCESS_KEY", - help = "AWS Secret Access Key for S3 authentication" - )] - pub secret_access_key: Option, - - /// AWS Region - #[arg( - long = "region", - env = "AWS_REGION", - default_value = "us-east-1", - help = "AWS region to use for S3 operations" - )] - pub region: String, - - /// Custom S3 endpoint URL - #[arg( - long = "endpoint-url", - env = "AWS_ENDPOINT_URL", - help = "Custom S3 endpoint URL (for MinIO, LocalStack, etc.)" - )] - pub endpoint_url: Option, - - /// Log level - #[arg( - long = "log-level", - env = "RUST_LOG", - default_value = "rustfs_mcp_server=info", - help = "Log level configuration" - )] - pub log_level: String, - - /// Force path-style addressing - #[arg( - long = "force-path-style", - help = "Force path-style S3 addressing (automatically enabled for custom endpoints)" - )] - pub force_path_style: bool, -} - -impl Config { - pub fn new() -> Self { - Config::parse() - } - - pub fn validate(&self) -> Result<()> { - if self.access_key_id.is_none() { - anyhow::bail!("AWS Access Key ID is required. Set via --access-key-id or AWS_ACCESS_KEY_ID environment variable"); - } - - if self.secret_access_key.is_none() { - anyhow::bail!( - "AWS Secret Access Key is required. Set via --secret-access-key or AWS_SECRET_ACCESS_KEY environment variable" - ); - } - - Ok(()) - } - - pub fn access_key_id(&self) -> &str { - self.access_key_id.as_ref().expect("Access key ID should be validated") - } - - pub fn secret_access_key(&self) -> &str { - self.secret_access_key - .as_ref() - .expect("Secret access key should be validated") - } - - pub fn log_configuration(&self) { - let access_key_display = self - .access_key_id - .as_ref() - .map(|key| { - if key.len() > 8 { - format!("{}...{}", &key[..4], &key[key.len() - 4..]) - } else { - "*".repeat(key.len()) - } - }) - .unwrap_or_else(|| "Not set".to_string()); - - let endpoint_display = self - .endpoint_url - .as_ref() - .map(|url| format!("Custom endpoint: {url}")) - .unwrap_or_else(|| "Default AWS endpoints".to_string()); - - info!("Configuration:"); - info!(" AWS Region: {}", self.region); - info!(" AWS Access Key ID: {}", access_key_display); - info!(" AWS Secret Access Key: [HIDDEN]"); - info!(" S3 Endpoint: {}", endpoint_display); - info!(" Force Path Style: {}", self.force_path_style); - info!(" Log Level: {}", self.log_level); - } -} - -impl Default for Config { - fn default() -> Self { - Config { - access_key_id: None, - secret_access_key: None, - region: "us-east-1".to_string(), - endpoint_url: None, - log_level: "rustfs_mcp_server=info".to_string(), - force_path_style: false, - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_config_validation_success() { - let config = Config { - access_key_id: Some("test_key".to_string()), - secret_access_key: Some("test_secret".to_string()), - ..Config::default() - }; - - assert!(config.validate().is_ok()); - assert_eq!(config.access_key_id(), "test_key"); - assert_eq!(config.secret_access_key(), "test_secret"); - } - - #[test] - fn test_config_validation_missing_access_key() { - let config = Config { - access_key_id: None, - secret_access_key: Some("test_secret".to_string()), - ..Config::default() - }; - - let result = config.validate(); - assert!(result.is_err()); - assert!(result.unwrap_err().to_string().contains("Access Key ID")); - } - - #[test] - fn test_config_validation_missing_secret_key() { - let config = Config { - access_key_id: Some("test_key".to_string()), - secret_access_key: None, - ..Config::default() - }; - - let result = config.validate(); - assert!(result.is_err()); - assert!(result.unwrap_err().to_string().contains("Secret Access Key")); - } - - #[test] - fn test_config_default() { - let config = Config::default(); - assert_eq!(config.region, "us-east-1"); - assert_eq!(config.log_level, "rustfs_mcp_server=info"); - assert!(!config.force_path_style); - assert!(config.access_key_id.is_none()); - assert!(config.secret_access_key.is_none()); - assert!(config.endpoint_url.is_none()); - } -} diff --git a/crates/mcp/src/lib.rs b/crates/mcp/src/lib.rs deleted file mode 100644 index 448622859..000000000 --- a/crates/mcp/src/lib.rs +++ /dev/null @@ -1,97 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -pub mod config; -pub mod s3_client; -pub mod server; - -pub use config::Config; -pub use s3_client::{BucketInfo, S3Client}; -pub use server::RustfsMcpServer; - -use anyhow::{Context, Result}; -use rmcp::ServiceExt; -use tokio::io::{stdin, stdout}; -use tracing::info; - -/// Run the MCP server with the provided configuration -pub async fn run_server_with_config(config: Config) -> Result<()> { - info!("Starting RustFS MCP Server with provided configuration"); - - config.validate().context("Configuration validation failed")?; - - let server = RustfsMcpServer::new(config).await?; - - info!("Running MCP server with stdio transport"); - - // Run the server with stdio - server - .serve((stdin(), stdout())) - .await - .context("Failed to serve MCP server")? - .waiting() - .await - .context("Error while waiting for server shutdown")?; - - Ok(()) -} - -/// Run the MCP server with default configuration (from environment variables) -pub async fn run_server() -> Result<()> { - info!("Starting RustFS MCP Server with default configuration"); - - let config = Config::default(); - run_server_with_config(config).await -} - -/// Validate environment configuration (legacy function for backward compatibility) -pub fn validate_environment() -> Result<()> { - use std::env; - - if env::var("AWS_ACCESS_KEY_ID").is_err() { - anyhow::bail!("AWS_ACCESS_KEY_ID environment variable is required"); - } - - if env::var("AWS_SECRET_ACCESS_KEY").is_err() { - anyhow::bail!("AWS_SECRET_ACCESS_KEY environment variable is required"); - } - - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_config_creation() { - let config = Config { - access_key_id: Some("test_key".to_string()), - secret_access_key: Some("test_secret".to_string()), - ..Config::default() - }; - - assert!(config.validate().is_ok()); - assert_eq!(config.access_key_id(), "test_key"); - assert_eq!(config.secret_access_key(), "test_secret"); - } - - #[tokio::test] - async fn test_run_server_with_invalid_config() { - let config = Config::default(); - - let result = run_server_with_config(config).await; - assert!(result.is_err()); - } -} diff --git a/crates/mcp/src/main.rs b/crates/mcp/src/main.rs deleted file mode 100644 index 73e638ff5..000000000 --- a/crates/mcp/src/main.rs +++ /dev/null @@ -1,104 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use anyhow::{Context, Result}; -use clap::Parser; -use rmcp::ServiceExt; -use rustfs_mcp::{Config, RustfsMcpServer}; -use std::env; -use tokio::io::{stdin, stdout}; -use tracing::{Level, error, info}; -use tracing_subscriber::{EnvFilter, FmtSubscriber}; - -#[tokio::main] -async fn main() -> Result<()> { - let config = Config::parse(); - - init_tracing(&config)?; - - info!("Starting RustFS MCP Server v{}", env!("CARGO_PKG_VERSION")); - - if let Err(e) = config.validate() { - error!("Configuration validation failed: {}", e); - print_usage_help(); - std::process::exit(1); - } - - config.log_configuration(); - - if let Err(e) = run_server(config).await { - error!("Server error: {}", e); - std::process::exit(1); - } - - info!("RustFS MCP Server shutdown complete"); - Ok(()) -} - -async fn run_server(config: Config) -> Result<()> { - info!("Initializing RustFS MCP Server"); - - let server = RustfsMcpServer::new(config).await?; - - info!("Starting MCP server with stdio transport"); - - server - .serve((stdin(), stdout())) - .await - .context("Failed to serve MCP server")? - .waiting() - .await - .context("Error while waiting for server shutdown")?; - - Ok(()) -} - -fn init_tracing(config: &Config) -> Result<()> { - let filter = EnvFilter::try_from_default_env() - .or_else(|_| EnvFilter::try_new(&config.log_level)) - .context("Failed to create log filter")?; - - let subscriber = FmtSubscriber::builder() - .with_max_level(Level::TRACE) - .with_env_filter(filter) - .with_target(false) - .with_thread_ids(false) - .with_thread_names(false) - .with_writer(std::io::stderr) // Force logs to stderr to avoid interfering with MCP protocol on stdout - .finish(); - - tracing::subscriber::set_global_default(subscriber).context("Failed to set global tracing subscriber")?; - - Ok(()) -} - -fn print_usage_help() { - eprintln!(); - eprintln!("RustFS MCP Server - Model Context Protocol server for S3 operations"); - eprintln!(); - eprintln!("For more help, run: rustfs-mcp --help"); - eprintln!(); - eprintln!("QUICK START:"); - eprintln!(" # Using command-line arguments"); - eprintln!(" rustfs-mcp --access-key-id YOUR_KEY --secret-access-key YOUR_SECRET"); - eprintln!(); - eprintln!(" # Using environment variables"); - eprintln!(" export AWS_ACCESS_KEY_ID=YOUR_KEY"); - eprintln!(" export AWS_SECRET_ACCESS_KEY=YOUR_SECRET"); - eprintln!(" rustfs-mcp"); - eprintln!(); - eprintln!(" # For local development with RustFS"); - eprintln!(" rustfs-mcp --access-key-id minioadmin --secret-access-key minioadmin --endpoint-url http://localhost:9000"); - eprintln!(); -} diff --git a/crates/mcp/src/s3_client.rs b/crates/mcp/src/s3_client.rs deleted file mode 100644 index ac7ec0898..000000000 --- a/crates/mcp/src/s3_client.rs +++ /dev/null @@ -1,835 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use anyhow::{Context, Result}; -use aws_sdk_s3::config::{Credentials, Region}; -use aws_sdk_s3::primitives::ByteStream; -use aws_sdk_s3::{Client, Config as S3Config}; -use aws_smithy_http_client::Builder as SmithyHttpClientBuilder; -use serde::{Deserialize, Serialize}; -use std::path::Path; -use tokio::io::AsyncWriteExt; -use tracing::{debug, info}; - -use crate::config::Config; - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct BucketInfo { - pub name: String, - pub creation_date: Option, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ObjectInfo { - pub key: String, - pub size: Option, - pub last_modified: Option, - pub etag: Option, - pub storage_class: Option, -} - -#[derive(Debug, Clone, Default)] -pub struct ListObjectsOptions { - pub prefix: Option, - pub delimiter: Option, - pub max_keys: Option, - pub continuation_token: Option, - pub start_after: Option, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ListObjectsResult { - pub objects: Vec, - pub common_prefixes: Vec, - pub is_truncated: bool, - pub next_continuation_token: Option, - pub max_keys: Option, - pub key_count: i32, -} - -#[derive(Debug, Clone, Default)] -pub struct UploadFileOptions { - pub content_type: Option, - pub metadata: Option>, - pub storage_class: Option, - pub server_side_encryption: Option, - pub cache_control: Option, - pub content_disposition: Option, - pub content_encoding: Option, - pub content_language: Option, -} - -#[derive(Debug, Clone, Default)] -pub struct GetObjectOptions { - pub version_id: Option, - pub range: Option, - pub if_modified_since: Option, - pub if_unmodified_since: Option, - pub max_content_size: Option, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub enum DetectedFileType { - Text, - NonText(String), // mime type for non-text files -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct GetObjectResult { - pub bucket: String, - pub key: String, - pub content_type: String, - pub content_length: u64, - pub last_modified: Option, - pub etag: Option, - pub version_id: Option, - pub detected_type: DetectedFileType, - pub content: Option>, // Raw content bytes - pub text_content: Option, // UTF-8 decoded content for text files -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct UploadResult { - pub bucket: String, - pub key: String, - pub etag: String, - pub location: String, - pub version_id: Option, - pub file_size: u64, - pub content_type: String, - pub upload_id: Option, -} -#[derive(Debug, Clone)] -pub struct S3Client { - client: Client, -} - -impl S3Client { - pub async fn new(config: &Config) -> Result { - info!("Initializing S3 client from configuration"); - - let access_key = config.access_key_id(); - let secret_key = config.secret_access_key(); - - debug!("Using AWS region: {}", config.region); - if let Some(ref endpoint) = config.endpoint_url { - debug!("Using custom endpoint: {}", endpoint); - } - - let credentials = Credentials::new(access_key, secret_key, None, None, "rustfs-mcp-server"); - - let mut config_builder = S3Config::builder() - .credentials_provider(credentials) - .region(Region::new(config.region.clone())) - .behavior_version(aws_sdk_s3::config::BehaviorVersion::latest()); - - if config - .endpoint_url - .as_deref() - .is_some_and(|endpoint| endpoint.starts_with("http://")) - { - config_builder = config_builder.http_client(SmithyHttpClientBuilder::new().build_http()); - } - - // Set force path style if custom endpoint or explicitly requested - let should_force_path_style = config.endpoint_url.is_some() || config.force_path_style; - if should_force_path_style { - config_builder = config_builder.force_path_style(true); - } - - if let Some(endpoint) = &config.endpoint_url { - config_builder = config_builder.endpoint_url(endpoint); - } - - let s3_config = config_builder.build(); - let client = Client::from_conf(s3_config); - - info!("S3 client initialized successfully"); - - Ok(Self { client }) - } - - pub async fn create_bucket(&self, bucket_name: &str) -> Result { - info!("Creating S3 bucket: {}", bucket_name); - - self.client - .create_bucket() - .bucket(bucket_name) - .send() - .await - .context(format!("Failed to create S3 bucket: {bucket_name}"))?; - - info!("Bucket '{}' created successfully", bucket_name); - Ok(BucketInfo { - name: bucket_name.to_string(), - creation_date: None, // Creation date not returned by create_bucket - }) - } - - pub async fn delete_bucket(&self, bucket_name: &str) -> Result<()> { - info!("Deleting S3 bucket: {}", bucket_name); - self.client - .delete_bucket() - .bucket(bucket_name) - .send() - .await - .context(format!("Failed to delete S3 bucket: {bucket_name}"))?; - - info!("Bucket '{}' deleted successfully", bucket_name); - Ok(()) - } - - pub async fn list_buckets(&self) -> Result> { - debug!("Listing S3 buckets"); - - let response = self.client.list_buckets().send().await.context("Failed to list S3 buckets")?; - - let buckets: Vec = response - .buckets() - .iter() - .map(|bucket| { - let name = bucket.name().unwrap_or("unknown").to_string(); - let creation_date = bucket - .creation_date() - .map(|dt| dt.fmt(aws_sdk_s3::primitives::DateTimeFormat::DateTime).unwrap()); - - BucketInfo { name, creation_date } - }) - .collect(); - - debug!("Found {} buckets", buckets.len()); - Ok(buckets) - } - - pub async fn list_objects_v2(&self, bucket_name: &str, options: ListObjectsOptions) -> Result { - debug!("Listing objects in bucket '{}' with options: {:?}", bucket_name, options); - - let mut request = self.client.list_objects_v2().bucket(bucket_name); - - if let Some(prefix) = options.prefix { - request = request.prefix(prefix); - } - - if let Some(delimiter) = options.delimiter { - request = request.delimiter(delimiter); - } - - if let Some(max_keys) = options.max_keys { - request = request.max_keys(max_keys); - } - - if let Some(continuation_token) = options.continuation_token { - request = request.continuation_token(continuation_token); - } - - if let Some(start_after) = options.start_after { - request = request.start_after(start_after); - } - - let response = request - .send() - .await - .context(format!("Failed to list objects in bucket '{bucket_name}'"))?; - - let objects: Vec = response - .contents() - .iter() - .map(|obj| { - let key = obj.key().unwrap_or("unknown").to_string(); - let size = obj.size(); - let last_modified = obj - .last_modified() - .map(|dt| dt.fmt(aws_sdk_s3::primitives::DateTimeFormat::DateTime).unwrap()); - let etag = obj.e_tag().map(|e| e.to_string()); - let storage_class = obj.storage_class().map(|sc| sc.as_str().to_string()); - - ObjectInfo { - key, - size, - last_modified, - etag, - storage_class, - } - }) - .collect(); - - let common_prefixes: Vec = response - .common_prefixes() - .iter() - .filter_map(|cp| cp.prefix()) - .map(|p| p.to_string()) - .collect(); - - let result = ListObjectsResult { - objects, - common_prefixes, - is_truncated: response.is_truncated().unwrap_or(false), - next_continuation_token: response.next_continuation_token().map(|t| t.to_string()), - max_keys: response.max_keys(), - key_count: response.key_count().unwrap_or(0), - }; - - debug!( - "Found {} objects and {} common prefixes in bucket '{}'", - result.objects.len(), - result.common_prefixes.len(), - bucket_name - ); - - Ok(result) - } - - pub async fn upload_file( - &self, - local_path: &str, - bucket_name: &str, - object_key: &str, - options: UploadFileOptions, - ) -> Result { - info!("Starting file upload: '{}' -> s3://{}/{}", local_path, bucket_name, object_key); - - let path = Path::new(local_path); - let canonical_path = path - .canonicalize() - .context(format!("Failed to resolve file path: {local_path}"))?; - - if !canonical_path.exists() { - anyhow::bail!("File does not exist: {local_path}"); - } - - if !canonical_path.is_file() { - anyhow::bail!("Path is not a file: {local_path}"); - } - - let metadata = tokio::fs::metadata(&canonical_path) - .await - .context(format!("Failed to read file metadata: {local_path}"))?; - - let file_size = metadata.len(); - debug!("File size: {file_size} bytes"); - - let content_type = options.content_type.unwrap_or_else(|| { - let detected = mime_guess::from_path(&canonical_path).first_or_octet_stream().to_string(); - debug!("Auto-detected content type: {detected}"); - detected - }); - - let file_content = tokio::fs::read(&canonical_path) - .await - .context(format!("Failed to read file content: {local_path}"))?; - - let byte_stream = ByteStream::from(file_content); - - let mut request = self - .client - .put_object() - .bucket(bucket_name) - .key(object_key) - .body(byte_stream) - .content_type(&content_type) - .content_length(file_size as i64); - - if let Some(storage_class) = &options.storage_class { - request = request.storage_class(storage_class.as_str().into()); - } - - if let Some(cache_control) = &options.cache_control { - request = request.cache_control(cache_control); - } - - if let Some(content_disposition) = &options.content_disposition { - request = request.content_disposition(content_disposition); - } - - if let Some(content_encoding) = &options.content_encoding { - request = request.content_encoding(content_encoding); - } - - if let Some(content_language) = &options.content_language { - request = request.content_language(content_language); - } - - if let Some(sse) = &options.server_side_encryption { - request = request.server_side_encryption(sse.as_str().into()); - } - - if let Some(metadata_map) = &options.metadata { - for (key, value) in metadata_map { - request = request.metadata(key, value); - } - } - - debug!("Executing S3 put_object request"); - let response = request - .send() - .await - .context(format!("Failed to upload file to s3://{bucket_name}/{object_key}"))?; - - let etag = response.e_tag().unwrap_or("unknown").to_string(); - let version_id = response.version_id().map(|v| v.to_string()); - - let location = format!("s3://{bucket_name}/{object_key}"); - - let upload_result = UploadResult { - bucket: bucket_name.to_string(), - key: object_key.to_string(), - etag, - location, - version_id, - file_size, - content_type, - upload_id: None, - }; - - info!( - "File upload completed successfully: {} bytes uploaded to s3://{}/{}", - file_size, bucket_name, object_key - ); - - Ok(upload_result) - } - - pub async fn get_object(&self, bucket_name: &str, object_key: &str, options: GetObjectOptions) -> Result { - info!("Getting object: s3://{}/{}", bucket_name, object_key); - - let mut request = self.client.get_object().bucket(bucket_name).key(object_key); - - if let Some(version_id) = &options.version_id { - request = request.version_id(version_id); - } - - if let Some(range) = &options.range { - request = request.range(range); - } - - if let Some(if_modified_since) = &options.if_modified_since { - request = request.if_modified_since( - aws_sdk_s3::primitives::DateTime::from_str(if_modified_since, aws_sdk_s3::primitives::DateTimeFormat::DateTime) - .context("Failed to parse if_modified_since date")?, - ); - } - - debug!("Executing S3 get_object request"); - let response = request - .send() - .await - .context(format!("Failed to get object from s3://{bucket_name}/{object_key}"))?; - - let content_type = response.content_type().unwrap_or("application/octet-stream").to_string(); - let content_length = response.content_length().unwrap_or(0) as u64; - let last_modified = response - .last_modified() - .map(|dt| dt.fmt(aws_sdk_s3::primitives::DateTimeFormat::DateTime).unwrap()); - let etag = response.e_tag().map(|e| e.to_string()); - let version_id = response.version_id().map(|v| v.to_string()); - - let max_size = options.max_content_size.unwrap_or(10 * 1024 * 1024); - let mut content = Vec::new(); - let mut byte_stream = response.body; - let mut total_read = 0; - - while let Some(bytes_result) = byte_stream.try_next().await.context("Failed to read object content")? { - if total_read + bytes_result.len() > max_size { - anyhow::bail!("Object size exceeds maximum allowed size of {max_size} bytes"); - } - content.extend_from_slice(&bytes_result); - total_read += bytes_result.len(); - } - - debug!("Read {} bytes from object", content.len()); - - let detected_type = Self::detect_file_type(Some(&content_type), &content); - debug!("Detected file type: {detected_type:?}"); - - let text_content = match &detected_type { - DetectedFileType::Text => match std::str::from_utf8(&content) { - Ok(text) => Some(text.to_string()), - Err(_) => { - debug!("Failed to decode content as UTF-8, treating as binary"); - None - } - }, - _ => None, - }; - - let result = GetObjectResult { - bucket: bucket_name.to_string(), - key: object_key.to_string(), - content_type, - content_length, - last_modified, - etag, - version_id, - detected_type, - content: Some(content), - text_content, - }; - - info!( - "Object retrieved successfully: {} bytes from s3://{}/{}", - result.content_length, bucket_name, object_key - ); - - Ok(result) - } - - fn detect_file_type(content_type: Option<&str>, content_bytes: &[u8]) -> DetectedFileType { - if let Some(ct) = content_type { - let ct_lower = ct.to_lowercase(); - - if ct_lower.starts_with("text/") - || ct_lower == "application/json" - || ct_lower == "application/xml" - || ct_lower == "application/yaml" - || ct_lower == "application/javascript" - || ct_lower == "application/x-yaml" - || ct_lower == "application/x-sh" - || ct_lower == "application/x-shellscript" - || ct_lower.contains("script") - || ct_lower.contains("xml") - || ct_lower.contains("json") - { - return DetectedFileType::Text; - } - - return DetectedFileType::NonText(ct.to_string()); - } - - if content_bytes.len() >= 4 { - match &content_bytes[0..4] { - // PNG: 89 50 4E 47 - [0x89, 0x50, 0x4E, 0x47] => return DetectedFileType::NonText("image/png".to_string()), - // JPEG: FF D8 FF - [0xFF, 0xD8, 0xFF, _] => return DetectedFileType::NonText("image/jpeg".to_string()), - // GIF: 47 49 46 38 - [0x47, 0x49, 0x46, 0x38] => return DetectedFileType::NonText("image/gif".to_string()), - // BMP: 42 4D - [0x42, 0x4D, _, _] => return DetectedFileType::NonText("image/bmp".to_string()), - // RIFF container (WebP/WAV) - [0x52, 0x49, 0x46, 0x46] if content_bytes.len() >= 12 => { - if &content_bytes[8..12] == b"WEBP" { - return DetectedFileType::NonText("image/webp".to_string()); - } else if &content_bytes[8..12] == b"WAVE" { - return DetectedFileType::NonText("audio/wav".to_string()); - } - return DetectedFileType::NonText("application/octet-stream".to_string()); - } - _ => {} - } - } - - // 3. Check if content is valid UTF-8 text as fallback - if std::str::from_utf8(content_bytes).is_ok() { - // Additional heuristics for text detection - let non_printable_count = content_bytes - .iter() - .filter(|&&b| b < 0x20 && b != 0x09 && b != 0x0A && b != 0x0D) // Control chars except tab, LF, CR - .count(); - let total_chars = content_bytes.len(); - - // If less than 5% are non-printable control characters, consider it text - if total_chars > 0 && (non_printable_count as f64 / total_chars as f64) < 0.05 { - return DetectedFileType::Text; - } - } - - // Default to non-text binary - DetectedFileType::NonText("application/octet-stream".to_string()) - } - - pub async fn download_object_to_file( - &self, - bucket_name: &str, - object_key: &str, - local_path: &str, - options: GetObjectOptions, - ) -> Result<(u64, String)> { - info!("Downloading object: s3://{}/{} -> {}", bucket_name, object_key, local_path); - - let mut request = self.client.get_object().bucket(bucket_name).key(object_key); - - if let Some(version_id) = &options.version_id { - request = request.version_id(version_id); - } - - if let Some(range) = &options.range { - request = request.range(range); - } - - if let Some(if_modified_since) = &options.if_modified_since { - request = request.if_modified_since( - aws_sdk_s3::primitives::DateTime::from_str(if_modified_since, aws_sdk_s3::primitives::DateTimeFormat::DateTime) - .context("Failed to parse if_modified_since date")?, - ); - } - - debug!("Executing S3 get_object request for download"); - let response = request - .send() - .await - .context(format!("Failed to get object from s3://{bucket_name}/{object_key}"))?; - - let local_file_path = Path::new(local_path); - - if let Some(parent) = local_file_path.parent() { - tokio::fs::create_dir_all(parent) - .await - .context(format!("Failed to create parent directories for {local_path}"))?; - } - - let mut file = tokio::fs::File::create(local_file_path) - .await - .context(format!("Failed to create local file: {local_path}"))?; - - let mut byte_stream = response.body; - let mut total_bytes = 0u64; - - while let Some(bytes_result) = byte_stream.try_next().await.context("Failed to read object content")? { - file.write_all(&bytes_result) - .await - .context(format!("Failed to write to local file: {local_path}"))?; - total_bytes += bytes_result.len() as u64; - } - - file.flush().await.context("Failed to flush file to disk")?; - - let absolute_path = local_file_path - .canonicalize() - .unwrap_or_else(|_| local_file_path.to_path_buf()) - .to_string_lossy() - .to_string(); - - info!( - "Object downloaded successfully: {} bytes from s3://{}/{} to {}", - total_bytes, bucket_name, object_key, absolute_path - ); - - Ok((total_bytes, absolute_path)) - } - - pub async fn health_check(&self) -> Result<()> { - debug!("Performing S3 health check"); - - self.client.list_buckets().send().await.context("S3 health check failed")?; - - debug!("S3 health check passed"); - Ok(()) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[tokio::test] - #[ignore] // Requires AWS credentials - async fn test_s3_client_creation() { - let config = Config { - access_key_id: Some("test_key".to_string()), - secret_access_key: Some("test_secret".to_string()), - region: "us-east-1".to_string(), - ..Config::default() - }; - - let result = S3Client::new(&config).await; - assert!(result.is_ok()); - } - - #[test] - fn test_bucket_info_serialization() { - let bucket = BucketInfo { - name: "test-bucket".to_string(), - creation_date: Some("2024-01-01T00:00:00Z".to_string()), - }; - - let json = serde_json::to_string(&bucket).unwrap(); - let deserialized: BucketInfo = serde_json::from_str(&json).unwrap(); - - assert_eq!(bucket.name, deserialized.name); - assert_eq!(bucket.creation_date, deserialized.creation_date); - } - - #[test] - fn test_detect_file_type_text_content_type() { - let test_cases = vec![ - ("text/plain", "Hello world"), - ("text/html", ""), - ("application/json", r#"{"key": "value"}"#), - ("application/xml", ""), - ("application/yaml", "key: value"), - ("application/javascript", "console.log('hello');"), - ]; - - for (content_type, content) in test_cases { - let result = S3Client::detect_file_type(Some(content_type), content.as_bytes()); - match result { - DetectedFileType::Text => {} - _ => panic!("Expected Text for content type {content_type}"), - } - } - } - - #[test] - fn test_detect_file_type_non_text_content_type() { - // Test various non-text content types - let test_cases = vec![ - ("image/png", "image/png"), - ("image/jpeg", "image/jpeg"), - ("audio/mp3", "audio/mp3"), - ("video/mp4", "video/mp4"), - ("application/pdf", "application/pdf"), - ]; - - for (content_type, expected_mime) in test_cases { - let result = S3Client::detect_file_type(Some(content_type), b"some content"); - match result { - DetectedFileType::NonText(mime_type) => { - assert_eq!(mime_type, expected_mime); - } - _ => panic!("Expected NonText for content type {content_type}"), - } - } - } - - #[test] - fn test_detect_file_type_magic_bytes_simplified() { - // Test magic bytes detection (now all return NonText) - let test_cases = vec![ - // PNG magic bytes: 89 50 4E 47 - (vec![0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A], "image/png"), - // JPEG magic bytes: FF D8 FF - (vec![0xFF, 0xD8, 0xFF, 0xE0], "image/jpeg"), - // GIF magic bytes: 47 49 46 38 - (vec![0x47, 0x49, 0x46, 0x38, 0x37, 0x61], "image/gif"), - ]; - - for (content, expected_mime) in test_cases { - let result = S3Client::detect_file_type(None, &content); - match result { - DetectedFileType::NonText(mime_type) => { - assert_eq!(mime_type, expected_mime); - } - _ => panic!("Expected NonText for magic bytes: {content:?}"), - } - } - } - - #[test] - fn test_detect_file_type_webp_magic_bytes() { - // WebP has more complex magic bytes: RIFF....WEBP - let mut webp_content = vec![0x52, 0x49, 0x46, 0x46]; // RIFF - webp_content.extend_from_slice(&[0x00, 0x00, 0x00, 0x00]); // Size (4 bytes) - webp_content.extend_from_slice(b"WEBP"); // WEBP signature - - let result = S3Client::detect_file_type(None, &webp_content); - match result { - DetectedFileType::NonText(mime_type) => { - assert_eq!(mime_type, "image/webp"); - } - _ => panic!("Expected WebP NonText detection"), - } - } - - #[test] - fn test_detect_file_type_wav_magic_bytes() { - // WAV has magic bytes: RIFF....WAVE - let mut wav_content = vec![0x52, 0x49, 0x46, 0x46]; // RIFF - wav_content.extend_from_slice(&[0x00, 0x00, 0x00, 0x00]); // Size (4 bytes) - wav_content.extend_from_slice(b"WAVE"); // WAVE signature - - let result = S3Client::detect_file_type(None, &wav_content); - match result { - DetectedFileType::NonText(mime_type) => { - assert_eq!(mime_type, "audio/wav"); - } - _ => panic!("Expected WAV NonText detection"), - } - } - - #[test] - fn test_detect_file_type_utf8_text() { - // Test UTF-8 text detection - let utf8_content = "Hello, World! 🌍".as_bytes(); - let result = S3Client::detect_file_type(None, utf8_content); - match result { - DetectedFileType::Text => {} - _ => panic!("Expected Text for UTF-8 content"), - } - - // Test ASCII text - let ascii_content = b"Hello, world! This is ASCII text."; - let result = S3Client::detect_file_type(None, ascii_content); - match result { - DetectedFileType::Text => {} - _ => panic!("Expected Text for ASCII content"), - } - } - - #[test] - fn test_detect_file_type_binary() { - // Test binary content that should not be detected as text - let binary_content = vec![0x00, 0x01, 0x02, 0x03, 0xFF, 0xFE, 0xFD, 0xFC]; - let result = S3Client::detect_file_type(None, &binary_content); - match result { - DetectedFileType::NonText(mime_type) => { - assert_eq!(mime_type, "application/octet-stream"); - } - _ => panic!("Expected NonText for binary content"), - } - } - - #[test] - fn test_detect_file_type_priority() { - // Content-Type should take priority over magic bytes - let png_magic_bytes = vec![0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A]; - - // Even with PNG magic bytes, text content-type should win - let result = S3Client::detect_file_type(Some("text/plain"), &png_magic_bytes); - match result { - DetectedFileType::Text => {} - _ => panic!("Expected Text due to content-type priority"), - } - } - - #[test] - fn test_get_object_options_default() { - let options = GetObjectOptions::default(); - assert!(options.version_id.is_none()); - assert!(options.range.is_none()); - assert!(options.if_modified_since.is_none()); - assert!(options.if_unmodified_since.is_none()); - assert!(options.max_content_size.is_none()); - } - - #[test] - fn test_detected_file_type_serialization() { - let test_cases = vec![ - DetectedFileType::Text, - DetectedFileType::NonText("image/png".to_string()), - DetectedFileType::NonText("audio/mpeg".to_string()), - DetectedFileType::NonText("application/octet-stream".to_string()), - ]; - - for file_type in test_cases { - let json = serde_json::to_string(&file_type).unwrap(); - let deserialized: DetectedFileType = serde_json::from_str(&json).unwrap(); - - match (&file_type, &deserialized) { - (DetectedFileType::Text, DetectedFileType::Text) => {} - (DetectedFileType::NonText(a), DetectedFileType::NonText(b)) => assert_eq!(a, b), - _ => panic!("Serialization/deserialization mismatch"), - } - } - } -} diff --git a/crates/mcp/src/server.rs b/crates/mcp/src/server.rs deleted file mode 100644 index 46ff8b9ae..000000000 --- a/crates/mcp/src/server.rs +++ /dev/null @@ -1,737 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use anyhow::Result; -use rmcp::{ - ErrorData, RoleServer, ServerHandler, - handler::server::{router::tool::ToolRouter, wrapper::Parameters}, - model::{Implementation, ProtocolVersion, ServerCapabilities, ServerInfo}, - service::{NotificationContext, RequestContext}, - tool, tool_handler, tool_router, -}; -use schemars::JsonSchema; -use serde::{Deserialize, Serialize}; -use tracing::{debug, error, info}; - -use crate::config::Config; -use crate::s3_client::{DetectedFileType, GetObjectOptions, ListObjectsOptions, S3Client, UploadFileOptions}; - -#[derive(Serialize, Deserialize, JsonSchema)] -pub struct ListObjectsRequest { - pub bucket_name: String, - #[serde(default)] - #[schemars(description = "Optional prefix to filter objects")] - pub prefix: Option, -} - -#[derive(Serialize, Deserialize, JsonSchema)] -pub struct UploadFileRequest { - #[schemars(description = "Path to the local file to upload")] - pub local_file_path: String, - #[schemars(description = "Name of the S3 bucket to upload to")] - pub bucket_name: String, - #[schemars(description = "S3 object key (path/filename in the bucket)")] - pub object_key: String, - #[serde(default)] - #[schemars(description = "Optional content type (auto-detected if not specified)")] - pub content_type: Option, - #[serde(default)] - #[schemars(description = "Optional storage class (STANDARD, REDUCED_REDUNDANCY, etc.)")] - pub storage_class: Option, - #[serde(default)] - #[schemars(description = "Optional cache control header")] - pub cache_control: Option, -} - -#[derive(Serialize, Deserialize, JsonSchema)] -pub struct CreateBucketReqeust { - #[schemars(description = "Name of the S3 bucket to create")] - pub bucket_name: String, -} - -#[derive(Serialize, Deserialize, JsonSchema)] -pub struct DeleteBucketReqeust { - #[schemars(description = "Name of the S3 bucket to delete")] - pub bucket_name: String, -} - -#[derive(Serialize, Deserialize, JsonSchema)] -pub struct GetObjectRequest { - #[schemars(description = "Name of the S3 bucket")] - pub bucket_name: String, - #[schemars(description = "S3 object key (path/filename in the bucket)")] - pub object_key: String, - #[serde(default)] - #[schemars(description = "Optional version ID for versioned objects")] - pub version_id: Option, - #[serde(default = "default_operation_mode")] - #[schemars(description = "Operation mode: read (return content) or download (save to local file)")] - pub mode: GetObjectMode, - #[serde(default)] - #[schemars(description = "Local file path for download mode (required when mode is download)")] - pub local_path: Option, - #[serde(default = "default_max_content_size")] - #[schemars(description = "Maximum content size to read in bytes for read mode (default: 1MB)")] - pub max_content_size: usize, -} - -#[derive(Serialize, Deserialize, JsonSchema, Debug, Clone, PartialEq)] -pub enum GetObjectMode { - #[serde(rename = "read")] - Read, - #[serde(rename = "download")] - Download, -} - -fn default_operation_mode() -> GetObjectMode { - GetObjectMode::Read -} -fn default_max_content_size() -> usize { - 1024 * 1024 -} - -#[derive(Debug, Clone)] -pub struct RustfsMcpServer { - s3_client: S3Client, - _config: Config, - tool_router: ToolRouter, -} - -#[tool_router(router = tool_router)] -impl RustfsMcpServer { - pub async fn new(config: Config) -> Result { - info!("Creating RustFS MCP Server"); - - let s3_client = S3Client::new(&config).await?; - - Ok(Self { - s3_client, - _config: config, - tool_router: Self::tool_router(), - }) - } - - #[tool(description = "Create a new S3 bucket with the specified name")] - pub async fn create_bucket(&self, Parameters(req): Parameters) -> String { - info!("Executing create_bucket tool for bucket: {}", req.bucket_name); - - match self.s3_client.create_bucket(&req.bucket_name).await { - Ok(_) => { - format!("Successfully created bucket: {}", req.bucket_name) - } - Err(e) => { - format!("Failed to create bucket '{}': {:?}", req.bucket_name, e) - } - } - } - - #[tool(description = "Delete an existing S3 bucket with the specified name")] - pub async fn delete_bucket(&self, Parameters(req): Parameters) -> String { - info!("Executing delete_bucket tool for bucket: {}", req.bucket_name); - - // check if bucket is empty, if not, can not delete bucket directly. - let object_result = match self - .s3_client - .list_objects_v2(&req.bucket_name, ListObjectsOptions::default()) - .await - { - Ok(result) => result, - Err(e) => { - error!("Failed to list objects in bucket '{}': {:?}", req.bucket_name, e); - return format!("Failed to list objects in bucket '{}': {:?}", req.bucket_name, e); - } - }; - - if !object_result.objects.is_empty() { - error!("Bucket '{}' is not empty", req.bucket_name); - return format!("Failed to delete bucket '{}': bucket is not empty", req.bucket_name); - } - - // delete the bucket. - match self.s3_client.delete_bucket(&req.bucket_name).await { - Ok(_) => { - format!("Successfully deleted bucket: {}", req.bucket_name) - } - Err(e) => { - format!("Failed to delete bucket '{}': {:?}", req.bucket_name, e) - } - } - } - - #[tool(description = "List all S3 buckets accessible with the configured credentials")] - pub async fn list_buckets(&self) -> String { - info!("Executing list_buckets tool"); - - match self.s3_client.list_buckets().await { - Ok(buckets) => { - debug!("Successfully retrieved {} buckets", buckets.len()); - - if buckets.is_empty() { - return "No S3 buckets found. The AWS credentials may not have access to any buckets, or no buckets exist in this account.".to_string(); - } - - let mut result_text = format!("Found {} S3 bucket(s):\n\n", buckets.len()); - - for (index, bucket) in buckets.iter().enumerate() { - result_text.push_str(&format!("{}. **{}**", index + 1, bucket.name)); - - if let Some(ref creation_date) = bucket.creation_date { - result_text.push_str(&format!("\n - Created: {creation_date}")); - } - result_text.push_str("\n\n"); - } - - result_text.push_str("---\n"); - result_text.push_str(&format!("Total buckets: {}\n", buckets.len())); - result_text.push_str("Note: Only buckets accessible with the current AWS credentials are shown."); - - info!("list_buckets tool executed successfully"); - result_text - } - Err(e) => { - error!("Failed to list buckets: {:?}", e); - - format!( - "Failed to list S3 buckets: {e}\n\nPossible causes:\n\ - • AWS credentials are not set or invalid\n\ - • Network connectivity issues\n\ - • AWS region is not set correctly\n\ - • Insufficient permissions to list buckets\n\ - • Custom endpoint is misconfigured\n\n\ - Please verify your AWS configuration and try again." - ) - } - } - } - - #[tool(description = "List objects in a specific S3 bucket with optional prefix filtering")] - pub async fn list_objects(&self, Parameters(req): Parameters) -> String { - info!("Executing list_objects tool for bucket: {}", req.bucket_name); - - let options = ListObjectsOptions { - prefix: req.prefix.clone(), - delimiter: None, - max_keys: Some(1000), - ..ListObjectsOptions::default() - }; - - match self.s3_client.list_objects_v2(&req.bucket_name, options).await { - Ok(result) => { - debug!( - "Successfully retrieved {} objects and {} common prefixes from bucket '{}'", - result.objects.len(), - result.common_prefixes.len(), - req.bucket_name - ); - - if result.objects.is_empty() && result.common_prefixes.is_empty() { - let prefix_msg = req.prefix.as_ref().map(|p| format!(" with prefix '{p}'")).unwrap_or_default(); - return format!( - "No objects found in bucket '{}'{prefix_msg}. The bucket may be empty or the prefix may not match any objects.", - req.bucket_name - ); - } - - let mut result_text = format!("Found {} object(s) in bucket **{}**", result.key_count, req.bucket_name); - - if let Some(ref p) = req.prefix { - result_text.push_str(&format!(" with prefix '{p}'")); - } - result_text.push_str(":\n\n"); - - if !result.common_prefixes.is_empty() { - result_text.push_str("**Directories:**\n"); - for (index, prefix) in result.common_prefixes.iter().enumerate() { - result_text.push_str(&format!("{}. 📁 {prefix}\n", index + 1)); - } - result_text.push('\n'); - } - - if !result.objects.is_empty() { - result_text.push_str("**Objects:**\n"); - for (index, obj) in result.objects.iter().enumerate() { - result_text.push_str(&format!("{}. **{}**\n", index + 1, obj.key)); - - if let Some(size) = obj.size { - result_text.push_str(&format!(" - Size: {size} bytes\n")); - } - - if let Some(ref last_modified) = obj.last_modified { - result_text.push_str(&format!(" - Last Modified: {last_modified}\n")); - } - - if let Some(ref etag) = obj.etag { - result_text.push_str(&format!(" - ETag: {etag}\n")); - } - - if let Some(ref storage_class) = obj.storage_class { - result_text.push_str(&format!(" - Storage Class: {storage_class}\n")); - } - - result_text.push('\n'); - } - } - - if result.is_truncated { - result_text.push_str("**Note:** Results are truncated. "); - if let Some(ref token) = result.next_continuation_token { - result_text.push_str(&format!("Use continuation token '{token}' to get more results.\n")); - } - result_text.push('\n'); - } - - result_text.push_str("---\n"); - result_text.push_str(&format!( - "Total: {} object(s), {} directory/ies", - result.objects.len(), - result.common_prefixes.len() - )); - - if let Some(max_keys) = result.max_keys { - result_text.push_str(&format!(", Max keys: {max_keys}")); - } - - info!("list_objects tool executed successfully for bucket '{}'", req.bucket_name); - result_text - } - Err(e) => { - error!("Failed to list objects in bucket '{}': {:?}", req.bucket_name, e); - - format!( - "Failed to list objects in S3 bucket '{}': {}\n\nPossible causes:\n\ - • Bucket does not exist or is not accessible\n\ - • AWS credentials lack permissions to list objects in this bucket\n\ - • Network connectivity issues\n\ - • Custom endpoint is misconfigured\n\ - • Bucket name contains invalid characters\n\n\ - Please verify the bucket name, your AWS configuration, and permissions.", - req.bucket_name, e - ) - } - } - } - - #[tool( - description = "Get/download an object from an S3 bucket - supports read mode for text files and download mode for all files" - )] - pub async fn get_object(&self, Parameters(req): Parameters) -> String { - info!( - "Executing get_object tool: s3://{}/{} (mode: {:?})", - req.bucket_name, req.object_key, req.mode - ); - - match req.mode { - GetObjectMode::Read => self.handle_read_mode(req).await, - GetObjectMode::Download => self.handle_download_mode(req).await, - } - } - - async fn handle_read_mode(&self, req: GetObjectRequest) -> String { - let options = GetObjectOptions { - version_id: req.version_id.clone(), - max_content_size: Some(req.max_content_size), - ..GetObjectOptions::default() - }; - - match self.s3_client.get_object(&req.bucket_name, &req.object_key, options).await { - Ok(result) => { - debug!( - "Successfully retrieved object s3://{}/{} ({} bytes)", - req.bucket_name, req.object_key, result.content_length - ); - - match result.detected_type { - DetectedFileType::Text => { - if let Some(ref text_content) = result.text_content { - format!( - "✅ **Text file content retrieved!**\n\n\ - **S3 Location:** s3://{}/{}\n\ - **File Size:** {} bytes\n\ - **Content Type:** {}\n\n\ - **Content:**\n```\n{}\n```", - result.bucket, result.key, result.content_length, result.content_type, text_content - ) - } else { - format!( - "⚠️ **Text file detected but content could not be decoded!**\n\n\ - **S3 Location:** s3://{}/{}\n\ - **File Size:** {} bytes\n\ - **Content Type:** {}\n\n\ - **Note:** Could not decode file as UTF-8 text. \ - Try using download mode instead.", - result.bucket, result.key, result.content_length, result.content_type - ) - } - } - DetectedFileType::NonText(ref mime_type) => { - let file_category = if mime_type.starts_with("image/") { - "Image" - } else if mime_type.starts_with("audio/") { - "Audio" - } else if mime_type.starts_with("video/") { - "Video" - } else { - "Binary" - }; - - format!( - "⚠️ **Non-text file detected!**\n\n\ - **S3 Location:** s3://{}/{}\n\ - **File Type:** {} ({})\n\ - **File Size:** {} bytes ({:.2} MB)\n\n\ - **Note:** This file type cannot be displayed as text.\n\ - Please use download mode to save it to a local file:\n\n\ - ```json\n{{\n \"mode\": \"download\",\n \"local_path\": \"/path/to/save/file\"\n}}\n```", - result.bucket, - result.key, - file_category, - mime_type, - result.content_length, - result.content_length as f64 / 1_048_576.0 - ) - } - } - } - Err(e) => { - error!("Failed to read object s3://{}/{}: {:?}", req.bucket_name, req.object_key, e); - self.format_error_message(&req, e) - } - } - } - - async fn handle_download_mode(&self, req: GetObjectRequest) -> String { - let local_path = match req.local_path { - Some(ref path) => path, - None => { - return "❌ **Error:** local_path is required when using download mode.\n\n\ - **Example:**\n```json\n{\n \"mode\": \"download\",\n \"local_path\": \"/path/to/save/file.ext\"\n}\n```" - .to_string(); - } - }; - - let options = GetObjectOptions { - version_id: req.version_id.clone(), - ..GetObjectOptions::default() - }; - - match self - .s3_client - .download_object_to_file(&req.bucket_name, &req.object_key, local_path, options) - .await - { - Ok((bytes_downloaded, absolute_path)) => { - info!( - "Successfully downloaded object s3://{}/{} to {} ({} bytes)", - req.bucket_name, req.object_key, absolute_path, bytes_downloaded - ); - - format!( - "✅ **File downloaded successfully!**\n\n\ - **S3 Location:** s3://{}/{}\n\ - **Local Path (requested):** {}\n\ - **Absolute Path:** {}\n\ - **File Size:** {} bytes ({:.2} MB)\n\n\ - **✨ File saved successfully!** You can now access it at:\n\ - `{}`", - req.bucket_name, - req.object_key, - local_path, - absolute_path, - bytes_downloaded, - bytes_downloaded as f64 / 1_048_576.0, - absolute_path - ) - } - Err(e) => { - error!( - "Failed to download object s3://{}/{} to {}: {:?}", - req.bucket_name, req.object_key, local_path, e - ); - - format!( - "❌ **Failed to download file from S3**\n\n\ - **S3 Location:** s3://{}/{}\n\ - **Local Path:** {}\n\ - **Error:** {}\n\n\ - **Possible causes:**\n\ - • Object does not exist in the specified bucket\n\ - • AWS credentials lack permissions to read this object\n\ - • Cannot write to the specified local path\n\ - • Insufficient disk space\n\ - • Network connectivity issues\n\n\ - **Troubleshooting steps:**\n\ - 1. Verify the object exists using list_objects\n\ - 2. Check your AWS credentials and permissions\n\ - 3. Ensure the local directory exists and is writable\n\ - 4. Check available disk space", - req.bucket_name, req.object_key, local_path, e - ) - } - } - } - - fn format_error_message(&self, req: &GetObjectRequest, error: anyhow::Error) -> String { - format!( - "❌ **Failed to get object from S3 bucket '{}'**\n\n\ - **Object Key:** {}\n\ - **Mode:** {:?}\n\ - **Error:** {}\n\n\ - **Possible causes:**\n\ - • Object does not exist in the specified bucket\n\ - • AWS credentials lack permissions to read this object\n\ - • Network connectivity issues\n\ - • Object key contains invalid characters\n\ - • Bucket does not exist or is not accessible\n\ - • Object is in a different AWS region\n\ - • Version ID is invalid (for versioned objects)\n\n\ - **Troubleshooting steps:**\n\ - 1. Verify the object exists using list_objects\n\ - 2. Check your AWS credentials and permissions\n\ - 3. Ensure the bucket name and object key are correct\n\ - 4. Try with a different object to test connectivity\n\ - 5. Check if the bucket has versioning enabled", - req.bucket_name, req.object_key, req.mode, error - ) - } - - #[tool(description = "Upload a local file to an S3 bucket")] - pub async fn upload_file(&self, Parameters(req): Parameters) -> String { - info!( - "Executing upload_file tool: '{}' -> s3://{}/{}", - req.local_file_path, req.bucket_name, req.object_key - ); - - let options = UploadFileOptions { - content_type: req.content_type.clone(), - storage_class: req.storage_class.clone(), - cache_control: req.cache_control.clone(), - ..UploadFileOptions::default() - }; - - match self - .s3_client - .upload_file(&req.local_file_path, &req.bucket_name, &req.object_key, options) - .await - { - Ok(result) => { - debug!( - "Successfully uploaded file '{}' to s3://{}/{} ({} bytes)", - req.local_file_path, req.bucket_name, req.object_key, result.file_size - ); - - let mut result_text = format!( - "✅ **File uploaded successfully!**\n\n\ - **Local File:** {}\n\ - **S3 Location:** s3://{}/{}\n\ - **File Size:** {} bytes ({:.2} MB)\n\ - **Content Type:** {}\n\ - **ETag:** {}\n", - req.local_file_path, - result.bucket, - result.key, - result.file_size, - result.file_size as f64 / 1_048_576.0, - result.content_type, - result.etag - ); - - if let Some(ref version_id) = result.version_id { - result_text.push_str(&format!("**Version ID:** {version_id}\n")); - } - - result_text.push_str("\n---\n"); - result_text.push_str("**Upload Summary:**\n"); - result_text.push_str(&format!("• Source: {}\n", req.local_file_path)); - result_text.push_str(&format!("• Destination: {}\n", result.location)); - result_text.push_str(&format!("• Size: {} bytes\n", result.file_size)); - result_text.push_str(&format!("• Type: {}\n", result.content_type)); - - if result.file_size > 5 * 1024 * 1024 { - result_text.push_str("\n💡 **Note:** Large file uploaded successfully. Consider using multipart upload for files larger than 100MB for better performance and reliability."); - } - - info!( - "upload_file tool executed successfully: {} bytes uploaded to s3://{}/{}", - result.file_size, req.bucket_name, req.object_key - ); - result_text - } - Err(e) => { - error!( - "Failed to upload file '{}' to s3://{}/{}: {:?}", - req.local_file_path, req.bucket_name, req.object_key, e - ); - - format!( - "❌ **Failed to upload file '{}' to S3 bucket '{}'**\n\n\ - **Error:** {}\n\n\ - **Possible causes:**\n\ - • Local file does not exist or is not readable\n\ - • AWS credentials lack permissions to upload to this bucket\n\ - • S3 bucket does not exist or is not accessible\n\ - • Network connectivity issues\n\ - • File path contains invalid characters or is too long\n\ - • Insufficient disk space or memory\n\ - • Custom endpoint is misconfigured\n\ - • File is locked by another process\n\n\ - **Troubleshooting steps:**\n\ - 1. Verify the local file exists and is readable\n\ - 2. Check your AWS credentials and permissions\n\ - 3. Ensure the bucket name is correct and accessible\n\ - 4. Try with a smaller file to test connectivity\n\ - 5. Check the file path for special characters\n\n\ - **File:** {}\n\ - **Bucket:** {}\n\ - **Object Key:** {}", - req.local_file_path, req.bucket_name, e, req.local_file_path, req.bucket_name, req.object_key - ) - } - } - } -} - -#[tool_handler(router = self.tool_router)] -impl ServerHandler for RustfsMcpServer { - fn get_info(&self) -> ServerInfo { - ServerInfo::new(ServerCapabilities::builder().enable_tools().build()) - .with_instructions("RustFS MCP Server providing S3 operations through Model Context Protocol") - .with_server_info(Implementation::new("rustfs-mcp-server", env!("CARGO_PKG_VERSION"))) - .with_protocol_version(ProtocolVersion::LATEST) - } - - async fn ping(&self, _ctx: RequestContext) -> Result<(), ErrorData> { - info!("Received ping request"); - Ok(()) - } - - async fn on_initialized(&self, _ctx: NotificationContext) { - info!("Client initialized successfully"); - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[tokio::test] - async fn test_server_creation() { - let config = Config { - access_key_id: Some("test_key".to_string()), - secret_access_key: Some("test_secret".to_string()), - endpoint_url: Some("http://127.0.0.1:9000".to_string()), - force_path_style: true, - ..Config::default() - }; - - let result = RustfsMcpServer::new(config).await; - assert!(result.is_err() || result.is_ok()); - } - - #[test] - fn test_get_object_request_defaults() { - let request = GetObjectRequest { - bucket_name: "test-bucket".to_string(), - object_key: "test-key".to_string(), - version_id: None, - mode: default_operation_mode(), - local_path: None, - max_content_size: default_max_content_size(), - }; - - assert_eq!(request.bucket_name, "test-bucket"); - assert_eq!(request.object_key, "test-key"); - assert!(request.version_id.is_none()); - assert_eq!(request.mode, GetObjectMode::Read); - assert!(request.local_path.is_none()); - assert_eq!(request.max_content_size, 1024 * 1024); - } - - #[test] - fn test_get_object_request_serialization() { - let request = GetObjectRequest { - bucket_name: "test-bucket".to_string(), - object_key: "test-key".to_string(), - version_id: Some("version123".to_string()), - mode: GetObjectMode::Download, - local_path: Some("/path/to/file".to_string()), - max_content_size: 2048, - }; - - let json = serde_json::to_string(&request).unwrap(); - let deserialized: GetObjectRequest = serde_json::from_str(&json).unwrap(); - - assert_eq!(request.bucket_name, deserialized.bucket_name); - assert_eq!(request.object_key, deserialized.object_key); - assert_eq!(request.version_id, deserialized.version_id); - assert_eq!(request.mode, deserialized.mode); - assert_eq!(request.local_path, deserialized.local_path); - assert_eq!(request.max_content_size, deserialized.max_content_size); - } - - #[test] - fn test_get_object_request_serde_with_defaults() { - let json = r#"{ - "bucket_name": "test-bucket", - "object_key": "test-key" - }"#; - - let request: GetObjectRequest = serde_json::from_str(json).unwrap(); - assert_eq!(request.bucket_name, "test-bucket"); - assert_eq!(request.object_key, "test-key"); - assert!(request.version_id.is_none()); - assert_eq!(request.mode, GetObjectMode::Read); - assert!(request.local_path.is_none()); - assert_eq!(request.max_content_size, 1024 * 1024); - } - - #[test] - fn test_default_functions() { - assert_eq!(default_operation_mode(), GetObjectMode::Read); - assert_eq!(default_max_content_size(), 1024 * 1024); - } - - #[test] - fn test_get_object_mode_serialization() { - let read_mode = GetObjectMode::Read; - let download_mode = GetObjectMode::Download; - - let read_json = serde_json::to_string(&read_mode).unwrap(); - let download_json = serde_json::to_string(&download_mode).unwrap(); - - assert_eq!(read_json, r#""read""#); - assert_eq!(download_json, r#""download""#); - - let read_mode_deser: GetObjectMode = serde_json::from_str(r#""read""#).unwrap(); - let download_mode_deser: GetObjectMode = serde_json::from_str(r#""download""#).unwrap(); - - assert_eq!(read_mode_deser, GetObjectMode::Read); - assert_eq!(download_mode_deser, GetObjectMode::Download); - } - - #[test] - fn test_bucket_creation() { - let request = CreateBucketReqeust { - bucket_name: "test-bucket".to_string(), - }; - assert_eq!(request.bucket_name, "test-bucket"); - } - - #[test] - fn test_bucket_deletion() { - let request = DeleteBucketReqeust { - bucket_name: "test-bucket".to_string(), - }; - assert_eq!(request.bucket_name, "test-bucket"); - } -} diff --git a/crates/protocols/Cargo.toml b/crates/protocols/Cargo.toml index 4ee045164..d8f4b601f 100644 --- a/crates/protocols/Cargo.toml +++ b/crates/protocols/Cargo.toml @@ -64,7 +64,7 @@ rustfs-utils = { workspace = true } rustfs-config = { workspace = true } # Async dependencies -tokio = { workspace = true, features = ["fs", "io-util", "sync", "time"] } +tokio = { workspace = true, features = ["fs", "io-util", "sync", "time","io-uring"] } tracing = { workspace = true } futures-util = { workspace = true } diff --git a/crates/scanner/Cargo.toml b/crates/scanner/Cargo.toml index d5b939ea5..c8a87e79e 100644 --- a/crates/scanner/Cargo.toml +++ b/crates/scanner/Cargo.toml @@ -33,7 +33,7 @@ workspace = true rustfs-config = { workspace = true } rustfs-common = { workspace = true } rustfs-utils = { workspace = true } -tokio = { workspace = true, features = ["full"] } +tokio = { workspace = true, features = ["fs","sync","rt","time","io-uring","macros"] } tracing = { workspace = true } serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } diff --git a/crates/utils/Cargo.toml b/crates/utils/Cargo.toml index 387ffdc27..b3eec08f4 100644 --- a/crates/utils/Cargo.toml +++ b/crates/utils/Cargo.toml @@ -43,7 +43,6 @@ local-ip-address = { workspace = true, optional = true } lz4 = { workspace = true, optional = true } md-5 = { workspace = true, optional = true } netif = { workspace = true, optional = true } -rand = { workspace = true, optional = true } regex = { workspace = true, optional = true } rustix = { workspace = true, optional = true } rustls = { workspace = true, optional = true } @@ -65,7 +64,6 @@ zstd = { workspace = true, optional = true } [dev-dependencies] tempfile = { workspace = true } -rand = { workspace = true } tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } temp-env = { workspace = true } diff --git a/crates/zip/Cargo.toml b/crates/zip/Cargo.toml index f07da5ba0..29241b06d 100644 --- a/crates/zip/Cargo.toml +++ b/crates/zip/Cargo.toml @@ -35,7 +35,7 @@ async-compression = { workspace = true, features = [ "zstd", "xz", ] } -tokio = { workspace = true, features = ["full"] } +tokio = { workspace = true, features = ["io-uring","fs","io-util","macros"] } tokio-stream = { workspace = true } astral-tokio-tar = { workspace = true } diff --git a/rustfs/Cargo.toml b/rustfs/Cargo.toml index df1042e3a..e0111f024 100644 --- a/rustfs/Cargo.toml +++ b/rustfs/Cargo.toml @@ -107,7 +107,7 @@ http-body.workspace = true http-body-util.workspace = true reqwest = { workspace = true } socket2 = { workspace = true } -tokio = { workspace = true, features = ["rt-multi-thread", "macros", "net", "signal", "process", "io-util"] } +tokio = { workspace = true, features = ["rt-multi-thread", "macros", "net", "signal", "process", "io-util","io-uring"] } tokio-rustls = { workspace = true } aws-sdk-s3 = { workspace = true } tokio-stream.workspace = true