From c28134c957dc190d4217c51a68ee73d4c1549494 Mon Sep 17 00:00:00 2001 From: houseme Date: Sun, 25 Jan 2026 02:50:29 +0800 Subject: [PATCH] Add support for success_action_status and success_action_redirect in AWS S3 POST object uploads (#1606) --- Cargo.lock | 343 +- Cargo.toml | 22 +- rustfs/src/init.rs | 2 +- rustfs/src/server/layer.rs | 4 +- rustfs/src/storage/ecfs.rs | 6866 ++++++++++------------------- rustfs/src/storage/ecfs_extend.rs | 1100 +++++ rustfs/src/storage/ecfs_test.rs | 1194 +++++ rustfs/src/storage/mod.rs | 5 + 8 files changed, 4859 insertions(+), 4677 deletions(-) create mode 100644 rustfs/src/storage/ecfs_extend.rs create mode 100644 rustfs/src/storage/ecfs_test.rs diff --git a/Cargo.lock b/Cargo.lock index cce8fdbba..47041e980 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1158,7 +1158,7 @@ dependencies = [ "miniz_oxide", "object 0.37.3", "rustc-demangle", - "windows-link 0.2.1", + "windows-link", ] [[package]] @@ -1522,7 +1522,7 @@ dependencies = [ "num-traits", "serde", "wasm-bindgen", - "windows-link 0.2.1", + "windows-link", ] [[package]] @@ -2294,9 +2294,9 @@ checksum = "d7a1e2f27636f116493b8b860f5546edb47c8d8f8ea73e1d2a20be88e28d1fea" [[package]] name = "datafusion" -version = "52.0.0" +version = "52.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f02e9a7e70f214e5282db11c8effba173f4e25a00977e520c6b811817e3a082b" +checksum = "d12ee9fdc6cdb5898c7691bb994f0ba606c4acc93a2258d78bb9f26ff8158bb3" dependencies = [ "arrow", "arrow-schema", @@ -2349,9 +2349,9 @@ dependencies = [ [[package]] name = "datafusion-catalog" -version = "52.0.0" +version = "52.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3e91b2603f906cf8cb8be84ba4e34f9d8fe6dbdfdd6916d55f22317074d1fdf" +checksum = "462dc9ef45e5d688aeaae49a7e310587e81b6016b9d03bace5626ad0043e5a9e" dependencies = [ "arrow", "async-trait", @@ -2374,9 +2374,9 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" -version = "52.0.0" +version = "52.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "919d20cdebddee4d8dca651aa0291a44c8104824d1ac288996a325c319ce31ba" +checksum = "1b96dbf1d728fc321817b744eb5080cdd75312faa6980b338817f68f3caa4208" dependencies = [ "arrow", "async-trait", @@ -2397,9 +2397,9 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "52.0.0" +version = "52.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31ff2c4e95be40ad954de93862167b165a6fb49248bb882dea8aef4f888bc767" +checksum = "3237a6ff0d2149af4631290074289cae548c9863c885d821315d54c6673a074a" dependencies = [ "ahash", "arrow", @@ -2421,9 +2421,9 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" -version = "52.0.0" +version = "52.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0dd9f820fe58c2600b6c33a14432228dbaaf233b96c83a1fd61f16d073d5c3c5" +checksum = "70b5e34026af55a1bfccb1ef0a763cf1f64e77c696ffcf5a128a278c31236528" dependencies = [ "futures", "log", @@ -2432,9 +2432,9 @@ dependencies = [ [[package]] name = "datafusion-datasource" -version = "52.0.0" +version = "52.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86b32b7b12645805d20b70aba6ba846cd262d7b073f7f617640c3294af108d44" +checksum = "1b2a6be734cc3785e18bbf2a7f2b22537f6b9fb960d79617775a51568c281842" dependencies = [ "arrow", "async-compression", @@ -2467,9 +2467,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-arrow" -version = "52.0.0" +version = "52.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "597695c8ebb723ee927b286139d43a3fbed6de7ad9210bd1a9fed5c721ac6fb1" +checksum = "1739b9b07c9236389e09c74f770e88aff7055250774e9def7d3f4f56b3dcc7be" dependencies = [ "arrow", "arrow-ipc", @@ -2491,9 +2491,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" -version = "52.0.0" +version = "52.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bb493d07d8da6d00a89ea9cc3e74a56795076d9faed5ac30284bd9ef37929e9" +checksum = "61c73bc54b518bbba7c7650299d07d58730293cfba4356f6f428cc94c20b7600" dependencies = [ "arrow", "async-trait", @@ -2514,9 +2514,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" -version = "52.0.0" +version = "52.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e9806521c4d3632f53b9a664041813c267c670232efa1452ef29faee71c3749" +checksum = "37812c8494c698c4d889374ecfabbff780f1f26d9ec095dd1bddfc2a8ca12559" dependencies = [ "arrow", "async-trait", @@ -2536,9 +2536,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-parquet" -version = "52.0.0" +version = "52.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6a3ccd48d5034f8461f522114d0e46dfb3a9f0ce01a4d53a721024ace95d60d" +checksum = "2210937ecd9f0e824c397e73f4b5385c97cd1aff43ab2b5836fcfd2d321523fb" dependencies = [ "arrow", "async-trait", @@ -2566,15 +2566,15 @@ dependencies = [ [[package]] name = "datafusion-doc" -version = "52.0.0" +version = "52.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff69a18418e9878d4840f35e2ad7f2a6386beedf192e9f065e628a7295ff5fbf" +checksum = "2c825f969126bc2ef6a6a02d94b3c07abff871acf4d6dd759ce1255edb7923ce" [[package]] name = "datafusion-execution" -version = "52.0.0" +version = "52.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccbc5e469b35d87c0b115327be83d68356ef9154684d32566315b5c071577e23" +checksum = "fa03ef05a2c2f90dd6c743e3e111078e322f4b395d20d4b4d431a245d79521ae" dependencies = [ "arrow", "async-trait", @@ -2593,9 +2593,9 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "52.0.0" +version = "52.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81ed3c02a3faf4e09356d5a314471703f440f0a6a14ca6addaf6cfb44ab14de5" +checksum = "ef33934c1f98ee695cc51192cc5f9ed3a8febee84fdbcd9131bf9d3a9a78276f" dependencies = [ "arrow", "async-trait", @@ -2616,9 +2616,9 @@ dependencies = [ [[package]] name = "datafusion-expr-common" -version = "52.0.0" +version = "52.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1567e60d21c372ca766dc9dde98efabe2b06d98f008d988fed00d93546bf5be7" +checksum = "000c98206e3dd47d2939a94b6c67af4bfa6732dd668ac4fafdbde408fd9134ea" dependencies = [ "arrow", "datafusion-common", @@ -2629,9 +2629,9 @@ dependencies = [ [[package]] name = "datafusion-functions" -version = "52.0.0" +version = "52.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4593538abd95c27eeeb2f86b7ad827cce07d0c474eae9b122f4f9675f8c20ad" +checksum = "379b01418ab95ca947014066248c22139fe9af9289354de10b445bd000d5d276" dependencies = [ "arrow", "arrow-buffer", @@ -2660,9 +2660,9 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" -version = "52.0.0" +version = "52.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f81cdf609f43cd26156934fd81beb7215d60dda40a776c2e1b83d73df69434f2" +checksum = "fd00d5454ba4c3f8ebbd04bd6a6a9dc7ced7c56d883f70f2076c188be8459e4c" dependencies = [ "ahash", "arrow", @@ -2681,9 +2681,9 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" -version = "52.0.0" +version = "52.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9173f1bcea2ede4a5c23630a48469f06c9db9a408eb5fd140d1ff9a5e0c40ebf" +checksum = "aec06b380729a87210a4e11f555ec2d729a328142253f8d557b87593622ecc9f" dependencies = [ "ahash", "arrow", @@ -2694,9 +2694,9 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" -version = "52.0.0" +version = "52.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d0b9f32e7735a3b94ae8b9596d89080dc63dd139029a91133be370da099490d" +checksum = "904f48d45e0f1eb7d0eb5c0f80f2b5c6046a85454364a6b16a2e0b46f62e7dff" dependencies = [ "arrow", "arrow-ord", @@ -2717,9 +2717,9 @@ dependencies = [ [[package]] name = "datafusion-functions-table" -version = "52.0.0" +version = "52.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57a29e8a6201b3b9fb2be17d88e287c6d427948d64220cd5ea72ced614a1aee5" +checksum = "e9a0d20e2b887e11bee24f7734d780a2588b925796ac741c3118dd06d5aa77f0" dependencies = [ "arrow", "async-trait", @@ -2733,9 +2733,9 @@ dependencies = [ [[package]] name = "datafusion-functions-window" -version = "52.0.0" +version = "52.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd412754964a31c515e5a814e5ce0edaf30f0ea975f3691e800eff115ee76dfb" +checksum = "d3414b0a07e39b6979fe3a69c7aa79a9f1369f1d5c8e52146e66058be1b285ee" dependencies = [ "arrow", "datafusion-common", @@ -2751,9 +2751,9 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" -version = "52.0.0" +version = "52.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d49be73a5ac0797398927a543118bd68e58e80bf95ebdabc77336bcd9c38a711" +checksum = "5bf2feae63cd4754e31add64ce75cae07d015bce4bb41cd09872f93add32523a" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -2761,9 +2761,9 @@ dependencies = [ [[package]] name = "datafusion-macros" -version = "52.0.0" +version = "52.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "439ff5489dcac4d34ed7a49a93310c3345018c4469e34726fa471cdda725346d" +checksum = "c4fe888aeb6a095c4bcbe8ac1874c4b9a4c7ffa2ba849db7922683ba20875aaf" dependencies = [ "datafusion-doc", "quote", @@ -2772,9 +2772,9 @@ dependencies = [ [[package]] name = "datafusion-optimizer" -version = "52.0.0" +version = "52.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a80bb7de8ff5a9948799bc7749c292eac5c629385cdb582893ef2d80b6e718c4" +checksum = "8a6527c063ae305c11be397a86d8193936f4b84d137fe40bd706dfc178cf733c" dependencies = [ "arrow", "chrono", @@ -2792,9 +2792,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "52.0.0" +version = "52.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83480008f66691a0047c5a88990bd76b7c1117dd8a49ca79959e214948b81f0a" +checksum = "0bb028323dd4efd049dd8a78d78fe81b2b969447b39c51424167f973ac5811d9" dependencies = [ "ahash", "arrow", @@ -2816,9 +2816,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-adapter" -version = "52.0.0" +version = "52.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b438306446646b359666a658cc29d5494b1e9873bc7a57707689760666fc82c" +checksum = "78fe0826aef7eab6b4b61533d811234a7a9e5e458331ebbf94152a51fc8ab433" dependencies = [ "arrow", "datafusion-common", @@ -2831,9 +2831,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" -version = "52.0.0" +version = "52.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95b1fbf739038e0b313473588331c5bf79985d1b842b9937c1f10b170665cae1" +checksum = "cfccd388620734c661bd8b7ca93c44cdd59fecc9b550eea416a78ffcbb29475f" dependencies = [ "ahash", "arrow", @@ -2848,9 +2848,9 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" -version = "52.0.0" +version = "52.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc4cd3a170faa0f1de04bd4365ccfe309056746dd802ed276e8787ccb8e8a0d4" +checksum = "bde5fa10e73259a03b705d5fddc136516814ab5f441b939525618a4070f5a059" dependencies = [ "arrow", "datafusion-common", @@ -2867,9 +2867,9 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" -version = "52.0.0" +version = "52.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a616a72b4ddf550652b36d5a7c0386eac4accea3ffc6c29a7b16c45f237e9882" +checksum = "0e1098760fb29127c24cc9ade3277051dc73c9ed0ac0131bd7bcd742e0ad7470" dependencies = [ "ahash", "arrow", @@ -2898,9 +2898,9 @@ dependencies = [ [[package]] name = "datafusion-pruning" -version = "52.0.0" +version = "52.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bf4b50be3ab65650452993eda4baf81edb245fb039b8714476b0f4c8801a527" +checksum = "64d0fef4201777b52951edec086c21a5b246f3c82621569ddb4a26f488bc38a9" dependencies = [ "arrow", "datafusion-common", @@ -2915,9 +2915,9 @@ dependencies = [ [[package]] name = "datafusion-session" -version = "52.0.0" +version = "52.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66e080e2c105284460580c18e751b2133cc306df298181e4349b5b134632811a" +checksum = "f71f1e39e8f2acbf1c63b0e93756c2e970a64729dab70ac789587d6237c4fde0" dependencies = [ "async-trait", "datafusion-common", @@ -2929,9 +2929,9 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "52.0.0" +version = "52.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3dac502db772ff9bffc2ceae321963091982e8d5f5dfcb877e8dc66fc9a093cc" +checksum = "f44693cfcaeb7a9f12d71d1c576c3a6dc025a12cef209375fa2d16fb3b5670ee" dependencies = [ "arrow", "bigdecimal", @@ -4567,7 +4567,7 @@ dependencies = [ "js-sys", "log", "wasm-bindgen", - "windows-core 0.62.2", + "windows-core", ] [[package]] @@ -5187,7 +5187,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" dependencies = [ "cfg-if", - "windows-link 0.2.1", + "windows-link", ] [[package]] @@ -5646,9 +5646,9 @@ dependencies = [ [[package]] name = "nix" -version = "0.30.1" +version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6" +checksum = "225e7cfe711e0ba79a68baeddb2982723e4235247aefce1482f2f16c27865b66" dependencies = [ "bitflags 2.10.0", "cfg-if", @@ -5781,9 +5781,9 @@ dependencies = [ [[package]] name = "num-conv" -version = "0.1.0" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" +checksum = "cf97ec579c3c42f953ef76dbf8d55ac91fb219dde70e49aa4a6b7d74e9919050" [[package]] name = "num-format" @@ -5923,9 +5923,9 @@ dependencies = [ [[package]] name = "object_store" -version = "0.12.4" +version = "0.12.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c1be0c6c22ec0817cdc77d3842f721a17fd30ab6965001415b5402a74e6b740" +checksum = "fbfbfff40aeccab00ec8a910b57ca8ecf4319b335c542f2edcd19dd25a1e2a00" dependencies = [ "async-trait", "bytes", @@ -6185,8 +6185,8 @@ dependencies = [ "sha2 0.10.9", "thiserror 1.0.69", "tokio", - "windows 0.62.2", - "windows-strings 0.5.1", + "windows", + "windows-strings", ] [[package]] @@ -6215,7 +6215,7 @@ dependencies = [ "libc", "redox_syscall 0.5.18", "smallvec", - "windows-link 0.2.1", + "windows-link", ] [[package]] @@ -7162,9 +7162,9 @@ dependencies = [ [[package]] name = "rcgen" -version = "0.14.6" +version = "0.14.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ec0a99f2de91c3cddc84b37e7db80e4d96b743e05607f647eb236fc0455907f" +checksum = "10b99e0098aa4082912d4c649628623db6aba77335e4f4569ff5083a6448b32e" dependencies = [ "pem", "ring", @@ -7389,9 +7389,9 @@ dependencies = [ [[package]] name = "rmcp" -version = "0.13.0" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1815dbc06c414d720f8bc1951eccd66bc99efc6376331f1e7093a119b3eb508" +checksum = "0a621b37a548ff6ab6292d57841eb25785a7f146d89391a19c9f199414bd13da" dependencies = [ "async-trait", "base64", @@ -7411,9 +7411,9 @@ dependencies = [ [[package]] name = "rmcp-macros" -version = "0.13.0" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11f0bc7008fa102e771a76c6d2c9b253be3f2baa5964e060464d038ae1cbc573" +checksum = "6b79ed92303f9262db79575aa8c3652581668e9d136be6fd0b9ededa78954c95" dependencies = [ "darling 0.23.0", "proc-macro2", @@ -7483,9 +7483,9 @@ dependencies = [ [[package]] name = "russh" -version = "0.56.0" +version = "0.57.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bdbb7dcdd62c17ac911307ff693f55b3ec6712004d2d66ffdb8c0fa00269fd66" +checksum = "01fe22d10a0e39c1134a971d5b8db8a40357b48ef22d81fa8d6eac22202dd782" dependencies = [ "aes 0.8.4", "aws-lc-rs", @@ -7524,8 +7524,8 @@ dependencies = [ "pkcs1", "pkcs5", "pkcs8 0.10.2", - "rand 0.8.5", - "rand_core 0.6.4", + "rand 0.9.2", + "rand_core 0.10.0-rc-3", "rsa", "russh-cryptovec", "russh-util", @@ -8339,7 +8339,7 @@ dependencies = [ "lz4", "md-5 0.11.0-rc.3", "netif", - "nix 0.30.1", + "nix 0.31.1", "rand 0.10.0-rc.6", "regex", "rustfs-config", @@ -8359,7 +8359,7 @@ dependencies = [ "tracing", "transform-stream", "url", - "windows 0.62.2", + "windows", "zstd", ] @@ -8970,9 +8970,9 @@ dependencies = [ [[package]] name = "shadow-rs" -version = "1.5.0" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff351910f271e7065781b6b4f0f43cb515d474d812f31176a0246d9058e47d5d" +checksum = "9b6fc9c7545235794143a8a63c1388de47039e859b90973cdc69be3649350d0d" dependencies = [ "cargo_metadata", "const_format", @@ -9187,9 +9187,9 @@ checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b" [[package]] name = "socket2" -version = "0.6.1" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17129e116933cf371d018bb80ae557e889637989d8638274fb25622827b03881" +checksum = "86f4aa3ad99f2088c990dfa82d367e19cb29268ed67c574d10d0a4bfe71f07e0" dependencies = [ "libc", "windows-sys 0.60.2", @@ -9620,9 +9620,9 @@ dependencies = [ [[package]] name = "sysinfo" -version = "0.37.2" +version = "0.38.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16607d5caffd1c07ce073528f9ed972d88db15dd44023fa57142963be3feb11f" +checksum = "fe840c5b1afe259a5657392a4dbb74473a14c8db999c3ec2f4ae812e028a94da" dependencies = [ "libc", "memchr", @@ -9630,7 +9630,7 @@ dependencies = [ "objc2-core-foundation", "objc2-io-kit", "rayon", - "windows 0.61.3", + "windows", ] [[package]] @@ -9808,12 +9808,13 @@ dependencies = [ [[package]] name = "time" -version = "0.3.45" +version = "0.3.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9e442fc33d7fdb45aa9bfeb312c095964abdf596f7567261062b2a7107aaabd" +checksum = "9da98b7d9b7dad93488a84b8248efc35352b0b2657397d4167e7ad67e5d535e5" dependencies = [ "deranged", "itoa", + "js-sys", "libc", "num-conv", "num_threads", @@ -9825,15 +9826,15 @@ dependencies = [ [[package]] name = "time-core" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b36ee98fd31ec7426d599183e8fe26932a8dc1fb76ddb6214d05493377d34ca" +checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" [[package]] name = "time-macros" -version = "0.2.25" +version = "0.2.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71e552d1249bf61ac2a52db88179fd0673def1e1ad8243a00d9ec9ed71fee3dd" +checksum = "78cc610bac2dcee56805c99642447d4c5dbde4d01f752ffea0199aee1f601dc4" dependencies = [ "num-conv", "time-core", @@ -10279,6 +10280,12 @@ version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" +[[package]] +name = "typed-path" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e43ffa54726cdc9ea78392023ffe9fe9cf9ac779e1c6fcb0d23f9862e3879d20" + [[package]] name = "typeid" version = "1.0.3" @@ -10695,38 +10702,16 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" -[[package]] -name = "windows" -version = "0.61.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9babd3a767a4c1aef6900409f85f5d53ce2544ccdfaa86dad48c91782c6d6893" -dependencies = [ - "windows-collections 0.2.0", - "windows-core 0.61.2", - "windows-future 0.2.1", - "windows-link 0.1.3", - "windows-numerics 0.2.0", -] - [[package]] name = "windows" version = "0.62.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "527fadee13e0c05939a6a05d5bd6eec6cd2e3dbd648b9f8e447c6518133d8580" dependencies = [ - "windows-collections 0.3.2", - "windows-core 0.62.2", - "windows-future 0.3.2", - "windows-numerics 0.3.1", -] - -[[package]] -name = "windows-collections" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3beeceb5e5cfd9eb1d76b381630e82c4241ccd0d27f1a39ed41b2760b255c5e8" -dependencies = [ - "windows-core 0.61.2", + "windows-collections", + "windows-core", + "windows-future", + "windows-numerics", ] [[package]] @@ -10735,20 +10720,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23b2d95af1a8a14a3c7367e1ed4fc9c20e0a26e79551b1454d72583c97cc6610" dependencies = [ - "windows-core 0.62.2", -] - -[[package]] -name = "windows-core" -version = "0.61.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3" -dependencies = [ - "windows-implement", - "windows-interface", - "windows-link 0.1.3", - "windows-result 0.3.4", - "windows-strings 0.4.2", + "windows-core", ] [[package]] @@ -10759,20 +10731,9 @@ checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" dependencies = [ "windows-implement", "windows-interface", - "windows-link 0.2.1", - "windows-result 0.4.1", - "windows-strings 0.5.1", -] - -[[package]] -name = "windows-future" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc6a41e98427b19fe4b73c550f060b59fa592d7d686537eebf9385621bfbad8e" -dependencies = [ - "windows-core 0.61.2", - "windows-link 0.1.3", - "windows-threading 0.1.0", + "windows-link", + "windows-result", + "windows-strings", ] [[package]] @@ -10781,9 +10742,9 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e1d6f90251fe18a279739e78025bd6ddc52a7e22f921070ccdc67dde84c605cb" dependencies = [ - "windows-core 0.62.2", - "windows-link 0.2.1", - "windows-threading 0.2.1", + "windows-core", + "windows-link", + "windows-threading", ] [[package]] @@ -10808,36 +10769,20 @@ dependencies = [ "syn 2.0.114", ] -[[package]] -name = "windows-link" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a" - [[package]] name = "windows-link" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" -[[package]] -name = "windows-numerics" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9150af68066c4c5c07ddc0ce30421554771e528bde427614c61038bc2c92c2b1" -dependencies = [ - "windows-core 0.61.2", - "windows-link 0.1.3", -] - [[package]] name = "windows-numerics" version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e2e40844ac143cdb44aead537bbf727de9b044e107a0f1220392177d15b0f26" dependencies = [ - "windows-core 0.62.2", - "windows-link 0.2.1", + "windows-core", + "windows-link", ] [[package]] @@ -10846,18 +10791,9 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" dependencies = [ - "windows-link 0.2.1", - "windows-result 0.4.1", - "windows-strings 0.5.1", -] - -[[package]] -name = "windows-result" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6" -dependencies = [ - "windows-link 0.1.3", + "windows-link", + "windows-result", + "windows-strings", ] [[package]] @@ -10866,16 +10802,7 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" dependencies = [ - "windows-link 0.2.1", -] - -[[package]] -name = "windows-strings" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56e6c93f3a0c3b36176cb1327a4958a0353d5d166c2a35cb268ace15e91d3b57" -dependencies = [ - "windows-link 0.1.3", + "windows-link", ] [[package]] @@ -10884,7 +10811,7 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" dependencies = [ - "windows-link 0.2.1", + "windows-link", ] [[package]] @@ -10920,7 +10847,7 @@ version = "0.61.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" dependencies = [ - "windows-link 0.2.1", + "windows-link", ] [[package]] @@ -10945,7 +10872,7 @@ version = "0.53.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" dependencies = [ - "windows-link 0.2.1", + "windows-link", "windows_aarch64_gnullvm 0.53.1", "windows_aarch64_msvc 0.53.1", "windows_i686_gnu 0.53.1", @@ -10956,22 +10883,13 @@ dependencies = [ "windows_x86_64_msvc 0.53.1", ] -[[package]] -name = "windows-threading" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b66463ad2e0ea3bbf808b7f1d371311c80e115c0b71d60efc142cafbcfb057a6" -dependencies = [ - "windows-link 0.1.3", -] - [[package]] name = "windows-threading" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3949bd5b99cafdf1c7ca86b43ca564028dfe27d66958f2470940f73d86d75b37" dependencies = [ - "windows-link 0.2.1", + "windows-link", ] [[package]] @@ -11291,9 +11209,9 @@ dependencies = [ [[package]] name = "zip" -version = "7.1.0" +version = "7.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9013f1222db8a6d680f13a7ccdc60a781199cd09c2fa4eff58e728bb181757fc" +checksum = "c42e33efc22a0650c311c2ef19115ce232583abbe80850bc8b66509ebef02de0" dependencies = [ "aes 0.8.4", "bzip2", @@ -11311,6 +11229,7 @@ dependencies = [ "ppmd-rust", "sha1 0.10.6", "time", + "typed-path", "zeroize", "zopfli", "zstd", diff --git a/Cargo.toml b/Cargo.toml index eb570698d..e93fd235d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -114,7 +114,7 @@ http = "1.4.0" http-body = "1.0.1" http-body-util = "0.1.3" reqwest = { version = "0.12.28", default-features = false, features = ["rustls-tls-no-provider", "charset", "http2", "system-proxy", "stream", "json", "blocking"] } -socket2 = { version = "0.6.1", features = ["all"] } +socket2 = { version = "0.6.2", features = ["all"] } tokio = { version = "1.49.0", features = ["fs", "rt-multi-thread"] } tokio-rustls = { version = "0.26.4", default-features = false, features = ["logging", "tls12", "aws-lc-rs"] } tokio-stream = { version = "0.1.18" } @@ -134,7 +134,7 @@ flatbuffers = "25.12.19" form_urlencoded = "1.2.2" prost = "0.14.3" quick-xml = "0.39.0" -rmcp = { version = "0.13.0" } +rmcp = { version = "0.14.0" } rmp = { version = "0.8.15" } rmp-serde = { version = "1.3.1" } serde = { version = "1.0.228", features = ["derive"] } @@ -164,7 +164,7 @@ zeroize = { version = "1.8.2", features = ["derive"] } chrono = { version = "0.4.43", features = ["serde"] } humantime = "2.3.0" jiff = { version = "0.2.18", features = ["serde"] } -time = { version = "0.3.45", features = ["std", "parsing", "formatting", "macros", "serde"] } +time = { version = "0.3.46", features = ["std", "parsing", "formatting", "macros", "serde"] } # Utilities and Tools anyhow = "1.0.100" @@ -185,7 +185,7 @@ const-str = { version = "1.0.0", features = ["std", "proc"] } convert_case = "0.10.0" criterion = { version = "0.8", features = ["html_reports"] } crossbeam-queue = "0.3.12" -datafusion = "52.0.0" +datafusion = "52.1.0" derive_builder = "0.20.2" dunce = "1.0.5" enumset = "1.1.10" @@ -211,11 +211,11 @@ md5 = "0.8.0" mime_guess = "2.0.5" moka = { version = "0.12.12", features = ["future"] } netif = "0.1.6" -nix = { version = "0.30.1", features = ["fs"] } +nix = { version = "0.31.1", features = ["fs"] } nu-ansi-term = "0.50.3" num_cpus = { version = "1.17.0" } nvml-wrapper = "0.11.0" -object_store = "0.12.4" +object_store = "0.12.5" parking_lot = "0.12.5" path-absolutize = "3.1.1" path-clean = "1.0.1" @@ -230,7 +230,7 @@ rust-embed = { version = "8.11.0" } rustc-hash = { version = "2.1.1" } s3s = { version = "0.13.0-alpha.2", features = ["minio"], git = "https://github.com/s3s-project/s3s.git", branch = "main" } serial_test = "3.3.1" -shadow-rs = { version = "1.5.0", default-features = false } +shadow-rs = { version = "1.6.0", default-features = false } siphasher = "1.0.1" smallvec = { version = "1.15.1", features = ["serde"] } smartstring = "1.0.1" @@ -238,7 +238,7 @@ snafu = "0.8.9" snap = "1.1.1" starshard = { version = "0.6.0", features = ["rayon", "async", "serde"] } strum = { version = "0.27.2", features = ["derive"] } -sysinfo = "0.37.2" +sysinfo = "0.38.0" temp-env = "0.3.6" tempfile = "3.24.0" test-case = "3.3.1" @@ -257,7 +257,7 @@ walkdir = "2.5.0" wildmatch = { version = "2.6.1", features = ["serde"] } windows = { version = "0.62.2" } xxhash-rust = { version = "0.8.15", features = ["xxh64", "xxh3"] } -zip = "7.1.0" +zip = "7.2.0" zstd = "0.13.3" # Observability and Metrics @@ -271,11 +271,11 @@ opentelemetry-stdout = { version = "0.31.0" } # FTP and SFTP libunftp = { version = "0.21.0" } -russh = { version = "0.56.0", features = ["aws-lc-rs", "rsa"], default-features = false } +russh = { version = "0.57.0", features = ["aws-lc-rs", "rsa"], default-features = false } russh-sftp = { version = "2.1.1" } ssh-key = { version = "0.7.0-rc.6", features = ["std", "rsa", "ed25519"] } suppaftp = { version = "7.1.0", features = ["tokio", "tokio-rustls", "rustls"] } -rcgen = "0.14.6" +rcgen = "0.14.7" # Performance Analysis and Memory Profiling mimalloc = "0.1" diff --git a/rustfs/src/init.rs b/rustfs/src/init.rs index adb15dc7d..35cc20456 100644 --- a/rustfs/src/init.rs +++ b/rustfs/src/init.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::storage::ecfs::{process_lambda_configurations, process_queue_configurations, process_topic_configurations}; +use crate::storage::{process_lambda_configurations, process_queue_configurations, process_topic_configurations}; use crate::{admin, config, version}; use rustfs_config::{DEFAULT_UPDATE_CHECK, ENV_UPDATE_CHECK}; use rustfs_ecstore::bucket::metadata_sys; diff --git a/rustfs/src/server/layer.rs b/rustfs/src/server/layer.rs index 705798d37..ccc6d3d9f 100644 --- a/rustfs/src/server/layer.rs +++ b/rustfs/src/server/layer.rs @@ -16,7 +16,7 @@ use crate::admin::console::is_console_path; use crate::server::cors; use crate::server::hybrid::HybridBody; use crate::server::{ADMIN_PREFIX, RPC_PREFIX}; -use crate::storage::ecfs; +use crate::storage::apply_cors_headers; use http::{HeaderMap, HeaderValue, Method, Request as HttpRequest, Response, StatusCode}; use hyper::body::Incoming; use std::future::Future; @@ -229,7 +229,7 @@ where if ConditionalCorsLayer::is_s3_path(&path) && !bucket.is_empty() && cors_origins.is_some() - && let Some(cors_headers) = ecfs::apply_cors_headers(&bucket, &method_clone, &request_headers_clone).await + && let Some(cors_headers) = apply_cors_headers(&bucket, &method_clone, &request_headers_clone).await { for (key, value) in cors_headers.iter() { response.headers_mut().insert(key, value.clone()); diff --git a/rustfs/src/storage/ecfs.rs b/rustfs/src/storage/ecfs.rs index 553e54d7c..1a96ade56 100644 --- a/rustfs/src/storage/ecfs.rs +++ b/rustfs/src/storage/ecfs.rs @@ -13,12 +13,9 @@ // limitations under the License. use crate::auth::get_condition_values; -use crate::config::workload_profiles::{ - RustFSBufferConfig, WorkloadProfile, get_global_buffer_config, is_buffer_profile_enabled, -}; +use crate::config::workload_profiles::get_global_buffer_config; use crate::error::ApiError; use crate::server::RemoteAddr; -use crate::server::cors; use crate::storage::concurrency::{ CachedGetObject, ConcurrencyManager, GetObjectGuard, get_concurrency_aware_buffer_size, get_concurrency_manager, }; @@ -32,6 +29,13 @@ use crate::storage::{ get_complete_multipart_upload_opts, get_opts, parse_copy_source_range, put_opts, }, }; +use crate::storage::{ + check_preconditions, create_managed_encryption_material, decrypt_managed_encryption_key, decrypt_multipart_managed_stream, + derive_part_nonce, get_buffer_size_opt_in, get_validated_store, has_replication_rules, is_managed_sse, + parse_object_lock_legal_hold, parse_object_lock_retention, process_lambda_configurations, process_queue_configurations, + process_topic_configurations, strip_managed_encryption_metadata, validate_bucket_object_lock_enabled, + validate_list_object_unordered_with_delimiter, validate_object_key, wrap_response_with_cors, +}; use base64::{Engine, engine::general_purpose::STANDARD as BASE64_STANDARD}; use bytes::Bytes; use datafusion::arrow::{ @@ -57,8 +61,8 @@ use rustfs_ecstore::{ policy_sys::PolicySys, quota::QuotaOperation, replication::{ - DeletedObjectReplicationInfo, ReplicationConfigurationExt, check_replicate_delete, get_must_replicate_options, - must_replicate, schedule_replication, schedule_replication_delete, + DeletedObjectReplicationInfo, check_replicate_delete, get_must_replicate_options, must_replicate, + schedule_replication, schedule_replication_delete, }, tagging::{decode_tags, encode_tags}, utils::serialize, @@ -88,13 +92,9 @@ use rustfs_ecstore::{ }, }; use rustfs_filemeta::REPLICATE_INCOMING_DELETE; -use rustfs_filemeta::{ObjectPartInfo, RestoreStatusOps}; +use rustfs_filemeta::RestoreStatusOps; use rustfs_filemeta::{ReplicationStatusType, ReplicationType, VersionPurgeStatusType}; -use rustfs_kms::{ - DataKey, - service_manager::get_global_encryption_service, - types::{EncryptionMetadata, ObjectEncryptionContext}, -}; +use rustfs_kms::DataKey; use rustfs_notify::{EventArgsBuilder, notifier_global}; use rustfs_policy::policy::{ action::{Action, S3Action}, @@ -108,7 +108,7 @@ use rustfs_s3select_api::{ use rustfs_s3select_query::get_global_db; use rustfs_targets::{ EventName, - arn::{ARN, TargetID, TargetIDError}, + arn::{ARN, TargetIDError}, }; use rustfs_utils::{ CompressionAlgorithm, extract_req_params_header, extract_resp_elements, get_request_host, get_request_user_agent, @@ -126,7 +126,6 @@ use rustfs_utils::{ use rustfs_zip::CompressionFormat; use s3s::header::{X_AMZ_RESTORE, X_AMZ_RESTORE_OUTPUT_PATH}; use s3s::{S3, S3Error, S3ErrorCode, S3Request, S3Response, S3Result, dto::*, s3_error}; -use serde_urlencoded::from_bytes; use std::convert::Infallible; use std::ops::Add; use std::{ @@ -136,7 +135,7 @@ use std::{ str::FromStr, sync::{Arc, LazyLock}, }; -use time::{OffsetDateTime, format_description::well_known::Rfc2822, format_description::well_known::Rfc3339}; +use time::{OffsetDateTime, format_description::well_known::Rfc3339}; use tokio::{ io::{AsyncRead, AsyncSeek}, sync::mpsc, @@ -159,238 +158,34 @@ macro_rules! try_ { }; } -static RUSTFS_OWNER: LazyLock = LazyLock::new(|| Owner { +pub(crate) static RUSTFS_OWNER: LazyLock = LazyLock::new(|| Owner { display_name: Some("rustfs".to_owned()), id: Some("c19050dbcee97fda828689dda99097a6321af2248fa760517237346e5d9c8a66".to_owned()), }); -/// Calculate adaptive buffer size with workload profile support. -/// -/// This enhanced version supports different workload profiles for optimal performance -/// across various use cases (AI/ML, web workloads, secure storage, etc.). -/// -/// # Arguments -/// * `file_size` - The size of the file in bytes, or -1 if unknown -/// * `profile` - Optional workload profile. If None, uses auto-detection or GeneralPurpose -/// -/// # Returns -/// Optimal buffer size in bytes based on the workload profile and file size -/// -/// # Examples -/// ```ignore -/// // Use general purpose profile (default) -/// let buffer_size = get_adaptive_buffer_size_with_profile(1024 * 1024, None); -/// -/// // Use AI training profile for large model files -/// let buffer_size = get_adaptive_buffer_size_with_profile( -/// 500 * 1024 * 1024, -/// Some(WorkloadProfile::AiTraining) -/// ); -/// -/// // Use secure storage profile for compliance scenarios -/// let buffer_size = get_adaptive_buffer_size_with_profile( -/// 10 * 1024 * 1024, -/// Some(WorkloadProfile::SecureStorage) -/// ); -/// ``` -/// -#[allow(dead_code)] -fn get_adaptive_buffer_size_with_profile(file_size: i64, profile: Option) -> usize { - let config = match profile { - Some(p) => RustFSBufferConfig::new(p), - None => { - // Auto-detect OS environment or use general purpose - RustFSBufferConfig::with_auto_detect() - } - }; - - config.get_buffer_size(file_size) -} - -/// Get adaptive buffer size using global workload profile configuration. -/// -/// This is the primary buffer sizing function that uses the workload profile -/// system configured at startup to provide optimal buffer sizes for different scenarios. -/// -/// The function automatically selects buffer sizes based on: -/// - Configured workload profile (default: GeneralPurpose) -/// - File size characteristics -/// - Optional performance metrics collection -/// -/// # Arguments -/// * `file_size` - The size of the file in bytes, or -1 if unknown -/// -/// # Returns -/// Optimal buffer size in bytes based on the configured workload profile -/// -/// # Performance Metrics -/// When compiled with the `metrics` feature flag, this function tracks: -/// - Buffer size distribution -/// - Selection frequency -/// - Buffer-to-file size ratios -/// -/// # Examples -/// ```ignore -/// // Uses configured profile (default: GeneralPurpose) -/// let buffer_size = get_buffer_size_opt_in(file_size); -/// ``` -fn get_buffer_size_opt_in(file_size: i64) -> usize { - let buffer_size = if is_buffer_profile_enabled() { - // Use globally configured workload profile (enabled by default in Phase 3) - let config = get_global_buffer_config(); - config.get_buffer_size(file_size) - } else { - // Opt-out mode: Use GeneralPurpose profile for consistent behavior - let config = RustFSBufferConfig::new(WorkloadProfile::GeneralPurpose); - config.get_buffer_size(file_size) - }; - - // Optional performance metrics collection for monitoring and optimization - #[cfg(feature = "metrics")] - { - use metrics::histogram; - histogram!("rustfs.buffer.size.bytes").record(buffer_size as f64); - counter!("rustfs.buffer.size.selections").increment(1); - - if file_size >= 0 { - let ratio = buffer_size as f64 / file_size as f64; - histogram!("rustfs.buffer.to.file.ratio").record(ratio); - } - } - - buffer_size -} - #[derive(Debug, Clone)] pub struct FS { // pub store: ECStore, } -struct ManagedEncryptionMaterial { - data_key: DataKey, - headers: HashMap, - kms_key_id: String, -} - -async fn create_managed_encryption_material( - bucket: &str, - key: &str, - algorithm: &ServerSideEncryption, - kms_key_id: Option, - original_size: i64, -) -> Result { - let Some(service) = get_global_encryption_service().await else { - return Err(ApiError::from(StorageError::other("KMS encryption service is not initialized"))); - }; - - if !is_managed_sse(algorithm) { - return Err(ApiError::from(StorageError::other(format!( - "Unsupported server-side encryption algorithm: {}", - algorithm.as_str() - )))); - } - - let algorithm_str = algorithm.as_str(); - - let mut context = ObjectEncryptionContext::new(bucket.to_string(), key.to_string()); - if original_size >= 0 { - context = context.with_size(original_size as u64); - } - - let mut kms_key_candidate = kms_key_id; - if kms_key_candidate.is_none() { - kms_key_candidate = service.get_default_key_id().cloned(); - } - - let kms_key_to_use = kms_key_candidate - .clone() - .ok_or_else(|| ApiError::from(StorageError::other("No KMS key available for managed server-side encryption")))?; - - let (data_key, encrypted_data_key) = service - .create_data_key(&kms_key_candidate, &context) - .await - .map_err(|e| ApiError::from(StorageError::other(format!("Failed to create data key: {e}"))))?; - - let metadata = EncryptionMetadata { - algorithm: algorithm_str.to_string(), - key_id: kms_key_to_use.clone(), - key_version: 1, - iv: data_key.nonce.to_vec(), - tag: None, - encryption_context: context.encryption_context.clone(), - encrypted_at: jiff::Zoned::now(), - original_size: if original_size >= 0 { original_size as u64 } else { 0 }, - encrypted_data_key, - }; - - let mut headers = service.metadata_to_headers(&metadata); - headers.insert("x-rustfs-encryption-original-size".to_string(), metadata.original_size.to_string()); - - Ok(ManagedEncryptionMaterial { - data_key, - headers, - kms_key_id: kms_key_to_use, - }) -} - -async fn decrypt_managed_encryption_key( - bucket: &str, - key: &str, - metadata: &HashMap, -) -> Result)>, ApiError> { - if !metadata.contains_key("x-rustfs-encryption-key") { - return Ok(None); - } - - let Some(service) = get_global_encryption_service().await else { - return Err(ApiError::from(StorageError::other("KMS encryption service is not initialized"))); - }; - - let parsed = service - .headers_to_metadata(metadata) - .map_err(|e| ApiError::from(StorageError::other(format!("Failed to parse encryption metadata: {e}"))))?; - - if parsed.iv.len() != 12 { - return Err(ApiError::from(StorageError::other("Invalid encryption nonce length; expected 12 bytes"))); - } - - let context = ObjectEncryptionContext::new(bucket.to_string(), key.to_string()); - let data_key = service - .decrypt_data_key(&parsed.encrypted_data_key, &context) - .await - .map_err(|e| ApiError::from(StorageError::other(format!("Failed to decrypt data key: {e}"))))?; - - let key_bytes = data_key.plaintext_key; - let mut nonce = [0u8; 12]; - nonce.copy_from_slice(&parsed.iv[..12]); - - let original_size = metadata - .get("x-rustfs-encryption-original-size") - .and_then(|s| s.parse::().ok()); - - Ok(Some((key_bytes, nonce, original_size))) -} - -fn derive_part_nonce(base: [u8; 12], part_number: usize) -> [u8; 12] { - let mut nonce = base; - let current = u32::from_be_bytes([nonce[8], nonce[9], nonce[10], nonce[11]]); - let incremented = current.wrapping_add(part_number as u32); - nonce[8..12].copy_from_slice(&incremented.to_be_bytes()); - nonce +pub(crate) struct ManagedEncryptionMaterial { + pub(crate) data_key: DataKey, + pub(crate) headers: HashMap, + pub(crate) kms_key_id: String, } #[derive(Debug, Default, serde::Deserialize)] -struct ListObjectUnorderedQuery { +pub(crate) struct ListObjectUnorderedQuery { #[serde(rename = "allow-unordered")] - allow_unordered: Option, + pub(crate) allow_unordered: Option, } -struct InMemoryAsyncReader { +pub(crate) struct InMemoryAsyncReader { cursor: std::io::Cursor>, } impl InMemoryAsyncReader { - fn new(data: Vec) -> Self { + pub(crate) fn new(data: Vec) -> Self { Self { cursor: std::io::Cursor::new(data), } @@ -423,386 +218,6 @@ impl AsyncSeek for InMemoryAsyncReader { } } -async fn decrypt_multipart_managed_stream( - mut encrypted_stream: Box, - parts: &[ObjectPartInfo], - key_bytes: [u8; 32], - base_nonce: [u8; 12], -) -> Result<(Box, i64), StorageError> { - let total_plain_capacity: usize = parts.iter().map(|part| part.actual_size.max(0) as usize).sum(); - - let mut plaintext = Vec::with_capacity(total_plain_capacity); - - for part in parts { - if part.size == 0 { - continue; - } - - let mut encrypted_part = vec![0u8; part.size]; - tokio::io::AsyncReadExt::read_exact(&mut encrypted_stream, &mut encrypted_part) - .await - .map_err(|e| StorageError::other(format!("failed to read encrypted multipart segment {}: {}", part.number, e)))?; - - let part_nonce = derive_part_nonce(base_nonce, part.number); - let cursor = std::io::Cursor::new(encrypted_part); - let mut decrypt_reader = DecryptReader::new(WarpReader::new(cursor), key_bytes, part_nonce); - - tokio::io::AsyncReadExt::read_to_end(&mut decrypt_reader, &mut plaintext) - .await - .map_err(|e| StorageError::other(format!("failed to decrypt multipart segment {}: {}", part.number, e)))?; - } - - let total_plain_size = plaintext.len() as i64; - let reader = Box::new(WarpReader::new(InMemoryAsyncReader::new(plaintext))) as Box; - - Ok((reader, total_plain_size)) -} - -fn strip_managed_encryption_metadata(metadata: &mut HashMap) { - const KEYS: [&str; 7] = [ - "x-amz-server-side-encryption", - "x-amz-server-side-encryption-aws-kms-key-id", - "x-rustfs-encryption-iv", - "x-rustfs-encryption-tag", - "x-rustfs-encryption-key", - "x-rustfs-encryption-context", - "x-rustfs-encryption-original-size", - ]; - - for key in KEYS.iter() { - metadata.remove(*key); - } -} - -fn is_managed_sse(algorithm: &ServerSideEncryption) -> bool { - matches!(algorithm.as_str(), "AES256" | "aws:kms") -} - -/// Validate object key for control characters and log special characters -/// -/// This function: -/// 1. Rejects keys containing control characters (null bytes, newlines, carriage returns) -/// 2. Logs debug information for keys containing spaces, plus signs, or percent signs -/// -/// The s3s library handles URL decoding, so keys are already decoded when they reach this function. -/// This validation ensures that invalid characters that could cause issues are rejected early. -fn validate_object_key(key: &str, operation: &str) -> S3Result<()> { - // Validate object key doesn't contain control characters - if key.contains(['\0', '\n', '\r']) { - return Err(S3Error::with_message( - S3ErrorCode::InvalidArgument, - format!("Object key contains invalid control characters: {key:?}"), - )); - } - - // Log debug info for keys with special characters to help diagnose encoding issues - if key.contains([' ', '+', '%']) { - debug!("{} object with special characters in key: {:?}", operation, key); - } - - Ok(()) -} - -/// Validate that 'allow-unordered' parameter is not used with a delimiter -/// -/// This function: -/// 1. Checks if a delimiter is specified in the ListObjects request -/// 2. Parses the query string to check for the 'allow-unordered' parameter -/// 3. Rejects the request if both 'delimiter' and 'allow-unordered=true' are present -/// -/// According to S3 compatibility requirements, unordered listing cannot be combined with -/// hierarchical directory traversal (delimited listing). This validation ensures -/// conflicting parameters are caught before processing the request. -fn validate_list_object_unordered_with_delimiter(delimiter: Option<&Delimiter>, query_string: Option<&str>) -> S3Result<()> { - if delimiter.is_none() { - return Ok(()); - } - - let Some(query) = query_string else { - return Ok(()); - }; - - if let Ok(params) = from_bytes::(query.as_bytes()) - && params.allow_unordered.as_deref() == Some("true") - { - return Err(S3Error::with_message( - S3ErrorCode::InvalidArgument, - "The allow-unordered parameter cannot be used when delimiter is specified.".to_string(), - )); - } - - Ok(()) -} - -fn parse_object_lock_retention(retention: Option) -> S3Result> { - let mut eval_metadata = HashMap::new(); - - if let Some(v) = retention { - let mode = match v.mode { - Some(mode) => match mode.as_str() { - ObjectLockRetentionMode::COMPLIANCE | ObjectLockRetentionMode::GOVERNANCE => mode.as_str().to_string(), - _ => { - return Err(S3Error::with_message( - S3ErrorCode::MalformedXML, - "The XML you provided was not well-formed or did not validate against our published schema".to_string(), - )); - } - }, - None => String::default(), - }; - - let now = OffsetDateTime::now_utc(); - - // Validate retain_until_date is in the future (S3 requirement) - // Only validate when both mode and date are provided (not clearing retention) - let retain_until_date = if let Some(date) = v.retain_until_date { - let retain_until = OffsetDateTime::from(date); - // Only validate future date when mode is set (not clearing retention) - if !mode.is_empty() && retain_until <= now { - return Err(S3Error::with_message( - S3ErrorCode::InvalidArgument, - "The retain until date must be in the future".to_string(), - )); - } - retain_until.format(&Rfc3339).unwrap() - } else { - String::default() - }; - - // This is intentional behavior. Empty string represents "retention cleared" which is different from "retention never set". Consistent with minio - eval_metadata.insert(AMZ_OBJECT_LOCK_MODE_LOWER.to_string(), mode); - eval_metadata.insert(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER.to_string(), retain_until_date); - eval_metadata.insert( - format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, "objectlock-retention-timestamp"), - format!("{}.{:09}Z", now.format(&Rfc3339).unwrap(), now.nanosecond()), - ); - } - Ok(eval_metadata) -} - -fn parse_object_lock_legal_hold(legal_hold: Option) -> S3Result> { - let mut eval_metadata = HashMap::new(); - if let Some(v) = legal_hold { - let status = match v.status { - Some(status) => match status.as_str() { - ObjectLockLegalHoldStatus::OFF | ObjectLockLegalHoldStatus::ON => status.as_str().to_string(), - _ => { - return Err(S3Error::with_message( - S3ErrorCode::MalformedXML, - "The XML you provided was not well-formed or did not validate against our published schema".to_string(), - )); - } - }, - None => String::default(), - }; - let now = OffsetDateTime::now_utc(); - // This is intentional behavior. Empty string represents "status cleared" which is different from "status never set". Consistent with minio - eval_metadata.insert(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER.to_string(), status); - eval_metadata.insert( - format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, "objectlock-legalhold-timestamp"), - format!("{}.{:09}Z", now.format(&Rfc3339).unwrap(), now.nanosecond()), - ); - } - Ok(eval_metadata) -} - -async fn validate_bucket_object_lock_enabled(bucket: &str) -> S3Result<()> { - match metadata_sys::get_object_lock_config(bucket).await { - Ok((cfg, _created)) => { - if cfg.object_lock_enabled != Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)) { - return Err(S3Error::with_message( - S3ErrorCode::InvalidRequest, - "Object Lock is not enabled for this bucket".to_string(), - )); - } - } - Err(err) => { - if err == StorageError::ConfigNotFound { - return Err(S3Error::with_message( - S3ErrorCode::InvalidRequest, - "Bucket is missing ObjectLockConfiguration".to_string(), - )); - } - warn!("get_object_lock_config err {:?}", err); - return Err(S3Error::with_message( - S3ErrorCode::InternalError, - "Failed to get bucket ObjectLockConfiguration".to_string(), - )); - } - } - Ok(()) -} - -/// Validates HTTP conditional request headers for a single object according to -/// RFC 7232 and S3 API semantics. -/// -/// This function evaluates the following headers, if present, in the standard -/// conditional request order: -/// - `If-Match` -/// - `If-None-Match` -/// - `If-Modified-Since` -/// - `If-Unmodified-Since` -/// -/// The `headers` parameter provides the incoming HTTP request headers, and -/// `info` supplies the object's current metadata (ETag and modification time) -/// used to evaluate those preconditions. -/// -/// The function returns `Ok(())` if either no relevant conditional headers are -/// set or all specified preconditions are satisfied. If any precondition fails, -/// it returns an `Err(S3Error)` with an appropriate S3 error code (for example, -/// `PreconditionFailed` for failed `If-Match` / `If-Unmodified-Since`, or -/// `NotModified` for `If-None-Match` / `If-Modified-Since` when the resource -/// has not changed), allowing the caller to translate this into the correct -/// HTTP response. -fn check_preconditions(headers: &HeaderMap, info: &ObjectInfo) -> S3Result<()> { - let mod_time = info.mod_time; - let etag = info.etag.as_deref(); - - if mod_time.is_none() && etag.is_none() { - return Ok(()); - } - - // If-Match: requires ETag to exist - if let Some(if_match_val) = headers.get("if-match").and_then(|v| v.to_str().ok()) { - match etag { - Some(e) if is_etag_equal(e, if_match_val) => {} - _ => return Err(S3Error::new(S3ErrorCode::PreconditionFailed)), - } - } - - // If-Unmodified-Since (only when If-Match is absent) - if headers.get("if-match").is_none() - && let Some(t) = mod_time - && let Some(if_unmodified_since) = headers.get("if-unmodified-since").and_then(|v| v.to_str().ok()) - && let Ok(given_time) = OffsetDateTime::parse(if_unmodified_since, &Rfc2822) - && t > given_time.add(time::Duration::seconds(1)) - { - return Err(S3Error::new(S3ErrorCode::PreconditionFailed)); - } - - // If-None-Match - if let Some(if_none_match) = headers.get("if-none-match").and_then(|v| v.to_str().ok()) - && let Some(e) = etag - && is_etag_equal(e, if_none_match) - { - let mut error_headers = HeaderMap::new(); - if let Ok(etag_header) = parse_etag(e) { - error_headers.insert("etag", etag_header); - } - if let Some(t) = mod_time - && let Ok(last_modified_str) = t.format(&Rfc2822) - && let Ok(last_modified_header) = HeaderValue::from_str(&last_modified_str) - { - error_headers.insert("last-modified", last_modified_header); - } - - let mut s3_error = S3Error::new(S3ErrorCode::NotModified); - s3_error.set_message("Not Modified".to_string()); - s3_error.set_status_code(StatusCode::NOT_MODIFIED); - s3_error.set_headers(error_headers); - return Err(s3_error); - } - - // If-Modified-Since (only when If-None-Match is absent - RFC 7232) - if headers.get("if-none-match").is_none() - && let Some(t) = mod_time - && let Some(if_modified_since) = headers.get("if-modified-since").and_then(|v| v.to_str().ok()) - && let Ok(given_time) = OffsetDateTime::parse(if_modified_since, &Rfc2822) - && t < given_time.add(time::Duration::seconds(1)) - { - let mut error_headers = HeaderMap::new(); - if let Some(e) = etag - && let Ok(etag_header) = parse_etag(e) - { - error_headers.insert("etag", etag_header); - } - if let Ok(last_modified_str) = t.format(&Rfc2822) - && let Ok(last_modified_header) = HeaderValue::from_str(&last_modified_str) - { - error_headers.insert("last-modified", last_modified_header); - } - - let mut s3_error = S3Error::new(S3ErrorCode::NotModified); - s3_error.set_message("Not Modified".to_string()); - s3_error.set_status_code(StatusCode::NOT_MODIFIED); - s3_error.set_headers(error_headers); - - return Err(s3_error); - } - - Ok(()) -} - -/// Compares an object ETag with an ETag value from an HTTP header. -/// -/// This helper implements HTTP ETag comparison semantics for headers such as -/// `If-Match` and `If-None-Match`: -/// - Supports the wildcard `*`, which matches any `object_etag`. -/// - Supports comma-separated ETag lists (e.g., `"etag1", "etag2"`), returning -/// `true` if any entry matches `object_etag`. -/// - Automatically trims surrounding whitespace and double quotes from both the -/// header entries and `object_etag` before comparison. -/// -/// # Parameters -/// - `object_etag`: The ETag associated with the stored object. -/// - `header_etag`: The raw ETag header value received in the request, which may -/// be a wildcard, a single ETag, or a comma-separated list of ETags. -/// -/// # Returns -/// `true` if the header value matches the object ETag according to the above -/// HTTP ETag comparison rules, otherwise `false`. -fn is_etag_equal(object_etag: &str, header_etag: &str) -> bool { - let header_etag = header_etag.trim(); - if header_etag == "*" { - return true; - } - header_etag - .split(',') - .map(|s| s.trim().trim_matches('"')) - .any(|e| e == object_etag.trim_matches('"')) -} - -/// Converts an object ETag string into an HTTP `HeaderValue` for use in response headers. -/// -/// This function normalizes ETag values by ensuring they are wrapped in double quotes, -/// which is the standard HTTP ETag format according to RFC 7232. If the input ETag -/// already contains quotes, they are removed before adding new quotes to avoid duplication. -/// -/// # Parameters -/// - `object_etag`: The ETag string, which may or may not already be wrapped in quotes. -/// Must not be empty after removing quotes. -/// -/// # Returns -/// - `Ok(HeaderValue)`: The normalized ETag value wrapped in double quotes, ready for use -/// in HTTP response headers. -/// - `Err(S3Error)`: Returns `InvalidArgument` if the ETag value is empty (after removing quotes) -/// or contains characters that cannot be represented in an HTTP header value. -/// -/// # Note -/// When this function returns an error, callers should log the error but continue with -/// the operation (e.g., return NotModified without the ETag header) rather than propagating -/// the InvalidArgument error, as the original operation semantics should be preserved. -pub fn parse_etag(object_etag: &str) -> Result { - let etag_trimmed = object_etag.trim_matches('"'); - - if etag_trimmed.trim().is_empty() { - return Err(S3Error::with_message(S3ErrorCode::InvalidArgument, "ETag cannot be empty".to_string())); - } - - let etag_quoted = format!("\"{}\"", etag_trimmed); - - HeaderValue::from_str(&etag_quoted).map_err(|e| { - warn!( - "Failed to convert ETag to HeaderValue (ETag: {}): {}. The ETag header will be omitted from the response.", - etag_quoted, e - ); - S3Error::with_message( - S3ErrorCode::InvalidArgument, - format!("Invalid ETag header value (ETag: {}): {}", etag_quoted, e), - ) - }) -} - impl FS { pub fn new() -> Self { // let store: ECStore = ECStore::new(address, endpoint_pools).await?; @@ -1073,258 +488,309 @@ impl FS { } } -/// Helper function to get store and validate bucket exists -async fn get_validated_store(bucket: &str) -> S3Result> { - let Some(store) = new_object_layer_fn() else { - return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); - }; - - // Validate bucket exists - store - .get_bucket_info(bucket, &BucketOptions::default()) - .await - .map_err(ApiError::from)?; - - Ok(store) -} - -/// Quick check if CORS processing is needed (lightweight check for Origin header) -/// This avoids unnecessary function calls for non-CORS requests -#[inline] -fn needs_cors_processing(headers: &HeaderMap) -> bool { - headers.contains_key(cors::standard::ORIGIN) -} - -/// Apply CORS headers to response based on bucket CORS configuration and request origin -/// -/// This function: -/// 1. Reads the Origin header from the request -/// 2. Retrieves the bucket's CORS configuration -/// 3. Matches the origin against CORS rules -/// 4. Validates AllowedHeaders if request headers are present -/// 5. Returns headers to add to the response if a match is found -/// -/// Note: This function should only be called if `needs_cors_processing()` returns true -/// to avoid unnecessary overhead for non-CORS requests. -pub(crate) async fn apply_cors_headers(bucket: &str, method: &http::Method, headers: &HeaderMap) -> Option { - use http::HeaderValue; - - // Get Origin header from request - let origin = headers.get(cors::standard::ORIGIN)?.to_str().ok()?; - - // Get CORS configuration for the bucket - let cors_config = match metadata_sys::get_cors_config(bucket).await { - Ok((config, _)) => config, - Err(_) => return None, // No CORS config, no headers to add - }; - - // Early return if no CORS rules configured - if cors_config.cors_rules.is_empty() { - return None; - } - - // Check if method is supported and get its string representation - const SUPPORTED_METHODS: &[&str] = &["GET", "PUT", "POST", "DELETE", "HEAD", "OPTIONS"]; - let method_str = method.as_str(); - if !SUPPORTED_METHODS.contains(&method_str) { - return None; - } - - // For OPTIONS (preflight) requests, check Access-Control-Request-Method - let is_preflight = method == http::Method::OPTIONS; - let requested_method = if is_preflight { - headers - .get(cors::request::ACCESS_CONTROL_REQUEST_METHOD) - .and_then(|v| v.to_str().ok()) - .unwrap_or(method_str) - } else { - method_str - }; - - // Get requested headers from preflight request - let requested_headers = if is_preflight { - headers - .get(cors::request::ACCESS_CONTROL_REQUEST_HEADERS) - .and_then(|v| v.to_str().ok()) - .map(|h| h.split(',').map(|s| s.trim().to_lowercase()).collect::>()) - } else { - None - }; - - // Find matching CORS rule - for rule in cors_config.cors_rules.iter() { - // Check if origin matches - let origin_matches = rule.allowed_origins.iter().any(|allowed_origin| { - if allowed_origin == "*" { - true - } else { - // Exact match or pattern match (support wildcards like https://*.example.com) - allowed_origin == origin || matches_origin_pattern(allowed_origin, origin) - } - }); - - if !origin_matches { - continue; - } - - // Check if method is allowed - let method_allowed = rule - .allowed_methods - .iter() - .any(|allowed_method| allowed_method.as_str() == requested_method); - - if !method_allowed { - continue; - } - - // Validate AllowedHeaders if present in the request - if let Some(ref req_headers) = requested_headers { - if let Some(ref allowed_headers) = rule.allowed_headers { - // Check if all requested headers are allowed - let all_headers_allowed = req_headers.iter().all(|req_header| { - allowed_headers.iter().any(|allowed_header| { - let allowed_lower = allowed_header.to_lowercase(); - // "*" allows all headers, or exact match - allowed_lower == "*" || allowed_lower == *req_header - }) - }); - - if !all_headers_allowed { - // If not all headers are allowed, skip this rule - continue; - } - } else if !req_headers.is_empty() { - // If no AllowedHeaders specified but headers were requested, skip this rule - // Unless the rule explicitly allows all headers - continue; - } - } - - // Found matching rule, build response headers - let mut response_headers = HeaderMap::new(); - - // Access-Control-Allow-Origin - // If origin is "*", use "*", otherwise echo back the origin - let has_wildcard_origin = rule.allowed_origins.iter().any(|o| o == "*"); - if has_wildcard_origin { - response_headers.insert(cors::response::ACCESS_CONTROL_ALLOW_ORIGIN, HeaderValue::from_static("*")); - } else if let Ok(origin_value) = HeaderValue::from_str(origin) { - response_headers.insert(cors::response::ACCESS_CONTROL_ALLOW_ORIGIN, origin_value); - } - - // Vary: Origin (required for caching, except when using wildcard) - if !has_wildcard_origin { - response_headers.insert(cors::standard::VARY, HeaderValue::from_static("Origin")); - } - - // Access-Control-Allow-Methods (required for preflight) - if is_preflight || !rule.allowed_methods.is_empty() { - let methods_str = rule.allowed_methods.iter().map(|m| m.as_str()).collect::>().join(", "); - if let Ok(methods_value) = HeaderValue::from_str(&methods_str) { - response_headers.insert(cors::response::ACCESS_CONTROL_ALLOW_METHODS, methods_value); - } - } - - // Access-Control-Allow-Headers (required for preflight if headers were requested) - if is_preflight && let Some(ref allowed_headers) = rule.allowed_headers { - let headers_str = allowed_headers.iter().map(|h| h.as_str()).collect::>().join(", "); - if let Ok(headers_value) = HeaderValue::from_str(&headers_str) { - response_headers.insert(cors::response::ACCESS_CONTROL_ALLOW_HEADERS, headers_value); - } - } - - // Access-Control-Expose-Headers (for actual requests) - if !is_preflight && let Some(ref expose_headers) = rule.expose_headers { - let expose_headers_str = expose_headers.iter().map(|h| h.as_str()).collect::>().join(", "); - if let Ok(expose_value) = HeaderValue::from_str(&expose_headers_str) { - response_headers.insert(cors::response::ACCESS_CONTROL_EXPOSE_HEADERS, expose_value); - } - } - - // Access-Control-Max-Age (for preflight requests) - if is_preflight - && let Some(max_age) = rule.max_age_seconds - && let Ok(max_age_value) = HeaderValue::from_str(&max_age.to_string()) - { - response_headers.insert(cors::response::ACCESS_CONTROL_MAX_AGE, max_age_value); - } - - return Some(response_headers); - } - - None // No matching rule found -} -/// Check if an origin matches a pattern (supports wildcards like https://*.example.com) -fn matches_origin_pattern(pattern: &str, origin: &str) -> bool { - // Simple wildcard matching: * matches any sequence - if pattern.contains('*') { - let pattern_parts: Vec<&str> = pattern.split('*').collect(); - if pattern_parts.len() == 2 { - origin.starts_with(pattern_parts[0]) && origin.ends_with(pattern_parts[1]) - } else { - false - } - } else { - pattern == origin - } -} - -/// Wrap S3Response with CORS headers if needed -/// This function performs a lightweight check first to avoid unnecessary CORS processing -/// for non-CORS requests (requests without Origin header) -async fn wrap_response_with_cors(bucket: &str, method: &http::Method, headers: &HeaderMap, output: T) -> S3Response { - let mut response = S3Response::new(output); - - // Quick check: only process CORS if Origin header is present - if needs_cors_processing(headers) - && let Some(cors_headers) = apply_cors_headers(bucket, method, headers).await - { - for (key, value) in cors_headers.iter() { - response.headers.insert(key, value.clone()); - } - } - - response -} - #[async_trait::async_trait] impl S3 for FS { - #[instrument( - level = "debug", - skip(self, req), - fields(start_time=?time::OffsetDateTime::now_utc()) - )] - async fn create_bucket(&self, req: S3Request) -> S3Result> { - let helper = OperationHelper::new(&req, EventName::BucketCreated, "s3:CreateBucket"); - let CreateBucketInput { - bucket, - object_lock_enabled_for_bucket, - .. + #[instrument(level = "debug", skip(self))] + async fn abort_multipart_upload( + &self, + req: S3Request, + ) -> S3Result> { + let AbortMultipartUploadInput { + bucket, key, upload_id, .. } = req.input; let Some(store) = new_object_layer_fn() else { return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); }; - counter!("rustfs_create_bucket_total").increment(1); + let opts = &ObjectOptions::default(); - store - .make_bucket( - &bucket, - &MakeBucketOptions { - force_create: false, // TODO: force support - lock_enabled: object_lock_enabled_for_bucket.is_some_and(|v| v), - ..Default::default() - }, - ) + // Special handling for abort_multipart_upload: Per AWS S3 API specification, this operation + // should return NoSuchUpload (404) when the upload_id doesn't exist, even if the format + // appears invalid. This differs from other multipart operations (upload_part, list_parts, + // complete_multipart_upload) which return InvalidArgument for malformed upload_ids. + // The lenient validation matches AWS S3 behavior where format validation is relaxed for + // abort operations to avoid leaking information about upload_id format requirements. + match store + .abort_multipart_upload(bucket.as_str(), key.as_str(), upload_id.as_str(), opts) + .await + { + Ok(_) => Ok(S3Response::new(AbortMultipartUploadOutput { ..Default::default() })), + Err(err) => { + // Convert MalformedUploadID to NoSuchUpload for S3 API compatibility + if matches!(err, StorageError::MalformedUploadID(_)) { + return Err(S3Error::new(S3ErrorCode::NoSuchUpload)); + } + Err(ApiError::from(err).into()) + } + } + } + + #[instrument(level = "debug", skip(self, req))] + async fn complete_multipart_upload( + &self, + req: S3Request, + ) -> S3Result> { + let helper = OperationHelper::new(&req, EventName::ObjectCreatedCompleteMultipartUpload, "s3:CompleteMultipartUpload"); + let input = req.input; + let CompleteMultipartUploadInput { + multipart_upload, + bucket, + key, + upload_id, + if_match, + if_none_match, + .. + } = input; + + if if_match.is_some() || if_none_match.is_some() { + let Some(store) = new_object_layer_fn() else { + return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); + }; + + match store.get_object_info(&bucket, &key, &ObjectOptions::default()).await { + Ok(info) => { + if !info.delete_marker { + if let Some(ifmatch) = if_match + && let Some(strong_etag) = ifmatch.into_etag() + && info + .etag + .as_ref() + .is_some_and(|etag| ETag::Strong(etag.clone()) != strong_etag) + { + return Err(s3_error!(PreconditionFailed)); + } + if let Some(ifnonematch) = if_none_match + && let Some(strong_etag) = ifnonematch.into_etag() + && info + .etag + .as_ref() + .is_some_and(|etag| ETag::Strong(etag.clone()) == strong_etag) + { + return Err(s3_error!(PreconditionFailed)); + } + } + } + Err(err) => { + if !is_err_object_not_found(&err) && !is_err_version_not_found(&err) { + return Err(ApiError::from(err).into()); + } + + if if_match.is_some() && (is_err_object_not_found(&err) || is_err_version_not_found(&err)) { + return Err(ApiError::from(err).into()); + } + } + } + } + + let Some(multipart_upload) = multipart_upload else { return Err(s3_error!(InvalidPart)) }; + + let opts = &get_complete_multipart_upload_opts(&req.headers).map_err(ApiError::from)?; + + let uploaded_parts_vec = multipart_upload + .parts + .unwrap_or_default() + .into_iter() + .map(CompletePart::from) + .collect::>(); + + // is part number sorted? + if !uploaded_parts_vec.is_sorted_by_key(|p| p.part_num) { + return Err(s3_error!(InvalidPart, "Part numbers must be sorted")); + } + + // Handle duplicate part numbers: according to S3 specification, when the same part number + // is uploaded multiple times, the last uploaded part (in the list order) should be used. + // This can happen in concurrent upload scenarios where a part is re-uploaded before completion. + // We deduplicate by keeping the last occurrence of each part number using a HashMap. + use std::collections::HashMap; + let mut part_map: HashMap = HashMap::new(); + for part in uploaded_parts_vec { + part_map.insert(part.part_num, part); + } + + // Reconstruct the parts list in sorted order, keeping only the last occurrence of each part number + let mut uploaded_parts: Vec = part_map.into_values().collect(); + uploaded_parts.sort_by_key(|p| p.part_num); + + // TODO: check object lock + + let Some(store) = new_object_layer_fn() else { + return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); + }; + + // TDD: Get multipart info to extract encryption configuration before completing + info!( + "TDD: Attempting to get multipart info for bucket={}, key={}, upload_id={}", + bucket, key, upload_id + ); + + let multipart_info = store + .get_multipart_info(&bucket, &key, &upload_id, &ObjectOptions::default()) .await .map_err(ApiError::from)?; - let output = CreateBucketOutput::default(); + info!("TDD: Got multipart info successfully"); + info!("TDD: Multipart info metadata: {:?}", multipart_info.user_defined); - let result = Ok(S3Response::new(output)); - let _ = helper.complete(&result); - result + // TDD: Extract encryption information from multipart upload metadata + let server_side_encryption = multipart_info + .user_defined + .get("x-amz-server-side-encryption") + .map(|s| ServerSideEncryption::from(s.clone())); + info!( + "TDD: Raw encryption from metadata: {:?} -> parsed: {:?}", + multipart_info.user_defined.get("x-amz-server-side-encryption"), + server_side_encryption + ); + + let ssekms_key_id = multipart_info + .user_defined + .get("x-amz-server-side-encryption-aws-kms-key-id") + .cloned(); + + info!( + "TDD: Extracted encryption info - SSE: {:?}, KMS Key: {:?}", + server_side_encryption, ssekms_key_id + ); + + let obj_info = store + .clone() + .complete_multipart_upload(&bucket, &key, &upload_id, uploaded_parts, opts) + .await + .map_err(ApiError::from)?; + + // check quota after completing multipart upload + if let Some(metadata_sys) = rustfs_ecstore::bucket::metadata_sys::GLOBAL_BucketMetadataSys.get() { + let quota_checker = QuotaChecker::new(metadata_sys.clone()); + + match quota_checker + .check_quota(&bucket, QuotaOperation::PutObject, obj_info.size as u64) + .await + { + Ok(check_result) => { + if !check_result.allowed { + // Quota exceeded, delete the completed object + let _ = store.delete_object(&bucket, &key, ObjectOptions::default()).await; + return Err(S3Error::with_message( + S3ErrorCode::InvalidRequest, + format!( + "Bucket quota exceeded. Current usage: {} bytes, limit: {} bytes", + check_result.current_usage, + check_result.quota_limit.unwrap_or(0) + ), + )); + } + // Update quota tracking after successful multipart upload + if rustfs_ecstore::bucket::metadata_sys::GLOBAL_BucketMetadataSys.get().is_some() { + rustfs_ecstore::data_usage::increment_bucket_usage_memory(&bucket, obj_info.size as u64).await; + } + } + Err(e) => { + warn!("Quota check failed for bucket {}: {}, allowing operation", bucket, e); + } + } + } + + // Invalidate cache for the completed multipart object + let manager = get_concurrency_manager(); + let mpu_bucket = bucket.clone(); + let mpu_key = key.clone(); + let mpu_version = obj_info.version_id.map(|v| v.to_string()); + let mpu_version_clone = mpu_version.clone(); + tokio::spawn(async move { + manager + .invalidate_cache_versioned(&mpu_bucket, &mpu_key, mpu_version_clone.as_deref()) + .await; + }); + + info!( + "TDD: Creating output with SSE: {:?}, KMS Key: {:?}", + server_side_encryption, ssekms_key_id + ); + + let mut checksum_crc32 = input.checksum_crc32; + let mut checksum_crc32c = input.checksum_crc32c; + let mut checksum_sha1 = input.checksum_sha1; + let mut checksum_sha256 = input.checksum_sha256; + let mut checksum_crc64nvme = input.checksum_crc64nvme; + let mut checksum_type = input.checksum_type; + + // checksum + let (checksums, _is_multipart) = obj_info + .decrypt_checksums(opts.part_number.unwrap_or(0), &req.headers) + .map_err(ApiError::from)?; + + for (key, checksum) in checksums { + if key == AMZ_CHECKSUM_TYPE { + checksum_type = Some(ChecksumType::from(checksum)); + continue; + } + + match rustfs_rio::ChecksumType::from_string(key.as_str()) { + rustfs_rio::ChecksumType::CRC32 => checksum_crc32 = Some(checksum), + rustfs_rio::ChecksumType::CRC32C => checksum_crc32c = Some(checksum), + rustfs_rio::ChecksumType::SHA1 => checksum_sha1 = Some(checksum), + rustfs_rio::ChecksumType::SHA256 => checksum_sha256 = Some(checksum), + rustfs_rio::ChecksumType::CRC64_NVME => checksum_crc64nvme = Some(checksum), + _ => (), + } + } + + let output = CompleteMultipartUploadOutput { + bucket: Some(bucket.clone()), + key: Some(key.clone()), + e_tag: obj_info.etag.clone().map(|etag| to_s3s_etag(&etag)), + location: Some("us-east-1".to_string()), + server_side_encryption: server_side_encryption.clone(), // TDD: Return encryption info + ssekms_key_id: ssekms_key_id.clone(), // TDD: Return KMS key ID if present + checksum_crc32: checksum_crc32.clone(), + checksum_crc32c: checksum_crc32c.clone(), + checksum_sha1: checksum_sha1.clone(), + checksum_sha256: checksum_sha256.clone(), + checksum_crc64nvme: checksum_crc64nvme.clone(), + checksum_type: checksum_type.clone(), + version_id: mpu_version, + ..Default::default() + }; + info!( + "TDD: Created output: SSE={:?}, KMS={:?}", + output.server_side_encryption, output.ssekms_key_id + ); + + let helper_output = entity::CompleteMultipartUploadOutput { + bucket: Some(bucket.clone()), + key: Some(key.clone()), + e_tag: obj_info.etag.clone().map(|etag| to_s3s_etag(&etag)), + location: Some("us-east-1".to_string()), + server_side_encryption, // TDD: Return encryption info + ssekms_key_id, // TDD: Return KMS key ID if present + checksum_crc32, + checksum_crc32c, + checksum_sha1, + checksum_sha256, + checksum_crc64nvme, + checksum_type, + ..Default::default() + }; + + let mt2 = HashMap::new(); + let repoptions = + get_must_replicate_options(&mt2, "".to_string(), ReplicationStatusType::Empty, ReplicationType::Object, opts.clone()); + + let dsc = must_replicate(&bucket, &key, repoptions).await; + + if dsc.replicate_any() { + warn!("need multipart replication"); + schedule_replication(obj_info, store, dsc, ReplicationType::Object).await; + } + info!( + "TDD: About to return S3Response with output: SSE={:?}, KMS={:?}", + output.server_side_encryption, output.ssekms_key_id + ); + let helper_result = Ok(S3Response::new(helper_output)); + let _ = helper.complete(&helper_result); + Ok(S3Response::new(output)) } /// Copy an object from one location to another @@ -1684,238 +1150,196 @@ impl S3 for FS { result } - async fn restore_object(&self, req: S3Request) -> S3Result> { - let RestoreObjectInput { + #[instrument( + level = "debug", + skip(self, req), + fields(start_time=?time::OffsetDateTime::now_utc()) + )] + async fn create_bucket(&self, req: S3Request) -> S3Result> { + let helper = OperationHelper::new(&req, EventName::BucketCreated, "s3:CreateBucket"); + let CreateBucketInput { bucket, - key: object, - restore_request: rreq, - version_id, + object_lock_enabled_for_bucket, .. - } = req.input.clone(); - let rreq = rreq.unwrap(); - - /*if let Err(e) = un_escape_path(object) { - warn!("post restore object failed, e: {:?}", e); - return Err(S3Error::with_message(S3ErrorCode::Custom("PostRestoreObjectFailed".into()), "post restore object failed")); - }*/ + } = req.input; let Some(store) = new_object_layer_fn() else { return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); }; - /*if Err(err) = check_request_auth_type(req, policy::RestoreObjectAction, bucket, object) { - return Err(S3Error::with_message(S3ErrorCode::Custom("PostRestoreObjectFailed".into()), "post restore object failed")); - }*/ + counter!("rustfs_create_bucket_total").increment(1); - /*if req.content_length <= 0 { - return Err(S3Error::with_message(S3ErrorCode::Custom("ErrEmptyRequestBody".into()), "post restore object failed")); - }*/ - let Ok(opts) = post_restore_opts(&version_id.unwrap(), &bucket, &object).await else { - return Err(S3Error::with_message( - S3ErrorCode::Custom("ErrEmptyRequestBody".into()), - "post restore object failed", - )); - }; - - let Ok(mut obj_info) = store.get_object_info(&bucket, &object, &opts).await else { - return Err(S3Error::with_message( - S3ErrorCode::Custom("ErrEmptyRequestBody".into()), - "post restore object failed", - )); - }; - - if obj_info.transitioned_object.status != lifecycle::TRANSITION_COMPLETE { - return Err(S3Error::with_message( - S3ErrorCode::Custom("ErrEmptyRequestBody".into()), - "post restore object failed", - )); - } - - //let mut api_err; - let mut _status_code = StatusCode::OK; - let mut already_restored = false; - if let Err(_err) = rreq.validate(store.clone()) { - //api_err = to_api_err(ErrMalformedXML); - //api_err.description = err.to_string(); - return Err(S3Error::with_message( - S3ErrorCode::Custom("ErrEmptyRequestBody".into()), - "post restore object failed", - )); - } else { - if obj_info.restore_ongoing && (rreq.type_.is_none() || rreq.type_.as_ref().unwrap().as_str() != "SELECT") { - return Err(S3Error::with_message( - S3ErrorCode::Custom("ErrObjectRestoreAlreadyInProgress".into()), - "post restore object failed", - )); - } - if !obj_info.restore_ongoing && obj_info.restore_expires.unwrap().unix_timestamp() != 0 { - _status_code = StatusCode::ACCEPTED; - already_restored = true; - } - } - let restore_expiry = lifecycle::expected_expiry_time(OffsetDateTime::now_utc(), *rreq.days.as_ref().unwrap()); - let mut metadata = obj_info.user_defined.clone(); - - let mut header = HeaderMap::new(); - - let obj_info_ = obj_info.clone(); - if rreq.type_.is_none() || rreq.type_.as_ref().unwrap().as_str() != "SELECT" { - obj_info.metadata_only = true; - metadata.insert(AMZ_RESTORE_EXPIRY_DAYS.to_string(), rreq.days.unwrap().to_string()); - metadata.insert(AMZ_RESTORE_REQUEST_DATE.to_string(), OffsetDateTime::now_utc().format(&Rfc3339).unwrap()); - if already_restored { - metadata.insert( - X_AMZ_RESTORE.as_str().to_string(), - RestoreStatus { - is_restore_in_progress: Some(false), - restore_expiry_date: Some(Timestamp::from(restore_expiry)), - } - .to_string(), - ); - } else { - metadata.insert( - X_AMZ_RESTORE.as_str().to_string(), - RestoreStatus { - is_restore_in_progress: Some(true), - restore_expiry_date: Some(Timestamp::from(OffsetDateTime::now_utc())), - } - .to_string(), - ); - } - obj_info.user_defined = metadata; - if let Err(_err) = store - .clone() - .copy_object( - &bucket, - &object, - &bucket, - &object, - &mut obj_info, - &ObjectOptions { - version_id: obj_info_.version_id.map(|e| e.to_string()), - ..Default::default() - }, - &ObjectOptions { - version_id: obj_info_.version_id.map(|e| e.to_string()), - mod_time: obj_info_.mod_time, - ..Default::default() - }, - ) - .await - { - return Err(S3Error::with_message( - S3ErrorCode::Custom("ErrInvalidObjectState".into()), - "post restore object failed", - )); - } - if already_restored { - let output = RestoreObjectOutput { - request_charged: Some(RequestCharged::from_static(RequestCharged::REQUESTER)), - restore_output_path: None, - }; - return Ok(S3Response::new(output)); - } - } - - let restore_object = Uuid::new_v4().to_string(); - //if let Some(rreq) = rreq { - if let Some(output_location) = &rreq.output_location - && let Some(s3) = &output_location.s3 - && !s3.bucket_name.is_empty() - { - header.insert( - X_AMZ_RESTORE_OUTPUT_PATH, - format!("{}{}{}", s3.bucket_name, s3.prefix, restore_object).parse().unwrap(), - ); - } - //} - /*send_event(EventArgs { - event_name: event::ObjectRestorePost, - bucket_name: bucket, - object: obj_info, - req_params: extract_req_params(r), - user_agent: req.user_agent(), - host: handlers::get_source_ip(r), - });*/ - tokio::spawn(async move { - /*if rreq.select_parameters.is_some() { - let actual_size = obj_info_.get_actual_size(); - if actual_size.is_err() { - return Err(S3Error::with_message(S3ErrorCode::Custom("ErrInvalidObjectState".into()), "post restore object failed")); - } - - let object_rsc = s3select.new_object_read_seek_closer( - |offset: i64| -> (ReadCloser, error) { - rs := &HTTPRangeSpec{ - IsSuffixLength: false, - Start: offset, - End: -1, - } - return get_transitioned_object_reader(bucket, object, rs, r.Header, - obj_info, ObjectOptions {version_id: obj_info_.version_id}); - }, - actual_size.unwrap(), - ); - if err = rreq.select_parameters.open(object_rsc); err != nil { - if serr, ok := err.(s3select.SelectError); ok { - let encoded_error_response = encodeResponse(APIErrorResponse { - code: serr.ErrorCode(), - message: serr.ErrorMessage(), - bucket_name: bucket, - key: object, - resource: r.URL.Path, - request_id: w.Header().Get(xhttp.AmzRequestID), - host_id: globalDeploymentID(), - }); - //writeResponse(w, serr.HTTPStatusCode(), encodedErrorResponse, mimeXML) - Ok(S3Response::with_headers((StatusCode::OK, Body::empty()), header)); - } else { - return Err(S3Error::with_message(S3ErrorCode::Custom("ErrInvalidObjectState".into()), "post restore object failed")); - } - return Ok(()); - } - let nr = httptest.NewRecorder(); - let rw = xhttp.NewResponseRecorder(nr); - rw.log_err_body = true; - rw.log_all_body = true; - rreq.select_parameters.evaluate(rw); - rreq.select_parameters.Close(); - return Ok(S3Response::with_headers((StatusCode::OK, Body::empty()), header)); - }*/ - let opts = ObjectOptions { - transition: TransitionOptions { - restore_request: rreq, - restore_expiry, + store + .make_bucket( + &bucket, + &MakeBucketOptions { + force_create: false, // TODO: force support + lock_enabled: object_lock_enabled_for_bucket.is_some_and(|v| v), ..Default::default() }, - version_id: obj_info_.version_id.map(|e| e.to_string()), - ..Default::default() - }; - if let Err(err) = store.clone().restore_transitioned_object(&bucket, &object, &opts).await { - warn!("unable to restore transitioned bucket/object {}/{}: {}", bucket, object, err.to_string()); - return Err(S3Error::with_message( - S3ErrorCode::Custom("ErrRestoreTransitionedObject".into()), - format!("unable to restore transitioned bucket/object {bucket}/{object}: {err}"), - )); - } + ) + .await + .map_err(ApiError::from)?; - /*send_event(EventArgs { - EventName: event.ObjectRestoreCompleted, - BucketName: bucket, - Object: objInfo, - ReqParams: extractReqParams(r), - UserAgent: r.UserAgent(), - Host: handlers.GetSourceIP(r), - });*/ - Ok(()) - }); + let output = CreateBucketOutput::default(); - let output = RestoreObjectOutput { - request_charged: Some(RequestCharged::from_static(RequestCharged::REQUESTER)), - restore_output_path: None, + let result = Ok(S3Response::new(output)); + let _ = helper.complete(&result); + result + } + + #[instrument(level = "debug", skip(self, req))] + async fn create_multipart_upload( + &self, + req: S3Request, + ) -> S3Result> { + let helper = OperationHelper::new(&req, EventName::ObjectCreatedPut, "s3:CreateMultipartUpload"); + let CreateMultipartUploadInput { + bucket, + key, + tagging, + version_id, + storage_class, + server_side_encryption, + sse_customer_algorithm, + sse_customer_key_md5, + ssekms_key_id, + .. + } = req.input.clone(); + + // Validate storage class if provided + if let Some(ref storage_class) = storage_class + && !is_valid_storage_class(storage_class.as_str()) + { + return Err(s3_error!(InvalidStorageClass)); + } + + // mc cp step 3 + + // debug!("create_multipart_upload meta {:?}", &metadata); + + let Some(store) = new_object_layer_fn() else { + return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); }; - Ok(S3Response::with_headers(output, header)) + let mut metadata = extract_metadata(&req.headers); + + if let Some(tags) = tagging { + metadata.insert(AMZ_OBJECT_TAGGING.to_owned(), tags); + } + + // TDD: Get bucket SSE configuration for multipart upload + let bucket_sse_config = metadata_sys::get_sse_config(&bucket).await.ok(); + debug!("TDD: Got bucket SSE config for multipart: {:?}", bucket_sse_config); + + // TDD: Determine effective encryption (request parameters override bucket defaults) + let original_sse = server_side_encryption.clone(); + let effective_sse = server_side_encryption.or_else(|| { + bucket_sse_config.as_ref().and_then(|(config, _timestamp)| { + debug!("TDD: Processing bucket SSE config for multipart: {:?}", config); + config.rules.first().and_then(|rule| { + debug!("TDD: Processing SSE rule for multipart: {:?}", rule); + rule.apply_server_side_encryption_by_default.as_ref().map(|sse| { + debug!("TDD: Found SSE default for multipart: {:?}", sse); + match sse.sse_algorithm.as_str() { + "AES256" => ServerSideEncryption::from_static(ServerSideEncryption::AES256), + "aws:kms" => ServerSideEncryption::from_static(ServerSideEncryption::AWS_KMS), + _ => ServerSideEncryption::from_static(ServerSideEncryption::AES256), // fallback to AES256 + } + }) + }) + }) + }); + debug!("TDD: effective_sse for multipart={:?} (original={:?})", effective_sse, original_sse); + + let _original_kms_key_id = ssekms_key_id.clone(); + let mut effective_kms_key_id = ssekms_key_id.or_else(|| { + bucket_sse_config.as_ref().and_then(|(config, _timestamp)| { + config.rules.first().and_then(|rule| { + rule.apply_server_side_encryption_by_default + .as_ref() + .and_then(|sse| sse.kms_master_key_id.clone()) + }) + }) + }); + + // Store effective SSE information in metadata for multipart upload + if let Some(sse_alg) = &sse_customer_algorithm { + metadata.insert( + "x-amz-server-side-encryption-customer-algorithm".to_string(), + sse_alg.as_str().to_string(), + ); + } + if let Some(sse_md5) = &sse_customer_key_md5 { + metadata.insert("x-amz-server-side-encryption-customer-key-md5".to_string(), sse_md5.clone()); + } + + if let Some(sse) = &effective_sse { + if is_managed_sse(sse) { + let material = create_managed_encryption_material(&bucket, &key, sse, effective_kms_key_id.clone(), 0).await?; + + let ManagedEncryptionMaterial { + data_key: _, + headers, + kms_key_id: kms_key_used, + } = material; + + metadata.extend(headers.into_iter()); + effective_kms_key_id = Some(kms_key_used.clone()); + } else { + metadata.insert("x-amz-server-side-encryption".to_string(), sse.as_str().to_string()); + } + } + + if let Some(kms_key_id) = &effective_kms_key_id { + metadata.insert("x-amz-server-side-encryption-aws-kms-key-id".to_string(), kms_key_id.clone()); + } + + if is_compressible(&req.headers, &key) { + metadata.insert( + format!("{RESERVED_METADATA_PREFIX_LOWER}compression"), + CompressionAlgorithm::default().to_string(), + ); + } + + let mut opts: ObjectOptions = put_opts(&bucket, &key, version_id, &req.headers, metadata) + .await + .map_err(ApiError::from)?; + + let checksum_type = rustfs_rio::ChecksumType::from_header(&req.headers); + if checksum_type.is(rustfs_rio::ChecksumType::INVALID) { + return Err(s3_error!(InvalidArgument, "Invalid checksum type")); + } else if checksum_type.is_set() && !checksum_type.is(rustfs_rio::ChecksumType::TRAILING) { + opts.want_checksum = Some(rustfs_rio::Checksum { + checksum_type, + ..Default::default() + }); + } + + let MultipartUploadResult { + upload_id, + checksum_algo, + checksum_type, + } = store + .new_multipart_upload(&bucket, &key, &opts) + .await + .map_err(ApiError::from)?; + + let output = CreateMultipartUploadOutput { + bucket: Some(bucket), + key: Some(key), + upload_id: Some(upload_id), + server_side_encryption: effective_sse, // TDD: Return effective encryption config + sse_customer_algorithm, + ssekms_key_id: effective_kms_key_id, // TDD: Return effective KMS key ID + checksum_algorithm: checksum_algo.map(ChecksumAlgorithm::from), + checksum_type: checksum_type.map(ChecksumType::from), + ..Default::default() + }; + + let result = Ok(S3Response::new(output)); + let _ = helper.complete(&result); + result } /// Delete a bucket @@ -1944,6 +1368,130 @@ impl S3 for FS { result } + #[instrument(level = "debug", skip(self))] + async fn delete_bucket_cors(&self, req: S3Request) -> S3Result> { + let DeleteBucketCorsInput { bucket, .. } = req.input; + + let Some(store) = new_object_layer_fn() else { + return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); + }; + + store + .get_bucket_info(&bucket, &BucketOptions::default()) + .await + .map_err(ApiError::from)?; + + metadata_sys::delete(&bucket, BUCKET_CORS_CONFIG) + .await + .map_err(ApiError::from)?; + + Ok(S3Response::new(DeleteBucketCorsOutput {})) + } + + async fn delete_bucket_encryption( + &self, + req: S3Request, + ) -> S3Result> { + let DeleteBucketEncryptionInput { bucket, .. } = req.input; + + let Some(store) = new_object_layer_fn() else { + return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); + }; + + store + .get_bucket_info(&bucket, &BucketOptions::default()) + .await + .map_err(ApiError::from)?; + metadata_sys::delete(&bucket, BUCKET_SSECONFIG) + .await + .map_err(ApiError::from)?; + + Ok(S3Response::new(DeleteBucketEncryptionOutput::default())) + } + + #[instrument(level = "debug", skip(self))] + async fn delete_bucket_lifecycle( + &self, + req: S3Request, + ) -> S3Result> { + let DeleteBucketLifecycleInput { bucket, .. } = req.input; + + let Some(store) = new_object_layer_fn() else { + return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); + }; + + store + .get_bucket_info(&bucket, &BucketOptions::default()) + .await + .map_err(ApiError::from)?; + + metadata_sys::delete(&bucket, BUCKET_LIFECYCLE_CONFIG) + .await + .map_err(ApiError::from)?; + + Ok(S3Response::new(DeleteBucketLifecycleOutput::default())) + } + + async fn delete_bucket_policy( + &self, + req: S3Request, + ) -> S3Result> { + let DeleteBucketPolicyInput { bucket, .. } = req.input; + + let Some(store) = new_object_layer_fn() else { + return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); + }; + + store + .get_bucket_info(&bucket, &BucketOptions::default()) + .await + .map_err(ApiError::from)?; + + metadata_sys::delete(&bucket, BUCKET_POLICY_CONFIG) + .await + .map_err(ApiError::from)?; + + Ok(S3Response::new(DeleteBucketPolicyOutput {})) + } + + async fn delete_bucket_replication( + &self, + req: S3Request, + ) -> S3Result> { + let DeleteBucketReplicationInput { bucket, .. } = req.input; + + let Some(store) = new_object_layer_fn() else { + return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); + }; + + store + .get_bucket_info(&bucket, &BucketOptions::default()) + .await + .map_err(ApiError::from)?; + metadata_sys::delete(&bucket, BUCKET_REPLICATION_CONFIG) + .await + .map_err(ApiError::from)?; + + // TODO: remove targets + error!("delete bucket"); + + Ok(S3Response::new(DeleteBucketReplicationOutput::default())) + } + + #[instrument(level = "debug", skip(self))] + async fn delete_bucket_tagging( + &self, + req: S3Request, + ) -> S3Result> { + let DeleteBucketTaggingInput { bucket, .. } = req.input; + + metadata_sys::delete(&bucket, BUCKET_TAGGING_CONFIG) + .await + .map_err(ApiError::from)?; + + Ok(S3Response::new(DeleteBucketTaggingOutput {})) + } + /// Delete an object #[instrument(level = "debug", skip(self, req))] async fn delete_object(&self, mut req: S3Request) -> S3Result> { @@ -1966,7 +1514,6 @@ impl S3 for FS { } let metadata = extract_metadata(&req.headers); - // Clone version_id before it's moved let version_id_clone = version_id.clone(); @@ -2105,6 +1652,64 @@ impl S3 for FS { result } + #[instrument(level = "debug", skip(self))] + async fn delete_object_tagging( + &self, + req: S3Request, + ) -> S3Result> { + let start_time = std::time::Instant::now(); + let mut helper = OperationHelper::new(&req, EventName::ObjectCreatedDeleteTagging, "s3:DeleteObjectTagging"); + let DeleteObjectTaggingInput { + bucket, + key: object, + version_id, + .. + } = req.input.clone(); + + let Some(store) = new_object_layer_fn() else { + error!("Store not initialized"); + return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); + }; + + // Support versioned objects + let version_id_for_parse = version_id.clone(); + let opts = ObjectOptions { + version_id: self.parse_version_id(version_id_for_parse)?.map(Into::into), + ..Default::default() + }; + + // TODO: Replicate (keep the original TODO, if further replication logic is needed) + store.delete_object_tags(&bucket, &object, &opts).await.map_err(|e| { + error!("Failed to delete object tags: {}", e); + ApiError::from(e) + })?; + + // Invalidate cache for the deleted tagged object + let manager = get_concurrency_manager(); + let version_id_clone = version_id.clone(); + tokio::spawn(async move { + manager + .invalidate_cache_versioned(&bucket, &object, version_id_clone.as_deref()) + .await; + debug!( + "Cache invalidated for deleted tagged object: bucket={}, object={}, version_id={:?}", + bucket, object, version_id_clone + ); + }); + + // Add metrics + counter!("rustfs.delete_object_tagging.success").increment(1); + + let version_id_resp = version_id.clone().unwrap_or_default(); + helper = helper.version_id(version_id_resp); + + let result = Ok(S3Response::new(DeleteObjectTaggingOutput { version_id })); + let _ = helper.complete(&result); + let duration = start_time.elapsed(); + histogram!("rustfs.object_tagging.operation.duration.seconds", "operation" => "delete").record(duration.as_secs_f64()); + result + } + /// Delete multiple objects #[instrument(level = "debug", skip(self, req))] async fn delete_objects(&self, mut req: S3Request) -> S3Result> { @@ -2433,6 +2038,127 @@ impl S3 for FS { result } + async fn get_bucket_acl(&self, req: S3Request) -> S3Result> { + let GetBucketAclInput { bucket, .. } = req.input; + + let Some(store) = new_object_layer_fn() else { + return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); + }; + + store + .get_bucket_info(&bucket, &BucketOptions::default()) + .await + .map_err(ApiError::from)?; + + let grants = vec![Grant { + grantee: Some(Grantee { + type_: Type::from_static(Type::CANONICAL_USER), + display_name: None, + email_address: None, + id: None, + uri: None, + }), + permission: Some(Permission::from_static(Permission::FULL_CONTROL)), + }]; + + Ok(S3Response::new(GetBucketAclOutput { + grants: Some(grants), + owner: Some(RUSTFS_OWNER.to_owned()), + })) + } + + #[instrument(level = "debug", skip(self))] + async fn get_bucket_cors(&self, req: S3Request) -> S3Result> { + let bucket = req.input.bucket.clone(); + // check bucket exists. + let _bucket = self + .head_bucket(req.map_input(|input| HeadBucketInput { + bucket: input.bucket, + expected_bucket_owner: None, + })) + .await?; + + let cors_configuration = match metadata_sys::get_cors_config(&bucket).await { + Ok((config, _)) => config, + Err(err) => { + if err == StorageError::ConfigNotFound { + return Err(S3Error::with_message( + S3ErrorCode::NoSuchCORSConfiguration, + "The CORS configuration does not exist".to_string(), + )); + } + warn!("get_cors_config err {:?}", &err); + return Err(ApiError::from(err).into()); + } + }; + + Ok(S3Response::new(GetBucketCorsOutput { + cors_rules: Some(cors_configuration.cors_rules), + })) + } + + async fn get_bucket_encryption( + &self, + req: S3Request, + ) -> S3Result> { + let GetBucketEncryptionInput { bucket, .. } = req.input; + + let Some(store) = new_object_layer_fn() else { + return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); + }; + + store + .get_bucket_info(&bucket, &BucketOptions::default()) + .await + .map_err(ApiError::from)?; + + let server_side_encryption_configuration = match metadata_sys::get_sse_config(&bucket).await { + Ok((cfg, _)) => Some(cfg), + Err(err) => { + // if BucketMetadataError::BucketLifecycleNotFound.is(&err) { + // return Err(s3_error!(ErrNoSuchBucketSSEConfig)); + // } + warn!("get_sse_config err {:?}", err); + None + } + }; + + Ok(S3Response::new(GetBucketEncryptionOutput { + server_side_encryption_configuration, + })) + } + + #[instrument(level = "debug", skip(self))] + async fn get_bucket_lifecycle_configuration( + &self, + req: S3Request, + ) -> S3Result> { + let GetBucketLifecycleConfigurationInput { bucket, .. } = req.input; + + let Some(store) = new_object_layer_fn() else { + return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); + }; + + store + .get_bucket_info(&bucket, &BucketOptions::default()) + .await + .map_err(ApiError::from)?; + + let rules = match metadata_sys::get_lifecycle_config(&bucket).await { + Ok((cfg, _)) => cfg.rules, + Err(_err) => { + // Return NoSuchLifecycleConfiguration error as expected by S3 clients + // This fixes issue #990 where Ansible S3 roles fail with KeyError: 'Rules' + return Err(s3_error!(NoSuchLifecycleConfiguration)); + } + }; + + Ok(S3Response::new(GetBucketLifecycleConfigurationOutput { + rules: Some(rules), + ..Default::default() + })) + } + /// Get bucket location #[instrument(level = "debug", skip(self, req))] async fn get_bucket_location(&self, req: S3Request) -> S3Result> { @@ -2458,6 +2184,229 @@ impl S3 for FS { Ok(S3Response::new(output)) } + async fn get_bucket_notification_configuration( + &self, + req: S3Request, + ) -> S3Result> { + let GetBucketNotificationConfigurationInput { bucket, .. } = req.input; + + let Some(store) = new_object_layer_fn() else { + return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); + }; + + store + .get_bucket_info(&bucket, &BucketOptions::default()) + .await + .map_err(ApiError::from)?; + + let has_notification_config = metadata_sys::get_notification_config(&bucket).await.unwrap_or_else(|err| { + warn!("get_notification_config err {:?}", err); + None + }); + + // TODO: valid target list + + if let Some(NotificationConfiguration { + event_bridge_configuration, + lambda_function_configurations, + queue_configurations, + topic_configurations, + }) = has_notification_config + { + Ok(S3Response::new(GetBucketNotificationConfigurationOutput { + event_bridge_configuration, + lambda_function_configurations, + queue_configurations, + topic_configurations, + })) + } else { + Ok(S3Response::new(GetBucketNotificationConfigurationOutput::default())) + } + } + + async fn get_bucket_policy(&self, req: S3Request) -> S3Result> { + let GetBucketPolicyInput { bucket, .. } = req.input; + + let Some(store) = new_object_layer_fn() else { + return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); + }; + + store + .get_bucket_info(&bucket, &BucketOptions::default()) + .await + .map_err(ApiError::from)?; + + // Return the raw policy JSON as originally stored to preserve exact format. + // This ensures GET returns the same document that was PUT, matching S3 behavior. + let (policy_str, _) = match metadata_sys::get_bucket_policy_raw(&bucket).await { + Ok(res) => res, + Err(err) => { + if StorageError::ConfigNotFound == err { + return Err(s3_error!(NoSuchBucketPolicy)); + } + return Err(S3Error::with_message(S3ErrorCode::InternalError, err.to_string())); + } + }; + + Ok(S3Response::new(GetBucketPolicyOutput { + policy: Some(policy_str), + })) + } + + async fn get_bucket_policy_status( + &self, + req: S3Request, + ) -> S3Result> { + let GetBucketPolicyStatusInput { bucket, .. } = req.input; + + let Some(store) = new_object_layer_fn() else { + return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); + }; + + store + .get_bucket_info(&bucket, &BucketOptions::default()) + .await + .map_err(ApiError::from)?; + + let remote_addr = req.extensions.get::>().and_then(|opt| opt.map(|a| a.0)); + let conditions = get_condition_values(&req.headers, &rustfs_credentials::Credentials::default(), None, None, remote_addr); + + let read_only = PolicySys::is_allowed(&BucketPolicyArgs { + bucket: &bucket, + action: Action::S3Action(S3Action::ListBucketAction), + is_owner: false, + account: "", + groups: &None, + conditions: &conditions, + object: "", + }) + .await; + + let write_only = PolicySys::is_allowed(&BucketPolicyArgs { + bucket: &bucket, + action: Action::S3Action(S3Action::PutObjectAction), + is_owner: false, + account: "", + groups: &None, + conditions: &conditions, + object: "", + }) + .await; + + let is_public = read_only && write_only; + + let output = GetBucketPolicyStatusOutput { + policy_status: Some(PolicyStatus { + is_public: Some(is_public), + }), + }; + + Ok(S3Response::new(output)) + } + + async fn get_bucket_replication( + &self, + req: S3Request, + ) -> S3Result> { + let GetBucketReplicationInput { bucket, .. } = req.input; + + let Some(store) = new_object_layer_fn() else { + return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); + }; + + store + .get_bucket_info(&bucket, &BucketOptions::default()) + .await + .map_err(ApiError::from)?; + + let rcfg = match get_replication_config(&bucket).await { + Ok((cfg, _created)) => Some(cfg), + Err(err) => { + error!("get_replication_config err {:?}", err); + if err == StorageError::ConfigNotFound { + return Err(S3Error::with_message( + S3ErrorCode::ReplicationConfigurationNotFoundError, + "replication not found".to_string(), + )); + } + return Err(ApiError::from(err).into()); + } + }; + + if rcfg.is_none() { + return Err(S3Error::with_message( + S3ErrorCode::ReplicationConfigurationNotFoundError, + "replication not found".to_string(), + )); + } + + // Ok(S3Response::new(GetBucketReplicationOutput { + // replication_configuration: rcfg, + // })) + + if rcfg.is_some() { + Ok(S3Response::new(GetBucketReplicationOutput { + replication_configuration: rcfg, + })) + } else { + let rep = ReplicationConfiguration { + role: "".to_string(), + rules: vec![], + }; + Ok(S3Response::new(GetBucketReplicationOutput { + replication_configuration: Some(rep), + })) + } + } + + #[instrument(level = "debug", skip(self))] + async fn get_bucket_tagging(&self, req: S3Request) -> S3Result> { + let bucket = req.input.bucket.clone(); + // check bucket exists. + let _bucket = self + .head_bucket(req.map_input(|input| HeadBucketInput { + bucket: input.bucket, + expected_bucket_owner: None, + })) + .await?; + + let Tagging { tag_set } = match metadata_sys::get_tagging_config(&bucket).await { + Ok((tags, _)) => tags, + Err(err) => { + if err == StorageError::ConfigNotFound { + return Err(S3Error::with_message(S3ErrorCode::NoSuchTagSet, "The TagSet does not exist".to_string())); + } + warn!("get_tagging_config err {:?}", &err); + return Err(ApiError::from(err).into()); + } + }; + + Ok(S3Response::new(GetBucketTaggingOutput { tag_set })) + } + + #[instrument(level = "debug", skip(self))] + async fn get_bucket_versioning( + &self, + req: S3Request, + ) -> S3Result> { + let GetBucketVersioningInput { bucket, .. } = req.input; + let Some(store) = new_object_layer_fn() else { + return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); + }; + + store + .get_bucket_info(&bucket, &BucketOptions::default()) + .await + .map_err(ApiError::from)?; + + let VersioningConfiguration { status, .. } = BucketVersioningSys::get(&bucket).await.map_err(ApiError::from)?; + + Ok(S3Response::new(GetBucketVersioningOutput { + status, + ..Default::default() + })) + } + /// Get bucket notification #[instrument( level = "debug", @@ -3148,6 +3097,247 @@ impl S3 for FS { result } + async fn get_object_acl(&self, req: S3Request) -> S3Result> { + let GetObjectAclInput { bucket, key, .. } = req.input; + + let Some(store) = new_object_layer_fn() else { + return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); + }; + + if let Err(e) = store.get_object_info(&bucket, &key, &ObjectOptions::default()).await { + return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("{e}"))); + } + + let grants = vec![Grant { + grantee: Some(Grantee { + type_: Type::from_static(Type::CANONICAL_USER), + display_name: None, + email_address: None, + id: None, + uri: None, + }), + permission: Some(Permission::from_static(Permission::FULL_CONTROL)), + }]; + + Ok(S3Response::new(GetObjectAclOutput { + grants: Some(grants), + owner: Some(RUSTFS_OWNER.to_owned()), + ..Default::default() + })) + } + + async fn get_object_attributes( + &self, + req: S3Request, + ) -> S3Result> { + let mut helper = OperationHelper::new(&req, EventName::ObjectAccessedAttributes, "s3:GetObjectAttributes"); + let GetObjectAttributesInput { bucket, key, .. } = req.input.clone(); + + let Some(store) = new_object_layer_fn() else { + return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); + }; + + if let Err(e) = store + .get_object_reader(&bucket, &key, None, HeaderMap::new(), &ObjectOptions::default()) + .await + { + return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("{e}"))); + } + + let output = GetObjectAttributesOutput { + delete_marker: None, + object_parts: None, + ..Default::default() + }; + + let version_id = req.input.version_id.clone().unwrap_or_default(); + helper = helper + .object(ObjectInfo { + name: key.clone(), + bucket, + ..Default::default() + }) + .version_id(version_id); + + let result = Ok(S3Response::new(output)); + let _ = helper.complete(&result); + result + } + + async fn get_object_legal_hold( + &self, + req: S3Request, + ) -> S3Result> { + let mut helper = OperationHelper::new(&req, EventName::ObjectAccessedGetLegalHold, "s3:GetObjectLegalHold"); + let GetObjectLegalHoldInput { + bucket, key, version_id, .. + } = req.input.clone(); + + let Some(store) = new_object_layer_fn() else { + return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); + }; + + let _ = store + .get_bucket_info(&bucket, &BucketOptions::default()) + .await + .map_err(ApiError::from)?; + + // check object lock + validate_bucket_object_lock_enabled(&bucket).await?; + + let opts: ObjectOptions = get_opts(&bucket, &key, version_id, None, &req.headers) + .await + .map_err(ApiError::from)?; + + let object_info = store.get_object_info(&bucket, &key, &opts).await.map_err(|e| { + error!("get_object_info failed, {}", e.to_string()); + s3_error!(InternalError, "{}", e.to_string()) + })?; + + let legal_hold = object_info + .user_defined + .get(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER) + .map(|v| v.as_str().to_string()); + + let status = if let Some(v) = legal_hold { + v + } else { + ObjectLockLegalHoldStatus::OFF.to_string() + }; + + let output = GetObjectLegalHoldOutput { + legal_hold: Some(ObjectLockLegalHold { + status: Some(ObjectLockLegalHoldStatus::from(status)), + }), + }; + + let version_id = req.input.version_id.clone().unwrap_or_else(|| Uuid::new_v4().to_string()); + helper = helper.object(object_info).version_id(version_id); + + let result = Ok(S3Response::new(output)); + let _ = helper.complete(&result); + result + } + + #[instrument(level = "debug", skip(self))] + async fn get_object_lock_configuration( + &self, + req: S3Request, + ) -> S3Result> { + let GetObjectLockConfigurationInput { bucket, .. } = req.input; + + let object_lock_configuration = match metadata_sys::get_object_lock_config(&bucket).await { + Ok((cfg, _created)) => Some(cfg), + Err(err) => { + if err == StorageError::ConfigNotFound { + return Err(S3Error::with_message( + S3ErrorCode::ObjectLockConfigurationNotFoundError, + "Object Lock configuration does not exist for this bucket".to_string(), + )); + } + warn!("get_object_lock_config err {:?}", err); + return Err(S3Error::with_message( + S3ErrorCode::InternalError, + "Failed to load Object Lock configuration".to_string(), + )); + } + }; + + // warn!("object_lock_configuration {:?}", &object_lock_configuration); + + Ok(S3Response::new(GetObjectLockConfigurationOutput { + object_lock_configuration, + })) + } + + async fn get_object_retention( + &self, + req: S3Request, + ) -> S3Result> { + let mut helper = OperationHelper::new(&req, EventName::ObjectAccessedGetRetention, "s3:GetObjectRetention"); + let GetObjectRetentionInput { + bucket, key, version_id, .. + } = req.input.clone(); + + let Some(store) = new_object_layer_fn() else { + return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); + }; + + // check object lock + validate_bucket_object_lock_enabled(&bucket).await?; + + let opts: ObjectOptions = get_opts(&bucket, &key, version_id, None, &req.headers) + .await + .map_err(ApiError::from)?; + + let object_info = store.get_object_info(&bucket, &key, &opts).await.map_err(|e| { + error!("get_object_info failed, {}", e.to_string()); + s3_error!(InternalError, "{}", e.to_string()) + })?; + + let mode = object_info + .user_defined + .get("x-amz-object-lock-mode") + .map(|v| ObjectLockRetentionMode::from(v.as_str().to_string())); + + let retain_until_date = object_info + .user_defined + .get("x-amz-object-lock-retain-until-date") + .and_then(|v| OffsetDateTime::parse(v.as_str(), &Rfc3339).ok()) + .map(Timestamp::from); + + let output = GetObjectRetentionOutput { + retention: Some(ObjectLockRetention { mode, retain_until_date }), + }; + let version_id = req.input.version_id.clone().unwrap_or_default(); + helper = helper.object(object_info).version_id(version_id); + + let result = Ok(S3Response::new(output)); + let _ = helper.complete(&result); + result + } + + #[instrument(level = "debug", skip(self))] + async fn get_object_tagging(&self, req: S3Request) -> S3Result> { + let start_time = std::time::Instant::now(); + let GetObjectTaggingInput { bucket, key: object, .. } = req.input; + + info!("Starting get_object_tagging for bucket: {}, object: {}", bucket, object); + + let Some(store) = new_object_layer_fn() else { + error!("Store not initialized"); + return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); + }; + + // Support versioned objects + let version_id = req.input.version_id.clone(); + let opts = ObjectOptions { + version_id: self.parse_version_id(version_id)?.map(Into::into), + ..Default::default() + }; + + let tags = store.get_object_tags(&bucket, &object, &opts).await.map_err(|e| { + if is_err_object_not_found(&e) { + error!("Object not found: {}", e); + return s3_error!(NoSuchKey); + } + error!("Failed to get object tags: {}", e); + ApiError::from(e).into() + })?; + + let tag_set = decode_tags(tags.as_str()); + debug!("Decoded tag set: {:?}", tag_set); + + // Add metrics + counter!("rustfs.get_object_tagging.success").increment(1); + let duration = start_time.elapsed(); + histogram!("rustfs.object_tagging.operation.duration.seconds", "operation" => "put").record(duration.as_secs_f64()); + Ok(S3Response::new(GetObjectTaggingOutput { + tag_set, + version_id: req.input.version_id.clone(), + })) + } + #[instrument(level = "debug", skip(self, _req))] async fn get_object_torrent(&self, _req: S3Request) -> S3Result> { // Torrent functionality is not implemented in RustFS @@ -3521,6 +3711,162 @@ impl S3 for FS { Ok(S3Response::new(output)) } + async fn list_multipart_uploads( + &self, + req: S3Request, + ) -> S3Result> { + let ListMultipartUploadsInput { + bucket, + prefix, + delimiter, + key_marker, + upload_id_marker, + max_uploads, + .. + } = req.input; + + let Some(store) = new_object_layer_fn() else { + return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); + }; + + let prefix = prefix.unwrap_or_default(); + + let max_uploads = max_uploads.map(|x| x as usize).unwrap_or(MAX_PARTS_COUNT); + + if let Some(key_marker) = &key_marker + && !key_marker.starts_with(prefix.as_str()) + { + return Err(s3_error!(NotImplemented, "Invalid key marker")); + } + + let result = store + .list_multipart_uploads(&bucket, &prefix, delimiter, key_marker, upload_id_marker, max_uploads) + .await + .map_err(ApiError::from)?; + + let output = ListMultipartUploadsOutput { + bucket: Some(bucket), + prefix: Some(prefix), + delimiter: result.delimiter, + key_marker: result.key_marker, + upload_id_marker: result.upload_id_marker, + max_uploads: Some(result.max_uploads as i32), + is_truncated: Some(result.is_truncated), + uploads: Some( + result + .uploads + .into_iter() + .map(|u| MultipartUpload { + key: Some(u.object), + upload_id: Some(u.upload_id), + initiated: u.initiated.map(Timestamp::from), + + ..Default::default() + }) + .collect(), + ), + common_prefixes: Some( + result + .common_prefixes + .into_iter() + .map(|c| CommonPrefix { prefix: Some(c) }) + .collect(), + ), + ..Default::default() + }; + + Ok(S3Response::new(output)) + } + + async fn list_object_versions( + &self, + req: S3Request, + ) -> S3Result> { + let ListObjectVersionsInput { + bucket, + delimiter, + key_marker, + version_id_marker, + max_keys, + prefix, + .. + } = req.input; + + let prefix = prefix.unwrap_or_default(); + let max_keys = max_keys.unwrap_or(1000); + + let key_marker = key_marker.filter(|v| !v.is_empty()); + let version_id_marker = version_id_marker.filter(|v| !v.is_empty()); + let delimiter = delimiter.filter(|v| !v.is_empty()); + + let store = get_validated_store(&bucket).await?; + + let object_infos = store + .list_object_versions(&bucket, &prefix, key_marker, version_id_marker, delimiter.clone(), max_keys) + .await + .map_err(ApiError::from)?; + + let objects: Vec = object_infos + .objects + .iter() + .filter(|v| !v.name.is_empty() && !v.delete_marker) + .map(|v| { + ObjectVersion { + key: Some(v.name.to_owned()), + last_modified: v.mod_time.map(Timestamp::from), + size: Some(v.size), + version_id: Some(v.version_id.map(|v| v.to_string()).unwrap_or_else(|| "null".to_string())), + is_latest: Some(v.is_latest), + e_tag: v.etag.clone().map(|etag| to_s3s_etag(&etag)), + storage_class: v.storage_class.clone().map(ObjectVersionStorageClass::from), + ..Default::default() // TODO: another fields + } + }) + .collect(); + + let common_prefixes = object_infos + .prefixes + .into_iter() + .map(|v| CommonPrefix { prefix: Some(v) }) + .collect(); + + let delete_markers = object_infos + .objects + .iter() + .filter(|o| o.delete_marker) + .map(|o| DeleteMarkerEntry { + key: Some(o.name.clone()), + version_id: Some(o.version_id.map(|v| v.to_string()).unwrap_or_else(|| "null".to_string())), + is_latest: Some(o.is_latest), + last_modified: o.mod_time.map(Timestamp::from), + ..Default::default() + }) + .collect::>(); + + // Only set next_key_marker and next_version_id_marker if they have values, per AWS S3 API spec + // boto3 expects them to be strings or omitted, not None or empty strings + let next_key_marker = object_infos.next_marker.filter(|v| !v.is_empty()); + let next_version_id_marker = object_infos.next_version_idmarker.filter(|v| !v.is_empty()); + + let output = ListObjectVersionsOutput { + is_truncated: Some(object_infos.is_truncated), + // max_keys should be the requested maximum number of keys, not the actual count returned + // Per AWS S3 API spec, this field represents the maximum number of keys that can be returned in the response + max_keys: Some(max_keys), + delimiter, + name: Some(bucket), + prefix: Some(prefix), + common_prefixes: Some(common_prefixes), + versions: Some(objects), + delete_markers: Some(delete_markers), + next_key_marker, + next_version_id_marker, + ..Default::default() + }; + + Ok(S3Response::new(output)) + } + #[instrument(level = "debug", skip(self, req))] async fn list_objects(&self, req: S3Request) -> S3Result> { // Capture the original marker from the request before conversion @@ -3737,96 +4083,388 @@ impl S3 for FS { Ok(S3Response::new(output)) } - async fn list_object_versions( - &self, - req: S3Request, - ) -> S3Result> { - let ListObjectVersionsInput { + #[instrument(level = "debug", skip(self, req))] + async fn list_parts(&self, req: S3Request) -> S3Result> { + let ListPartsInput { bucket, - delimiter, - key_marker, - version_id_marker, - max_keys, - prefix, + key, + upload_id, + part_number_marker, + max_parts, .. } = req.input; - let prefix = prefix.unwrap_or_default(); - let max_keys = max_keys.unwrap_or(1000); + let Some(store) = new_object_layer_fn() else { + return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); + }; - let key_marker = key_marker.filter(|v| !v.is_empty()); - let version_id_marker = version_id_marker.filter(|v| !v.is_empty()); - let delimiter = delimiter.filter(|v| !v.is_empty()); + let part_number_marker = part_number_marker.map(|x| x as usize); + let max_parts = match max_parts { + Some(parts) => { + if !(1..=1000).contains(&parts) { + return Err(s3_error!(InvalidArgument, "max-parts must be between 1 and 1000")); + } + parts as usize + } + None => 1000, + }; - let store = get_validated_store(&bucket).await?; - - let object_infos = store - .list_object_versions(&bucket, &prefix, key_marker, version_id_marker, delimiter.clone(), max_keys) + let res = store + .list_object_parts(&bucket, &key, &upload_id, part_number_marker, max_parts, &ObjectOptions::default()) .await .map_err(ApiError::from)?; - let objects: Vec = object_infos - .objects - .iter() - .filter(|v| !v.name.is_empty() && !v.delete_marker) - .map(|v| { - ObjectVersion { - key: Some(v.name.to_owned()), - last_modified: v.mod_time.map(Timestamp::from), - size: Some(v.size), - version_id: Some(v.version_id.map(|v| v.to_string()).unwrap_or_else(|| "null".to_string())), - is_latest: Some(v.is_latest), - e_tag: v.etag.clone().map(|etag| to_s3s_etag(&etag)), - storage_class: v.storage_class.clone().map(ObjectVersionStorageClass::from), - ..Default::default() // TODO: another fields - } - }) - .collect(); - - let common_prefixes = object_infos - .prefixes - .into_iter() - .map(|v| CommonPrefix { prefix: Some(v) }) - .collect(); - - let delete_markers = object_infos - .objects - .iter() - .filter(|o| o.delete_marker) - .map(|o| DeleteMarkerEntry { - key: Some(o.name.clone()), - version_id: Some(o.version_id.map(|v| v.to_string()).unwrap_or_else(|| "null".to_string())), - is_latest: Some(o.is_latest), - last_modified: o.mod_time.map(Timestamp::from), - ..Default::default() - }) - .collect::>(); - - // Only set next_key_marker and next_version_id_marker if they have values, per AWS S3 API spec - // boto3 expects them to be strings or omitted, not None or empty strings - let next_key_marker = object_infos.next_marker.filter(|v| !v.is_empty()); - let next_version_id_marker = object_infos.next_version_idmarker.filter(|v| !v.is_empty()); - - let output = ListObjectVersionsOutput { - is_truncated: Some(object_infos.is_truncated), - // max_keys should be the requested maximum number of keys, not the actual count returned - // Per AWS S3 API spec, this field represents the maximum number of keys that can be returned in the response - max_keys: Some(max_keys), - delimiter, - name: Some(bucket), - prefix: Some(prefix), - common_prefixes: Some(common_prefixes), - versions: Some(objects), - delete_markers: Some(delete_markers), - next_key_marker, - next_version_id_marker, + let output = ListPartsOutput { + bucket: Some(res.bucket), + key: Some(res.object), + upload_id: Some(res.upload_id), + parts: Some( + res.parts + .into_iter() + .map(|p| Part { + e_tag: p.etag.map(|etag| to_s3s_etag(&etag)), + last_modified: p.last_mod.map(Timestamp::from), + part_number: Some(p.part_num as i32), + size: Some(p.size as i64), + ..Default::default() + }) + .collect(), + ), + owner: Some(RUSTFS_OWNER.to_owned()), + initiator: Some(Initiator { + id: RUSTFS_OWNER.id.clone(), + display_name: RUSTFS_OWNER.display_name.clone(), + }), + is_truncated: Some(res.is_truncated), + next_part_number_marker: res.next_part_number_marker.try_into().ok(), + max_parts: res.max_parts.try_into().ok(), + part_number_marker: res.part_number_marker.try_into().ok(), + storage_class: if res.storage_class.is_empty() { + None + } else { + Some(res.storage_class.into()) + }, ..Default::default() }; - Ok(S3Response::new(output)) } - // #[instrument(level = "debug", skip(self, req))] + async fn put_bucket_acl(&self, req: S3Request) -> S3Result> { + let PutBucketAclInput { + bucket, + acl, + access_control_policy, + .. + } = req.input; + + // TODO:checkRequestAuthType + + let Some(store) = new_object_layer_fn() else { + return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); + }; + + store + .get_bucket_info(&bucket, &BucketOptions::default()) + .await + .map_err(ApiError::from)?; + + if let Some(canned_acl) = acl { + if canned_acl.as_str() != BucketCannedACL::PRIVATE { + return Err(s3_error!(NotImplemented)); + } + } else { + let is_full_control = access_control_policy.is_some_and(|v| { + v.grants.is_some_and(|gs| { + // + !gs.is_empty() + && gs.first().is_some_and(|g| { + g.to_owned() + .permission + .is_some_and(|p| p.as_str() == Permission::FULL_CONTROL) + }) + }) + }); + + if !is_full_control { + return Err(s3_error!(NotImplemented)); + } + } + Ok(S3Response::new(PutBucketAclOutput::default())) + } + + #[instrument(level = "debug", skip(self))] + async fn put_bucket_cors(&self, req: S3Request) -> S3Result> { + let PutBucketCorsInput { + bucket, + cors_configuration, + .. + } = req.input; + + let Some(store) = new_object_layer_fn() else { + return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); + }; + + store + .get_bucket_info(&bucket, &BucketOptions::default()) + .await + .map_err(ApiError::from)?; + + let data = try_!(serialize(&cors_configuration)); + + metadata_sys::update(&bucket, BUCKET_CORS_CONFIG, data) + .await + .map_err(ApiError::from)?; + + Ok(S3Response::new(PutBucketCorsOutput::default())) + } + + async fn put_bucket_encryption( + &self, + req: S3Request, + ) -> S3Result> { + let PutBucketEncryptionInput { + bucket, + server_side_encryption_configuration, + .. + } = req.input; + + info!("sse_config {:?}", &server_side_encryption_configuration); + + let Some(store) = new_object_layer_fn() else { + return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); + }; + + store + .get_bucket_info(&bucket, &BucketOptions::default()) + .await + .map_err(ApiError::from)?; + + // TODO: check kms + + let data = try_!(serialize(&server_side_encryption_configuration)); + metadata_sys::update(&bucket, BUCKET_SSECONFIG, data) + .await + .map_err(ApiError::from)?; + Ok(S3Response::new(PutBucketEncryptionOutput::default())) + } + + #[instrument(level = "debug", skip(self))] + async fn put_bucket_lifecycle_configuration( + &self, + req: S3Request, + ) -> S3Result> { + let PutBucketLifecycleConfigurationInput { + bucket, + lifecycle_configuration, + .. + } = req.input; + + let Some(input_cfg) = lifecycle_configuration else { return Err(s3_error!(InvalidArgument)) }; + + let rcfg = metadata_sys::get_object_lock_config(&bucket).await; + if let Ok(rcfg) = rcfg + && let Err(err) = input_cfg.validate(&rcfg.0).await + { + //return Err(S3Error::with_message(S3ErrorCode::Custom("BucketLockValidateFailed".into()), err.to_string())); + return Err(S3Error::with_message(S3ErrorCode::Custom("ValidateFailed".into()), err.to_string())); + } + + if let Err(err) = validate_transition_tier(&input_cfg).await { + //warn!("lifecycle_configuration add failed, err: {:?}", err); + return Err(S3Error::with_message(S3ErrorCode::Custom("CustomError".into()), err.to_string())); + } + + let data = try_!(serialize(&input_cfg)); + metadata_sys::update(&bucket, BUCKET_LIFECYCLE_CONFIG, data) + .await + .map_err(ApiError::from)?; + + Ok(S3Response::new(PutBucketLifecycleConfigurationOutput::default())) + } + + async fn put_bucket_notification_configuration( + &self, + req: S3Request, + ) -> S3Result> { + let PutBucketNotificationConfigurationInput { + bucket, + notification_configuration, + .. + } = req.input; + + let Some(store) = new_object_layer_fn() else { + return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); + }; + + // Verify that the bucket exists + store + .get_bucket_info(&bucket, &BucketOptions::default()) + .await + .map_err(ApiError::from)?; + + // Persist the new notification configuration + let data = try_!(serialize(¬ification_configuration)); + metadata_sys::update(&bucket, BUCKET_NOTIFICATION_CONFIG, data) + .await + .map_err(ApiError::from)?; + + // Determine region (BucketInfo has no region field) -> use global region or default + let region = rustfs_ecstore::global::get_global_region().unwrap_or_else(|| req.region.clone().unwrap_or_default()); + + // Purge old rules and resolve new rules in parallel + let clear_rules = notifier_global::clear_bucket_notification_rules(&bucket); + let parse_rules = async { + let mut event_rules = Vec::new(); + + process_queue_configurations(&mut event_rules, notification_configuration.queue_configurations.clone(), |arn_str| { + ARN::parse(arn_str) + .map(|arn| arn.target_id) + .map_err(|e| TargetIDError::InvalidFormat(e.to_string())) + })?; + process_topic_configurations(&mut event_rules, notification_configuration.topic_configurations.clone(), |arn_str| { + ARN::parse(arn_str) + .map(|arn| arn.target_id) + .map_err(|e| TargetIDError::InvalidFormat(e.to_string())) + })?; + process_lambda_configurations( + &mut event_rules, + notification_configuration.lambda_function_configurations.clone(), + |arn_str| { + ARN::parse(arn_str) + .map(|arn| arn.target_id) + .map_err(|e| TargetIDError::InvalidFormat(e.to_string())) + }, + )?; + + Ok::<_, TargetIDError>(event_rules) + }; + + let (clear_result, event_rules_result) = tokio::join!(clear_rules, parse_rules); + + clear_result.map_err(|e| s3_error!(InternalError, "Failed to clear rules: {e}"))?; + let event_rules = + event_rules_result.map_err(|e| s3_error!(InvalidArgument, "Invalid ARN in notification configuration: {e}"))?; + warn!("notify event rules: {:?}", &event_rules); + + // Add a new notification rule + notifier_global::add_event_specific_rules(&bucket, ®ion, &event_rules) + .await + .map_err(|e| s3_error!(InternalError, "Failed to add rules: {e}"))?; + + Ok(S3Response::new(PutBucketNotificationConfigurationOutput {})) + } + + async fn put_bucket_policy(&self, req: S3Request) -> S3Result> { + let PutBucketPolicyInput { bucket, policy, .. } = req.input; + + let Some(store) = new_object_layer_fn() else { + return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); + }; + + store + .get_bucket_info(&bucket, &BucketOptions::default()) + .await + .map_err(ApiError::from)?; + + // warn!("input policy {}", &policy); + + let cfg: BucketPolicy = + serde_json::from_str(&policy).map_err(|e| s3_error!(InvalidArgument, "parse policy failed {:?}", e))?; + + if let Err(err) = cfg.is_valid() { + warn!("put_bucket_policy err input {:?}, {:?}", &policy, err); + return Err(s3_error!(InvalidPolicyDocument)); + } + + let data = serde_json::to_vec(&cfg).map_err(|e| s3_error!(InternalError, "parse policy failed {:?}", e))?; + + metadata_sys::update(&bucket, BUCKET_POLICY_CONFIG, data) + .await + .map_err(ApiError::from)?; + + Ok(S3Response::new(PutBucketPolicyOutput {})) + } + + async fn put_bucket_replication( + &self, + req: S3Request, + ) -> S3Result> { + let PutBucketReplicationInput { + bucket, + replication_configuration, + .. + } = req.input; + warn!("put bucket replication"); + + let Some(store) = new_object_layer_fn() else { + return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); + }; + + store + .get_bucket_info(&bucket, &BucketOptions::default()) + .await + .map_err(ApiError::from)?; + + // TODO: check enable, versioning enable + let data = try_!(serialize(&replication_configuration)); + + metadata_sys::update(&bucket, BUCKET_REPLICATION_CONFIG, data) + .await + .map_err(ApiError::from)?; + + Ok(S3Response::new(PutBucketReplicationOutput::default())) + } + + #[instrument(level = "debug", skip(self))] + async fn put_bucket_tagging(&self, req: S3Request) -> S3Result> { + let PutBucketTaggingInput { bucket, tagging, .. } = req.input; + + let Some(store) = new_object_layer_fn() else { + return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); + }; + + store + .get_bucket_info(&bucket, &BucketOptions::default()) + .await + .map_err(ApiError::from)?; + + let data = try_!(serialize(&tagging)); + + metadata_sys::update(&bucket, BUCKET_TAGGING_CONFIG, data) + .await + .map_err(ApiError::from)?; + + Ok(S3Response::new(Default::default())) + } + + #[instrument(level = "debug", skip(self))] + async fn put_bucket_versioning( + &self, + req: S3Request, + ) -> S3Result> { + let PutBucketVersioningInput { + bucket, + versioning_configuration, + .. + } = req.input; + + // TODO: check other sys + // check site replication enable + // check bucket object lock enable + // check replication suspended + + let data = try_!(serialize(&versioning_configuration)); + + metadata_sys::update(&bucket, BUCKET_VERSIONING_CONFIG, data) + .await + .map_err(ApiError::from)?; + + // TODO: globalSiteReplicationSys.BucketMetaHook + + Ok(S3Response::new(PutBucketVersioningOutput {})) + } + + #[instrument(level = "debug", skip(self, req))] async fn put_object(&self, req: S3Request) -> S3Result> { let mut helper = OperationHelper::new(&req, EventName::ObjectCreatedPut, "s3:PutObject"); if req @@ -4057,6 +4695,7 @@ impl S3 for FS { if is_compressible(&req.headers, &key) && size > MIN_COMPRESSIBLE_SIZE as i64 { let algorithm = CompressionAlgorithm::default(); metadata.insert(format!("{RESERVED_METADATA_PREFIX_LOWER}compression"), algorithm.to_string()); + metadata.insert(format!("{RESERVED_METADATA_PREFIX_LOWER}actual-size",), size.to_string()); let mut hrd = HashReader::new(reader, size as i64, size as i64, md5hex, sha256hex, false).map_err(ApiError::from)?; @@ -4254,164 +4893,672 @@ impl S3 for FS { ..Default::default() }; + // TODO fix response for POST Policy (multipart/form-data) ，wait s3s crate update,fix issue #1564 + // // If it is a POST Policy(multipart/form-data) path, the PutObjectInput carries the success_action_* field + // // Here, the response is uniformly rewritten, with the default being 204, redirect prioritizing 303, and status supporting 200/201/204 + // if input.success_action_status.is_some() || input.success_action_redirect.is_some() { + // let mut form_fields = HashMap::::new(); + // if let Some(v) = &input.success_action_status { + // form_fields.insert("success_action_status".to_string(), v.to_string()); + // } + // if let Some(v) = &input.success_action_redirect { + // form_fields.insert("success_action_redirect".to_string(), v.to_string()); + // } + // + // // obj_info.etag has been converted to e_tag (s3s etag) above, so try to pass the original string here + // let etag_str = e_tag.as_ref().map(|v| v.as_str()); + // + // // Returns using POST semantics: 204/303/201/200 + // let resp = build_post_object_success_response(&form_fields, &bucket, &key, etag_str, None)?; + // + // // Keep helper event complete (note: (StatusCode, Body) is returned here instead of PutObjectOutput) + // let result = Ok(resp); + // let _ = helper.complete(&result); + // return result; + // } + + let result = Ok(S3Response::new(output)); + let _ = helper.complete(&result); + result + } + + async fn put_object_acl(&self, req: S3Request) -> S3Result> { + let PutObjectAclInput { + bucket, + key, + acl, + access_control_policy, + .. + } = req.input; + + let Some(store) = new_object_layer_fn() else { + return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); + }; + + if let Err(e) = store.get_object_info(&bucket, &key, &ObjectOptions::default()).await { + return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("{e}"))); + } + + if let Some(canned_acl) = acl { + if canned_acl.as_str() != BucketCannedACL::PRIVATE { + return Err(s3_error!(NotImplemented)); + } + } else { + let is_full_control = access_control_policy.is_some_and(|v| { + v.grants.is_some_and(|gs| { + // + !gs.is_empty() + && gs.first().is_some_and(|g| { + g.to_owned() + .permission + .is_some_and(|p| p.as_str() == Permission::FULL_CONTROL) + }) + }) + }); + + if !is_full_control { + return Err(s3_error!(NotImplemented)); + } + } + Ok(S3Response::new(PutObjectAclOutput::default())) + } + + async fn put_object_legal_hold( + &self, + req: S3Request, + ) -> S3Result> { + let mut helper = OperationHelper::new(&req, EventName::ObjectCreatedPutLegalHold, "s3:PutObjectLegalHold"); + let PutObjectLegalHoldInput { + bucket, + key, + legal_hold, + version_id, + .. + } = req.input.clone(); + + let Some(store) = new_object_layer_fn() else { + return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); + }; + + let _ = store + .get_bucket_info(&bucket, &BucketOptions::default()) + .await + .map_err(ApiError::from)?; + + // check object lock + validate_bucket_object_lock_enabled(&bucket).await?; + let opts: ObjectOptions = get_opts(&bucket, &key, version_id, None, &req.headers) + .await + .map_err(ApiError::from)?; + + let eval_metadata = parse_object_lock_legal_hold(legal_hold)?; + + let popts = ObjectOptions { + mod_time: opts.mod_time, + version_id: opts.version_id, + eval_metadata: Some(eval_metadata), + ..Default::default() + }; + + let info = store.put_object_metadata(&bucket, &key, &popts).await.map_err(|e| { + error!("put_object_metadata failed, {}", e.to_string()); + s3_error!(InternalError, "{}", e.to_string()) + })?; + + let output = PutObjectLegalHoldOutput { + request_charged: Some(RequestCharged::from_static(RequestCharged::REQUESTER)), + }; + let version_id = req.input.version_id.clone().unwrap_or_default(); + helper = helper.object(info).version_id(version_id); + + let result = Ok(S3Response::new(output)); + let _ = helper.complete(&result); + result + } + + #[instrument(level = "debug", skip(self))] + async fn put_object_lock_configuration( + &self, + req: S3Request, + ) -> S3Result> { + let PutObjectLockConfigurationInput { + bucket, + object_lock_configuration, + .. + } = req.input; + + let Some(input_cfg) = object_lock_configuration else { return Err(s3_error!(InvalidArgument)) }; + + let Some(store) = new_object_layer_fn() else { + return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); + }; + + store + .get_bucket_info(&bucket, &BucketOptions::default()) + .await + .map_err(ApiError::from)?; + + let _ = match metadata_sys::get_object_lock_config(&bucket).await { + Ok(_) => {} + Err(err) => { + if err == StorageError::ConfigNotFound { + return Err(S3Error::with_message( + S3ErrorCode::InvalidBucketState, + "Object Lock configuration cannot be enabled on existing buckets".to_string(), + )); + } + warn!("get_object_lock_config err {:?}", err); + return Err(S3Error::with_message( + S3ErrorCode::InternalError, + "Failed to get bucket ObjectLockConfiguration".to_string(), + )); + } + }; + + let data = try_!(serialize(&input_cfg)); + + metadata_sys::update(&bucket, OBJECT_LOCK_CONFIG, data) + .await + .map_err(ApiError::from)?; + + // When Object Lock is enabled, automatically enable versioning if not already enabled + // This matches AWS S3 and MinIO behavior + let versioning_config = BucketVersioningSys::get(&bucket).await.map_err(ApiError::from)?; + if !versioning_config.enabled() { + let enable_versioning_config = VersioningConfiguration { + status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)), + ..Default::default() + }; + let versioning_data = try_!(serialize(&enable_versioning_config)); + metadata_sys::update(&bucket, BUCKET_VERSIONING_CONFIG, versioning_data) + .await + .map_err(ApiError::from)?; + } + + Ok(S3Response::new(PutObjectLockConfigurationOutput::default())) + } + + async fn put_object_retention( + &self, + req: S3Request, + ) -> S3Result> { + let mut helper = OperationHelper::new(&req, EventName::ObjectCreatedPutRetention, "s3:PutObjectRetention"); + let PutObjectRetentionInput { + bucket, + key, + retention, + version_id, + .. + } = req.input.clone(); + + let Some(store) = new_object_layer_fn() else { + return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); + }; + + // check object lock + validate_bucket_object_lock_enabled(&bucket).await?; + + // Extract new retain_until_date for validation + let new_retain_until = retention + .as_ref() + .and_then(|r| r.retain_until_date.as_ref()) + .map(|d| OffsetDateTime::from(d.clone())); + + // Check if object already has retention and if modification is allowed + // This follows AWS S3 and MinIO behavior: + // - COMPLIANCE mode: can only extend retention period, never shorten + // - GOVERNANCE mode: requires bypass header to modify/shorten retention + // + // TODO: Known race condition (fix in future PR) + // There's a TOCTOU (time-of-check-time-of-use) window between retention check here + // and the actual update at put_object_metadata. In theory: + // Thread A: reads GOVERNANCE mode, checks bypass header + // Thread B: updates retention to COMPLIANCE mode + // Thread A: modifies retention, bypassing what is now COMPLIANCE mode + // This violates S3 spec that COMPLIANCE cannot be modified even with bypass. + // Fix options: + // 1. Pass expected retention mode to storage layer, verify before update + // 2. Use optimistic concurrency with version/etag checks + // 3. Perform check within the same lock scope as update in storage layer + // Current mitigation: Storage layer has fast_lock_manager which provides some protection + let check_opts: ObjectOptions = get_opts(&bucket, &key, version_id.clone(), None, &req.headers) + .await + .map_err(ApiError::from)?; + + if let Ok(existing_obj_info) = store.get_object_info(&bucket, &key, &check_opts).await { + let bypass_governance = has_bypass_governance_header(&req.headers); + if let Some(block_reason) = + check_retention_for_modification(&existing_obj_info.user_defined, new_retain_until, bypass_governance) + { + return Err(S3Error::with_message(S3ErrorCode::AccessDenied, block_reason.error_message())); + } + } + + let eval_metadata = parse_object_lock_retention(retention)?; + + let mut opts: ObjectOptions = get_opts(&bucket, &key, version_id, None, &req.headers) + .await + .map_err(ApiError::from)?; + opts.eval_metadata = Some(eval_metadata); + + let object_info = store.put_object_metadata(&bucket, &key, &opts).await.map_err(|e| { + error!("put_object_metadata failed, {}", e.to_string()); + s3_error!(InternalError, "{}", e.to_string()) + })?; + + let output = PutObjectRetentionOutput { + request_charged: Some(RequestCharged::from_static(RequestCharged::REQUESTER)), + }; + + let version_id = req.input.version_id.clone().unwrap_or_else(|| Uuid::new_v4().to_string()); + helper = helper.object(object_info).version_id(version_id); + let result = Ok(S3Response::new(output)); let _ = helper.complete(&result); result } #[instrument(level = "debug", skip(self, req))] - async fn create_multipart_upload( - &self, - req: S3Request, - ) -> S3Result> { - let helper = OperationHelper::new(&req, EventName::ObjectCreatedPut, "s3:CreateMultipartUpload"); - let CreateMultipartUploadInput { + async fn put_object_tagging(&self, req: S3Request) -> S3Result> { + let start_time = std::time::Instant::now(); + let mut helper = OperationHelper::new(&req, EventName::ObjectCreatedPutTagging, "s3:PutObjectTagging"); + let PutObjectTaggingInput { bucket, - key, + key: object, tagging, - version_id, - storage_class, - server_side_encryption, - sse_customer_algorithm, - sse_customer_key_md5, - ssekms_key_id, .. } = req.input.clone(); - // Validate storage class if provided - if let Some(ref storage_class) = storage_class - && !is_valid_storage_class(storage_class.as_str()) - { - return Err(s3_error!(InvalidStorageClass)); + if tagging.tag_set.len() > 10 { + // TOTO: Note that Amazon S3 limits the maximum number of tags to 10 tags per object. + // Reference: https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-tagging.html + // Reference: https://docs.aws.amazon.com/zh_cn/AmazonS3/latest/API/API_PutObjectTagging.html + // https://github.com/minio/mint/blob/master/run/core/aws-sdk-go-v2/main.go#L1647 + error!("Tag set exceeds maximum of 10 tags: {}", tagging.tag_set.len()); + return Err(s3_error!(InvalidTag, "Cannot have more than 10 tags per object")); } - // mc cp step 3 - - // debug!("create_multipart_upload meta {:?}", &metadata); - let Some(store) = new_object_layer_fn() else { return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); }; - let mut metadata = extract_metadata(&req.headers); + let mut tag_keys = std::collections::HashSet::with_capacity(tagging.tag_set.len()); + for tag in &tagging.tag_set { + let key = tag.key.as_ref().filter(|k| !k.is_empty()).ok_or_else(|| { + error!("Empty tag key"); + s3_error!(InvalidTag, "Tag key cannot be empty") + })?; - if let Some(tags) = tagging { - metadata.insert(AMZ_OBJECT_TAGGING.to_owned(), tags); - } + if key.len() > 128 { + error!("Tag key too long: {} bytes", key.len()); + return Err(s3_error!(InvalidTag, "Tag key is too long, maximum allowed length is 128 characters")); + } - // TDD: Get bucket SSE configuration for multipart upload - let bucket_sse_config = metadata_sys::get_sse_config(&bucket).await.ok(); - debug!("TDD: Got bucket SSE config for multipart: {:?}", bucket_sse_config); + let value = tag.value.as_ref().ok_or_else(|| { + error!("Null tag value"); + s3_error!(InvalidTag, "Tag value cannot be null") + })?; - // TDD: Determine effective encryption (request parameters override bucket defaults) - let original_sse = server_side_encryption.clone(); - let effective_sse = server_side_encryption.or_else(|| { - bucket_sse_config.as_ref().and_then(|(config, _timestamp)| { - debug!("TDD: Processing bucket SSE config for multipart: {:?}", config); - config.rules.first().and_then(|rule| { - debug!("TDD: Processing SSE rule for multipart: {:?}", rule); - rule.apply_server_side_encryption_by_default.as_ref().map(|sse| { - debug!("TDD: Found SSE default for multipart: {:?}", sse); - match sse.sse_algorithm.as_str() { - "AES256" => ServerSideEncryption::from_static(ServerSideEncryption::AES256), - "aws:kms" => ServerSideEncryption::from_static(ServerSideEncryption::AWS_KMS), - _ => ServerSideEncryption::from_static(ServerSideEncryption::AES256), // fallback to AES256 - } - }) - }) - }) - }); - debug!("TDD: effective_sse for multipart={:?} (original={:?})", effective_sse, original_sse); + if value.is_empty() { + error!("Empty tag value"); + return Err(s3_error!(InvalidTag, "Tag value cannot be empty")); + } - let _original_kms_key_id = ssekms_key_id.clone(); - let mut effective_kms_key_id = ssekms_key_id.or_else(|| { - bucket_sse_config.as_ref().and_then(|(config, _timestamp)| { - config.rules.first().and_then(|rule| { - rule.apply_server_side_encryption_by_default - .as_ref() - .and_then(|sse| sse.kms_master_key_id.clone()) - }) - }) - }); + if value.len() > 256 { + error!("Tag value too long: {} bytes", value.len()); + return Err(s3_error!(InvalidTag, "Tag value is too long, maximum allowed length is 256 characters")); + } - // Store effective SSE information in metadata for multipart upload - if let Some(sse_alg) = &sse_customer_algorithm { - metadata.insert( - "x-amz-server-side-encryption-customer-algorithm".to_string(), - sse_alg.as_str().to_string(), - ); - } - if let Some(sse_md5) = &sse_customer_key_md5 { - metadata.insert("x-amz-server-side-encryption-customer-key-md5".to_string(), sse_md5.clone()); - } - - if let Some(sse) = &effective_sse { - if is_managed_sse(sse) { - let material = create_managed_encryption_material(&bucket, &key, sse, effective_kms_key_id.clone(), 0).await?; - - let ManagedEncryptionMaterial { - data_key: _, - headers, - kms_key_id: kms_key_used, - } = material; - - metadata.extend(headers.into_iter()); - effective_kms_key_id = Some(kms_key_used.clone()); - } else { - metadata.insert("x-amz-server-side-encryption".to_string(), sse.as_str().to_string()); + if !tag_keys.insert(key) { + error!("Duplicate tag key: {}", key); + return Err(s3_error!(InvalidTag, "Cannot provide multiple Tags with the same key")); } } - if let Some(kms_key_id) = &effective_kms_key_id { - metadata.insert("x-amz-server-side-encryption-aws-kms-key-id".to_string(), kms_key_id.clone()); - } + let tags = encode_tags(tagging.tag_set); + debug!("Encoded tags: {}", tags); - if is_compressible(&req.headers, &key) { - metadata.insert( - format!("{RESERVED_METADATA_PREFIX_LOWER}compression"), - CompressionAlgorithm::default().to_string(), - ); - } - - let mut opts: ObjectOptions = put_opts(&bucket, &key, version_id, &req.headers, metadata) - .await - .map_err(ApiError::from)?; - - let checksum_type = rustfs_rio::ChecksumType::from_header(&req.headers); - if checksum_type.is(rustfs_rio::ChecksumType::INVALID) { - return Err(s3_error!(InvalidArgument, "Invalid checksum type")); - } else if checksum_type.is_set() && !checksum_type.is(rustfs_rio::ChecksumType::TRAILING) { - opts.want_checksum = Some(rustfs_rio::Checksum { - checksum_type, - ..Default::default() - }); - } - - let MultipartUploadResult { - upload_id, - checksum_algo, - checksum_type, - } = store - .new_multipart_upload(&bucket, &key, &opts) - .await - .map_err(ApiError::from)?; - - let output = CreateMultipartUploadOutput { - bucket: Some(bucket), - key: Some(key), - upload_id: Some(upload_id), - server_side_encryption: effective_sse, // TDD: Return effective encryption config - sse_customer_algorithm, - ssekms_key_id: effective_kms_key_id, // TDD: Return effective KMS key ID - checksum_algorithm: checksum_algo.map(ChecksumAlgorithm::from), - checksum_type: checksum_type.map(ChecksumType::from), + // TODO: getOpts, Replicate + // Support versioned objects + let version_id = req.input.version_id.clone(); + let opts = ObjectOptions { + version_id: self.parse_version_id(version_id)?.map(Into::into), ..Default::default() }; - let result = Ok(S3Response::new(output)); + store.put_object_tags(&bucket, &object, &tags, &opts).await.map_err(|e| { + error!("Failed to put object tags: {}", e); + counter!("rustfs.put_object_tagging.failure").increment(1); + ApiError::from(e) + })?; + + // Invalidate cache for the tagged object + let manager = get_concurrency_manager(); + let version_id = req.input.version_id.clone(); + let cache_key = ConcurrencyManager::make_cache_key(&bucket, &object, version_id.clone().as_deref()); + tokio::spawn(async move { + manager + .invalidate_cache_versioned(&bucket, &object, version_id.as_deref()) + .await; + debug!("Cache invalidated for tagged object: {}", cache_key); + }); + + // Add metrics + counter!("rustfs.put_object_tagging.success").increment(1); + + let version_id_resp = req.input.version_id.clone().unwrap_or_default(); + helper = helper.version_id(version_id_resp); + + let result = Ok(S3Response::new(PutObjectTaggingOutput { + version_id: req.input.version_id.clone(), + })); let _ = helper.complete(&result); + let duration = start_time.elapsed(); + histogram!("rustfs.object_tagging.operation.duration.seconds", "operation" => "put").record(duration.as_secs_f64()); result } + async fn restore_object(&self, req: S3Request) -> S3Result> { + let RestoreObjectInput { + bucket, + key: object, + restore_request: rreq, + version_id, + .. + } = req.input.clone(); + let rreq = rreq.unwrap(); + + /*if let Err(e) = un_escape_path(object) { + warn!("post restore object failed, e: {:?}", e); + return Err(S3Error::with_message(S3ErrorCode::Custom("PostRestoreObjectFailed".into()), "post restore object failed")); + }*/ + + let Some(store) = new_object_layer_fn() else { + return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); + }; + + /*if Err(err) = check_request_auth_type(req, policy::RestoreObjectAction, bucket, object) { + return Err(S3Error::with_message(S3ErrorCode::Custom("PostRestoreObjectFailed".into()), "post restore object failed")); + }*/ + + /*if req.content_length <= 0 { + return Err(S3Error::with_message(S3ErrorCode::Custom("ErrEmptyRequestBody".into()), "post restore object failed")); + }*/ + let Ok(opts) = post_restore_opts(&version_id.unwrap(), &bucket, &object).await else { + return Err(S3Error::with_message( + S3ErrorCode::Custom("ErrEmptyRequestBody".into()), + "post restore object failed", + )); + }; + + let Ok(mut obj_info) = store.get_object_info(&bucket, &object, &opts).await else { + return Err(S3Error::with_message( + S3ErrorCode::Custom("ErrEmptyRequestBody".into()), + "post restore object failed", + )); + }; + + if obj_info.transitioned_object.status != lifecycle::TRANSITION_COMPLETE { + return Err(S3Error::with_message( + S3ErrorCode::Custom("ErrEmptyRequestBody".into()), + "post restore object failed", + )); + } + + //let mut api_err; + let mut _status_code = StatusCode::OK; + let mut already_restored = false; + if let Err(_err) = rreq.validate(store.clone()) { + //api_err = to_api_err(ErrMalformedXML); + //api_err.description = err.to_string(); + return Err(S3Error::with_message( + S3ErrorCode::Custom("ErrEmptyRequestBody".into()), + "post restore object failed", + )); + } else { + if obj_info.restore_ongoing && (rreq.type_.is_none() || rreq.type_.as_ref().unwrap().as_str() != "SELECT") { + return Err(S3Error::with_message( + S3ErrorCode::Custom("ErrObjectRestoreAlreadyInProgress".into()), + "post restore object failed", + )); + } + if !obj_info.restore_ongoing && obj_info.restore_expires.unwrap().unix_timestamp() != 0 { + _status_code = StatusCode::ACCEPTED; + already_restored = true; + } + } + let restore_expiry = lifecycle::expected_expiry_time(OffsetDateTime::now_utc(), *rreq.days.as_ref().unwrap()); + let mut metadata = obj_info.user_defined.clone(); + + let mut header = HeaderMap::new(); + + let obj_info_ = obj_info.clone(); + if rreq.type_.is_none() || rreq.type_.as_ref().unwrap().as_str() != "SELECT" { + obj_info.metadata_only = true; + metadata.insert(AMZ_RESTORE_EXPIRY_DAYS.to_string(), rreq.days.unwrap().to_string()); + metadata.insert(AMZ_RESTORE_REQUEST_DATE.to_string(), OffsetDateTime::now_utc().format(&Rfc3339).unwrap()); + if already_restored { + metadata.insert( + X_AMZ_RESTORE.as_str().to_string(), + RestoreStatus { + is_restore_in_progress: Some(false), + restore_expiry_date: Some(Timestamp::from(restore_expiry)), + } + .to_string(), + ); + } else { + metadata.insert( + X_AMZ_RESTORE.as_str().to_string(), + RestoreStatus { + is_restore_in_progress: Some(true), + restore_expiry_date: Some(Timestamp::from(OffsetDateTime::now_utc())), + } + .to_string(), + ); + } + obj_info.user_defined = metadata; + if let Err(_err) = store + .clone() + .copy_object( + &bucket, + &object, + &bucket, + &object, + &mut obj_info, + &ObjectOptions { + version_id: obj_info_.version_id.map(|e| e.to_string()), + ..Default::default() + }, + &ObjectOptions { + version_id: obj_info_.version_id.map(|e| e.to_string()), + mod_time: obj_info_.mod_time, + ..Default::default() + }, + ) + .await + { + return Err(S3Error::with_message( + S3ErrorCode::Custom("ErrInvalidObjectState".into()), + "post restore object failed", + )); + } + if already_restored { + let output = RestoreObjectOutput { + request_charged: Some(RequestCharged::from_static(RequestCharged::REQUESTER)), + restore_output_path: None, + }; + return Ok(S3Response::new(output)); + } + } + + let restore_object = Uuid::new_v4().to_string(); + //if let Some(rreq) = rreq { + if let Some(output_location) = &rreq.output_location + && let Some(s3) = &output_location.s3 + && !s3.bucket_name.is_empty() + { + header.insert( + X_AMZ_RESTORE_OUTPUT_PATH, + format!("{}{}{}", s3.bucket_name, s3.prefix, restore_object).parse().unwrap(), + ); + } + //} + /*send_event(EventArgs { + event_name: event::ObjectRestorePost, + bucket_name: bucket, + object: obj_info, + req_params: extract_req_params(r), + user_agent: req.user_agent(), + host: handlers::get_source_ip(r), + });*/ + tokio::spawn(async move { + /*if rreq.select_parameters.is_some() { + let actual_size = obj_info_.get_actual_size(); + if actual_size.is_err() { + return Err(S3Error::with_message(S3ErrorCode::Custom("ErrInvalidObjectState".into()), "post restore object failed")); + } + + let object_rsc = s3select.new_object_read_seek_closer( + |offset: i64| -> (ReadCloser, error) { + rs := &HTTPRangeSpec{ + IsSuffixLength: false, + Start: offset, + End: -1, + } + return get_transitioned_object_reader(bucket, object, rs, r.Header, + obj_info, ObjectOptions {version_id: obj_info_.version_id}); + }, + actual_size.unwrap(), + ); + if err = rreq.select_parameters.open(object_rsc); err != nil { + if serr, ok := err.(s3select.SelectError); ok { + let encoded_error_response = encodeResponse(APIErrorResponse { + code: serr.ErrorCode(), + message: serr.ErrorMessage(), + bucket_name: bucket, + key: object, + resource: r.URL.Path, + request_id: w.Header().Get(xhttp.AmzRequestID), + host_id: globalDeploymentID(), + }); + //writeResponse(w, serr.HTTPStatusCode(), encodedErrorResponse, mimeXML) + Ok(S3Response::with_headers((StatusCode::OK, Body::empty()), header)); + } else { + return Err(S3Error::with_message(S3ErrorCode::Custom("ErrInvalidObjectState".into()), "post restore object failed")); + } + return Ok(()); + } + let nr = httptest.NewRecorder(); + let rw = xhttp.NewResponseRecorder(nr); + rw.log_err_body = true; + rw.log_all_body = true; + rreq.select_parameters.evaluate(rw); + rreq.select_parameters.Close(); + return Ok(S3Response::with_headers((StatusCode::OK, Body::empty()), header)); + }*/ + let opts = ObjectOptions { + transition: TransitionOptions { + restore_request: rreq, + restore_expiry, + ..Default::default() + }, + version_id: obj_info_.version_id.map(|e| e.to_string()), + ..Default::default() + }; + if let Err(err) = store.clone().restore_transitioned_object(&bucket, &object, &opts).await { + warn!("unable to restore transitioned bucket/object {}/{}: {}", bucket, object, err.to_string()); + return Err(S3Error::with_message( + S3ErrorCode::Custom("ErrRestoreTransitionedObject".into()), + format!("unable to restore transitioned bucket/object {bucket}/{object}: {err}"), + )); + } + + /*send_event(EventArgs { + EventName: event.ObjectRestoreCompleted, + BucketName: bucket, + Object: objInfo, + ReqParams: extractReqParams(r), + UserAgent: r.UserAgent(), + Host: handlers.GetSourceIP(r), + });*/ + Ok(()) + }); + + let output = RestoreObjectOutput { + request_charged: Some(RequestCharged::from_static(RequestCharged::REQUESTER)), + restore_output_path: None, + }; + + Ok(S3Response::with_headers(output, header)) + } + + async fn select_object_content( + &self, + req: S3Request, + ) -> S3Result> { + info!("handle select_object_content"); + + let input = Arc::new(req.input); + info!("{:?}", input); + + let db = get_global_db((*input).clone(), false).await.map_err(|e| { + error!("get global db failed, {}", e.to_string()); + s3_error!(InternalError, "{}", e.to_string()) + })?; + let query = Query::new(Context { input: input.clone() }, input.request.expression.clone()); + let result = db + .execute(&query) + .await + .map_err(|e| s3_error!(InternalError, "{}", e.to_string()))?; + + let results = result.result().chunk_result().await.unwrap().to_vec(); + + let mut buffer = Vec::new(); + if input.request.output_serialization.csv.is_some() { + let mut csv_writer = CsvWriterBuilder::new().with_header(false).build(&mut buffer); + for batch in results { + csv_writer + .write(&batch) + .map_err(|e| s3_error!(InternalError, "can't encode output to csv. e: {}", e.to_string()))?; + } + } else if input.request.output_serialization.json.is_some() { + let mut json_writer = JsonWriterBuilder::new() + .with_explicit_nulls(true) + .build::<_, JsonArray>(&mut buffer); + for batch in results { + json_writer + .write(&batch) + .map_err(|e| s3_error!(InternalError, "can't encode output to json. e: {}", e.to_string()))?; + } + json_writer + .finish() + .map_err(|e| s3_error!(InternalError, "writer output into json error, e: {}", e.to_string()))?; + } else { + return Err(s3_error!( + InvalidArgument, + "Unsupported output format. Supported formats are CSV and JSON" + )); + } + + let (tx, rx) = mpsc::channel::>(2); + let stream = ReceiverStream::new(rx); + tokio::spawn(async move { + let _ = tx + .send(Ok(SelectObjectContentEvent::Cont(ContinuationEvent::default()))) + .await; + let _ = tx + .send(Ok(SelectObjectContentEvent::Records(RecordsEvent { + payload: Some(Bytes::from(buffer)), + }))) + .await; + let _ = tx.send(Ok(SelectObjectContentEvent::End(EndEvent::default()))).await; + + drop(tx); + }); + + Ok(S3Response::new(SelectObjectContentOutput { + payload: Some(SelectObjectContentEventStream::new(stream)), + })) + } #[instrument(level = "debug", skip(self, req))] async fn upload_part(&self, req: S3Request) -> S3Result> { @@ -4850,3187 +5997,4 @@ impl S3 for FS { Ok(S3Response::new(output)) } - - #[instrument(level = "debug", skip(self, req))] - async fn list_parts(&self, req: S3Request) -> S3Result> { - let ListPartsInput { - bucket, - key, - upload_id, - part_number_marker, - max_parts, - .. - } = req.input; - - let Some(store) = new_object_layer_fn() else { - return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); - }; - - let part_number_marker = part_number_marker.map(|x| x as usize); - let max_parts = match max_parts { - Some(parts) => { - if !(1..=1000).contains(&parts) { - return Err(s3_error!(InvalidArgument, "max-parts must be between 1 and 1000")); - } - parts as usize - } - None => 1000, - }; - - let res = store - .list_object_parts(&bucket, &key, &upload_id, part_number_marker, max_parts, &ObjectOptions::default()) - .await - .map_err(ApiError::from)?; - - let output = ListPartsOutput { - bucket: Some(res.bucket), - key: Some(res.object), - upload_id: Some(res.upload_id), - parts: Some( - res.parts - .into_iter() - .map(|p| Part { - e_tag: p.etag.map(|etag| to_s3s_etag(&etag)), - last_modified: p.last_mod.map(Timestamp::from), - part_number: Some(p.part_num as i32), - size: Some(p.size as i64), - ..Default::default() - }) - .collect(), - ), - owner: Some(RUSTFS_OWNER.to_owned()), - initiator: Some(Initiator { - id: RUSTFS_OWNER.id.clone(), - display_name: RUSTFS_OWNER.display_name.clone(), - }), - is_truncated: Some(res.is_truncated), - next_part_number_marker: res.next_part_number_marker.try_into().ok(), - max_parts: res.max_parts.try_into().ok(), - part_number_marker: res.part_number_marker.try_into().ok(), - storage_class: if res.storage_class.is_empty() { - None - } else { - Some(res.storage_class.into()) - }, - ..Default::default() - }; - Ok(S3Response::new(output)) - } - - async fn list_multipart_uploads( - &self, - req: S3Request, - ) -> S3Result> { - let ListMultipartUploadsInput { - bucket, - prefix, - delimiter, - key_marker, - upload_id_marker, - max_uploads, - .. - } = req.input; - - let Some(store) = new_object_layer_fn() else { - return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); - }; - - let prefix = prefix.unwrap_or_default(); - - let max_uploads = max_uploads.map(|x| x as usize).unwrap_or(MAX_PARTS_COUNT); - - if let Some(key_marker) = &key_marker - && !key_marker.starts_with(prefix.as_str()) - { - return Err(s3_error!(NotImplemented, "Invalid key marker")); - } - - let result = store - .list_multipart_uploads(&bucket, &prefix, delimiter, key_marker, upload_id_marker, max_uploads) - .await - .map_err(ApiError::from)?; - - let output = ListMultipartUploadsOutput { - bucket: Some(bucket), - prefix: Some(prefix), - delimiter: result.delimiter, - key_marker: result.key_marker, - upload_id_marker: result.upload_id_marker, - max_uploads: Some(result.max_uploads as i32), - is_truncated: Some(result.is_truncated), - uploads: Some( - result - .uploads - .into_iter() - .map(|u| MultipartUpload { - key: Some(u.object), - upload_id: Some(u.upload_id), - initiated: u.initiated.map(Timestamp::from), - - ..Default::default() - }) - .collect(), - ), - common_prefixes: Some( - result - .common_prefixes - .into_iter() - .map(|c| CommonPrefix { prefix: Some(c) }) - .collect(), - ), - ..Default::default() - }; - - Ok(S3Response::new(output)) - } - - #[instrument(level = "debug", skip(self, req))] - async fn complete_multipart_upload( - &self, - req: S3Request, - ) -> S3Result> { - let helper = OperationHelper::new(&req, EventName::ObjectCreatedCompleteMultipartUpload, "s3:CompleteMultipartUpload"); - let input = req.input; - let CompleteMultipartUploadInput { - multipart_upload, - bucket, - key, - upload_id, - if_match, - if_none_match, - .. - } = input; - - if if_match.is_some() || if_none_match.is_some() { - let Some(store) = new_object_layer_fn() else { - return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); - }; - - match store.get_object_info(&bucket, &key, &ObjectOptions::default()).await { - Ok(info) => { - if !info.delete_marker { - if let Some(ifmatch) = if_match - && let Some(strong_etag) = ifmatch.into_etag() - && info - .etag - .as_ref() - .is_some_and(|etag| ETag::Strong(etag.clone()) != strong_etag) - { - return Err(s3_error!(PreconditionFailed)); - } - if let Some(ifnonematch) = if_none_match - && let Some(strong_etag) = ifnonematch.into_etag() - && info - .etag - .as_ref() - .is_some_and(|etag| ETag::Strong(etag.clone()) == strong_etag) - { - return Err(s3_error!(PreconditionFailed)); - } - } - } - Err(err) => { - if !is_err_object_not_found(&err) && !is_err_version_not_found(&err) { - return Err(ApiError::from(err).into()); - } - - if if_match.is_some() && (is_err_object_not_found(&err) || is_err_version_not_found(&err)) { - return Err(ApiError::from(err).into()); - } - } - } - } - - let Some(multipart_upload) = multipart_upload else { return Err(s3_error!(InvalidPart)) }; - - let opts = &get_complete_multipart_upload_opts(&req.headers).map_err(ApiError::from)?; - - let uploaded_parts_vec = multipart_upload - .parts - .unwrap_or_default() - .into_iter() - .map(CompletePart::from) - .collect::>(); - - // is part number sorted? - if !uploaded_parts_vec.is_sorted_by_key(|p| p.part_num) { - return Err(s3_error!(InvalidPart, "Part numbers must be sorted")); - } - - // Handle duplicate part numbers: according to S3 specification, when the same part number - // is uploaded multiple times, the last uploaded part (in the list order) should be used. - // This can happen in concurrent upload scenarios where a part is re-uploaded before completion. - // We deduplicate by keeping the last occurrence of each part number using a HashMap. - use std::collections::HashMap; - let mut part_map: HashMap = HashMap::new(); - for part in uploaded_parts_vec { - part_map.insert(part.part_num, part); - } - - // Reconstruct the parts list in sorted order, keeping only the last occurrence of each part number - let mut uploaded_parts: Vec = part_map.into_values().collect(); - uploaded_parts.sort_by_key(|p| p.part_num); - - // TODO: check object lock - - let Some(store) = new_object_layer_fn() else { - return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); - }; - - // TDD: Get multipart info to extract encryption configuration before completing - info!( - "TDD: Attempting to get multipart info for bucket={}, key={}, upload_id={}", - bucket, key, upload_id - ); - - let multipart_info = store - .get_multipart_info(&bucket, &key, &upload_id, &ObjectOptions::default()) - .await - .map_err(ApiError::from)?; - - info!("TDD: Got multipart info successfully"); - info!("TDD: Multipart info metadata: {:?}", multipart_info.user_defined); - - // TDD: Extract encryption information from multipart upload metadata - let server_side_encryption = multipart_info - .user_defined - .get("x-amz-server-side-encryption") - .map(|s| ServerSideEncryption::from(s.clone())); - info!( - "TDD: Raw encryption from metadata: {:?} -> parsed: {:?}", - multipart_info.user_defined.get("x-amz-server-side-encryption"), - server_side_encryption - ); - - let ssekms_key_id = multipart_info - .user_defined - .get("x-amz-server-side-encryption-aws-kms-key-id") - .cloned(); - - info!( - "TDD: Extracted encryption info - SSE: {:?}, KMS Key: {:?}", - server_side_encryption, ssekms_key_id - ); - - let obj_info = store - .clone() - .complete_multipart_upload(&bucket, &key, &upload_id, uploaded_parts, opts) - .await - .map_err(ApiError::from)?; - - // check quota after completing multipart upload - if let Some(metadata_sys) = rustfs_ecstore::bucket::metadata_sys::GLOBAL_BucketMetadataSys.get() { - let quota_checker = QuotaChecker::new(metadata_sys.clone()); - - match quota_checker - .check_quota(&bucket, QuotaOperation::PutObject, obj_info.size as u64) - .await - { - Ok(check_result) => { - if !check_result.allowed { - // Quota exceeded, delete the completed object - let _ = store.delete_object(&bucket, &key, ObjectOptions::default()).await; - return Err(S3Error::with_message( - S3ErrorCode::InvalidRequest, - format!( - "Bucket quota exceeded. Current usage: {} bytes, limit: {} bytes", - check_result.current_usage, - check_result.quota_limit.unwrap_or(0) - ), - )); - } - // Update quota tracking after successful multipart upload - if rustfs_ecstore::bucket::metadata_sys::GLOBAL_BucketMetadataSys.get().is_some() { - rustfs_ecstore::data_usage::increment_bucket_usage_memory(&bucket, obj_info.size as u64).await; - } - } - Err(e) => { - warn!("Quota check failed for bucket {}: {}, allowing operation", bucket, e); - } - } - } - - // Invalidate cache for the completed multipart object - let manager = get_concurrency_manager(); - let mpu_bucket = bucket.clone(); - let mpu_key = key.clone(); - let mpu_version = obj_info.version_id.map(|v| v.to_string()); - let mpu_version_clone = mpu_version.clone(); - tokio::spawn(async move { - manager - .invalidate_cache_versioned(&mpu_bucket, &mpu_key, mpu_version_clone.as_deref()) - .await; - }); - - info!( - "TDD: Creating output with SSE: {:?}, KMS Key: {:?}", - server_side_encryption, ssekms_key_id - ); - - let mut checksum_crc32 = input.checksum_crc32; - let mut checksum_crc32c = input.checksum_crc32c; - let mut checksum_sha1 = input.checksum_sha1; - let mut checksum_sha256 = input.checksum_sha256; - let mut checksum_crc64nvme = input.checksum_crc64nvme; - let mut checksum_type = input.checksum_type; - - // checksum - let (checksums, _is_multipart) = obj_info - .decrypt_checksums(opts.part_number.unwrap_or(0), &req.headers) - .map_err(ApiError::from)?; - - for (key, checksum) in checksums { - if key == AMZ_CHECKSUM_TYPE { - checksum_type = Some(ChecksumType::from(checksum)); - continue; - } - - match rustfs_rio::ChecksumType::from_string(key.as_str()) { - rustfs_rio::ChecksumType::CRC32 => checksum_crc32 = Some(checksum), - rustfs_rio::ChecksumType::CRC32C => checksum_crc32c = Some(checksum), - rustfs_rio::ChecksumType::SHA1 => checksum_sha1 = Some(checksum), - rustfs_rio::ChecksumType::SHA256 => checksum_sha256 = Some(checksum), - rustfs_rio::ChecksumType::CRC64_NVME => checksum_crc64nvme = Some(checksum), - _ => (), - } - } - - let output = CompleteMultipartUploadOutput { - bucket: Some(bucket.clone()), - key: Some(key.clone()), - e_tag: obj_info.etag.clone().map(|etag| to_s3s_etag(&etag)), - location: Some("us-east-1".to_string()), - server_side_encryption: server_side_encryption.clone(), // TDD: Return encryption info - ssekms_key_id: ssekms_key_id.clone(), // TDD: Return KMS key ID if present - checksum_crc32: checksum_crc32.clone(), - checksum_crc32c: checksum_crc32c.clone(), - checksum_sha1: checksum_sha1.clone(), - checksum_sha256: checksum_sha256.clone(), - checksum_crc64nvme: checksum_crc64nvme.clone(), - checksum_type: checksum_type.clone(), - version_id: mpu_version, - ..Default::default() - }; - info!( - "TDD: Created output: SSE={:?}, KMS={:?}", - output.server_side_encryption, output.ssekms_key_id - ); - - let helper_output = entity::CompleteMultipartUploadOutput { - bucket: Some(bucket.clone()), - key: Some(key.clone()), - e_tag: obj_info.etag.clone().map(|etag| to_s3s_etag(&etag)), - location: Some("us-east-1".to_string()), - server_side_encryption, // TDD: Return encryption info - ssekms_key_id, // TDD: Return KMS key ID if present - checksum_crc32, - checksum_crc32c, - checksum_sha1, - checksum_sha256, - checksum_crc64nvme, - checksum_type, - ..Default::default() - }; - - let mt2 = HashMap::new(); - let repoptions = - get_must_replicate_options(&mt2, "".to_string(), ReplicationStatusType::Empty, ReplicationType::Object, opts.clone()); - - let dsc = must_replicate(&bucket, &key, repoptions).await; - - if dsc.replicate_any() { - warn!("need multipart replication"); - schedule_replication(obj_info, store, dsc, ReplicationType::Object).await; - } - info!( - "TDD: About to return S3Response with output: SSE={:?}, KMS={:?}", - output.server_side_encryption, output.ssekms_key_id - ); - let helper_result = Ok(S3Response::new(helper_output)); - let _ = helper.complete(&helper_result); - Ok(S3Response::new(output)) - } - - #[instrument(level = "debug", skip(self))] - async fn abort_multipart_upload( - &self, - req: S3Request, - ) -> S3Result> { - let AbortMultipartUploadInput { - bucket, key, upload_id, .. - } = req.input; - - let Some(store) = new_object_layer_fn() else { - return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); - }; - - let opts = &ObjectOptions::default(); - - // Special handling for abort_multipart_upload: Per AWS S3 API specification, this operation - // should return NoSuchUpload (404) when the upload_id doesn't exist, even if the format - // appears invalid. This differs from other multipart operations (upload_part, list_parts, - // complete_multipart_upload) which return InvalidArgument for malformed upload_ids. - // The lenient validation matches AWS S3 behavior where format validation is relaxed for - // abort operations to avoid leaking information about upload_id format requirements. - match store - .abort_multipart_upload(bucket.as_str(), key.as_str(), upload_id.as_str(), opts) - .await - { - Ok(_) => Ok(S3Response::new(AbortMultipartUploadOutput { ..Default::default() })), - Err(err) => { - // Convert MalformedUploadID to NoSuchUpload for S3 API compatibility - if matches!(err, StorageError::MalformedUploadID(_)) { - return Err(S3Error::new(S3ErrorCode::NoSuchUpload)); - } - Err(ApiError::from(err).into()) - } - } - } - - #[instrument(level = "debug", skip(self))] - async fn get_bucket_tagging(&self, req: S3Request) -> S3Result> { - let bucket = req.input.bucket.clone(); - // check bucket exists. - let _bucket = self - .head_bucket(req.map_input(|input| HeadBucketInput { - bucket: input.bucket, - expected_bucket_owner: None, - })) - .await?; - - let Tagging { tag_set } = match metadata_sys::get_tagging_config(&bucket).await { - Ok((tags, _)) => tags, - Err(err) => { - if err == StorageError::ConfigNotFound { - return Err(S3Error::with_message(S3ErrorCode::NoSuchTagSet, "The TagSet does not exist".to_string())); - } - warn!("get_tagging_config err {:?}", &err); - return Err(ApiError::from(err).into()); - } - }; - - Ok(S3Response::new(GetBucketTaggingOutput { tag_set })) - } - - #[instrument(level = "debug", skip(self))] - async fn put_bucket_tagging(&self, req: S3Request) -> S3Result> { - let PutBucketTaggingInput { bucket, tagging, .. } = req.input; - - let Some(store) = new_object_layer_fn() else { - return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); - }; - - store - .get_bucket_info(&bucket, &BucketOptions::default()) - .await - .map_err(ApiError::from)?; - - let data = try_!(serialize(&tagging)); - - metadata_sys::update(&bucket, BUCKET_TAGGING_CONFIG, data) - .await - .map_err(ApiError::from)?; - - Ok(S3Response::new(Default::default())) - } - - #[instrument(level = "debug", skip(self))] - async fn delete_bucket_tagging( - &self, - req: S3Request, - ) -> S3Result> { - let DeleteBucketTaggingInput { bucket, .. } = req.input; - - metadata_sys::delete(&bucket, BUCKET_TAGGING_CONFIG) - .await - .map_err(ApiError::from)?; - - Ok(S3Response::new(DeleteBucketTaggingOutput {})) - } - - #[instrument(level = "debug", skip(self))] - async fn get_bucket_cors(&self, req: S3Request) -> S3Result> { - let bucket = req.input.bucket.clone(); - // check bucket exists. - let _bucket = self - .head_bucket(req.map_input(|input| HeadBucketInput { - bucket: input.bucket, - expected_bucket_owner: None, - })) - .await?; - - let cors_configuration = match metadata_sys::get_cors_config(&bucket).await { - Ok((config, _)) => config, - Err(err) => { - if err == StorageError::ConfigNotFound { - return Err(S3Error::with_message( - S3ErrorCode::NoSuchCORSConfiguration, - "The CORS configuration does not exist".to_string(), - )); - } - warn!("get_cors_config err {:?}", &err); - return Err(ApiError::from(err).into()); - } - }; - - Ok(S3Response::new(GetBucketCorsOutput { - cors_rules: Some(cors_configuration.cors_rules), - })) - } - - #[instrument(level = "debug", skip(self))] - async fn put_bucket_cors(&self, req: S3Request) -> S3Result> { - let PutBucketCorsInput { - bucket, - cors_configuration, - .. - } = req.input; - - let Some(store) = new_object_layer_fn() else { - return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); - }; - - store - .get_bucket_info(&bucket, &BucketOptions::default()) - .await - .map_err(ApiError::from)?; - - let data = try_!(serialize(&cors_configuration)); - - metadata_sys::update(&bucket, BUCKET_CORS_CONFIG, data) - .await - .map_err(ApiError::from)?; - - Ok(S3Response::new(PutBucketCorsOutput::default())) - } - - #[instrument(level = "debug", skip(self))] - async fn delete_bucket_cors(&self, req: S3Request) -> S3Result> { - let DeleteBucketCorsInput { bucket, .. } = req.input; - - let Some(store) = new_object_layer_fn() else { - return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); - }; - - store - .get_bucket_info(&bucket, &BucketOptions::default()) - .await - .map_err(ApiError::from)?; - - metadata_sys::delete(&bucket, BUCKET_CORS_CONFIG) - .await - .map_err(ApiError::from)?; - - Ok(S3Response::new(DeleteBucketCorsOutput {})) - } - - #[instrument(level = "debug", skip(self, req))] - async fn put_object_tagging(&self, req: S3Request) -> S3Result> { - let start_time = std::time::Instant::now(); - let mut helper = OperationHelper::new(&req, EventName::ObjectCreatedPutTagging, "s3:PutObjectTagging"); - let PutObjectTaggingInput { - bucket, - key: object, - tagging, - .. - } = req.input.clone(); - - if tagging.tag_set.len() > 10 { - // TOTO: Note that Amazon S3 limits the maximum number of tags to 10 tags per object. - // Reference: https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-tagging.html - // Reference: https://docs.aws.amazon.com/zh_cn/AmazonS3/latest/API/API_PutObjectTagging.html - // https://github.com/minio/mint/blob/master/run/core/aws-sdk-go-v2/main.go#L1647 - error!("Tag set exceeds maximum of 10 tags: {}", tagging.tag_set.len()); - return Err(s3_error!(InvalidTag, "Cannot have more than 10 tags per object")); - } - - let Some(store) = new_object_layer_fn() else { - return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); - }; - - let mut tag_keys = std::collections::HashSet::with_capacity(tagging.tag_set.len()); - for tag in &tagging.tag_set { - let key = tag.key.as_ref().filter(|k| !k.is_empty()).ok_or_else(|| { - error!("Empty tag key"); - s3_error!(InvalidTag, "Tag key cannot be empty") - })?; - - if key.len() > 128 { - error!("Tag key too long: {} bytes", key.len()); - return Err(s3_error!(InvalidTag, "Tag key is too long, maximum allowed length is 128 characters")); - } - - let value = tag.value.as_ref().ok_or_else(|| { - error!("Null tag value"); - s3_error!(InvalidTag, "Tag value cannot be null") - })?; - - if value.is_empty() { - error!("Empty tag value"); - return Err(s3_error!(InvalidTag, "Tag value cannot be empty")); - } - - if value.len() > 256 { - error!("Tag value too long: {} bytes", value.len()); - return Err(s3_error!(InvalidTag, "Tag value is too long, maximum allowed length is 256 characters")); - } - - if !tag_keys.insert(key) { - error!("Duplicate tag key: {}", key); - return Err(s3_error!(InvalidTag, "Cannot provide multiple Tags with the same key")); - } - } - - let tags = encode_tags(tagging.tag_set); - debug!("Encoded tags: {}", tags); - - // TODO: getOpts, Replicate - // Support versioned objects - let version_id = req.input.version_id.clone(); - let opts = ObjectOptions { - version_id: self.parse_version_id(version_id)?.map(Into::into), - ..Default::default() - }; - - store.put_object_tags(&bucket, &object, &tags, &opts).await.map_err(|e| { - error!("Failed to put object tags: {}", e); - counter!("rustfs.put_object_tagging.failure").increment(1); - ApiError::from(e) - })?; - - // Invalidate cache for the tagged object - let manager = get_concurrency_manager(); - let version_id = req.input.version_id.clone(); - let cache_key = ConcurrencyManager::make_cache_key(&bucket, &object, version_id.clone().as_deref()); - tokio::spawn(async move { - manager - .invalidate_cache_versioned(&bucket, &object, version_id.as_deref()) - .await; - debug!("Cache invalidated for tagged object: {}", cache_key); - }); - - // Add metrics - counter!("rustfs.put_object_tagging.success").increment(1); - - let version_id_resp = req.input.version_id.clone().unwrap_or_default(); - helper = helper.version_id(version_id_resp); - - let result = Ok(S3Response::new(PutObjectTaggingOutput { - version_id: req.input.version_id.clone(), - })); - let _ = helper.complete(&result); - let duration = start_time.elapsed(); - histogram!("rustfs.object_tagging.operation.duration.seconds", "operation" => "put").record(duration.as_secs_f64()); - result - } - - #[instrument(level = "debug", skip(self))] - async fn get_object_tagging(&self, req: S3Request) -> S3Result> { - let start_time = std::time::Instant::now(); - let GetObjectTaggingInput { bucket, key: object, .. } = req.input; - - info!("Starting get_object_tagging for bucket: {}, object: {}", bucket, object); - - let Some(store) = new_object_layer_fn() else { - error!("Store not initialized"); - return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); - }; - - // Support versioned objects - let version_id = req.input.version_id.clone(); - let opts = ObjectOptions { - version_id: self.parse_version_id(version_id)?.map(Into::into), - ..Default::default() - }; - - let tags = store.get_object_tags(&bucket, &object, &opts).await.map_err(|e| { - if is_err_object_not_found(&e) { - error!("Object not found: {}", e); - return s3_error!(NoSuchKey); - } - error!("Failed to get object tags: {}", e); - ApiError::from(e).into() - })?; - - let tag_set = decode_tags(tags.as_str()); - debug!("Decoded tag set: {:?}", tag_set); - - // Add metrics - counter!("rustfs.get_object_tagging.success").increment(1); - let duration = start_time.elapsed(); - histogram!("rustfs.object_tagging.operation.duration.seconds", "operation" => "put").record(duration.as_secs_f64()); - Ok(S3Response::new(GetObjectTaggingOutput { - tag_set, - version_id: req.input.version_id.clone(), - })) - } - - #[instrument(level = "debug", skip(self))] - async fn delete_object_tagging( - &self, - req: S3Request, - ) -> S3Result> { - let start_time = std::time::Instant::now(); - let mut helper = OperationHelper::new(&req, EventName::ObjectCreatedDeleteTagging, "s3:DeleteObjectTagging"); - let DeleteObjectTaggingInput { - bucket, - key: object, - version_id, - .. - } = req.input.clone(); - - let Some(store) = new_object_layer_fn() else { - error!("Store not initialized"); - return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); - }; - - // Support versioned objects - let version_id_for_parse = version_id.clone(); - let opts = ObjectOptions { - version_id: self.parse_version_id(version_id_for_parse)?.map(Into::into), - ..Default::default() - }; - - // TODO: Replicate (keep the original TODO, if further replication logic is needed) - store.delete_object_tags(&bucket, &object, &opts).await.map_err(|e| { - error!("Failed to delete object tags: {}", e); - ApiError::from(e) - })?; - - // Invalidate cache for the deleted tagged object - let manager = get_concurrency_manager(); - let version_id_clone = version_id.clone(); - tokio::spawn(async move { - manager - .invalidate_cache_versioned(&bucket, &object, version_id_clone.as_deref()) - .await; - debug!( - "Cache invalidated for deleted tagged object: bucket={}, object={}, version_id={:?}", - bucket, object, version_id_clone - ); - }); - - // Add metrics - counter!("rustfs.delete_object_tagging.success").increment(1); - - let version_id_resp = version_id.clone().unwrap_or_default(); - helper = helper.version_id(version_id_resp); - - let result = Ok(S3Response::new(DeleteObjectTaggingOutput { version_id })); - let _ = helper.complete(&result); - let duration = start_time.elapsed(); - histogram!("rustfs.object_tagging.operation.duration.seconds", "operation" => "delete").record(duration.as_secs_f64()); - result - } - - #[instrument(level = "debug", skip(self))] - async fn get_bucket_versioning( - &self, - req: S3Request, - ) -> S3Result> { - let GetBucketVersioningInput { bucket, .. } = req.input; - let Some(store) = new_object_layer_fn() else { - return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); - }; - - store - .get_bucket_info(&bucket, &BucketOptions::default()) - .await - .map_err(ApiError::from)?; - - let VersioningConfiguration { status, .. } = BucketVersioningSys::get(&bucket).await.map_err(ApiError::from)?; - - Ok(S3Response::new(GetBucketVersioningOutput { - status, - ..Default::default() - })) - } - - #[instrument(level = "debug", skip(self))] - async fn put_bucket_versioning( - &self, - req: S3Request, - ) -> S3Result> { - let PutBucketVersioningInput { - bucket, - versioning_configuration, - .. - } = req.input; - - // TODO: check other sys - // check site replication enable - // check bucket object lock enable - // check replication suspended - - let data = try_!(serialize(&versioning_configuration)); - - metadata_sys::update(&bucket, BUCKET_VERSIONING_CONFIG, data) - .await - .map_err(ApiError::from)?; - - // TODO: globalSiteReplicationSys.BucketMetaHook - - Ok(S3Response::new(PutBucketVersioningOutput {})) - } - - async fn get_bucket_policy_status( - &self, - req: S3Request, - ) -> S3Result> { - let GetBucketPolicyStatusInput { bucket, .. } = req.input; - - let Some(store) = new_object_layer_fn() else { - return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); - }; - - store - .get_bucket_info(&bucket, &BucketOptions::default()) - .await - .map_err(ApiError::from)?; - - let remote_addr = req.extensions.get::>().and_then(|opt| opt.map(|a| a.0)); - let conditions = get_condition_values(&req.headers, &rustfs_credentials::Credentials::default(), None, None, remote_addr); - - let read_only = PolicySys::is_allowed(&BucketPolicyArgs { - bucket: &bucket, - action: Action::S3Action(S3Action::ListBucketAction), - is_owner: false, - account: "", - groups: &None, - conditions: &conditions, - object: "", - }) - .await; - - let write_only = PolicySys::is_allowed(&BucketPolicyArgs { - bucket: &bucket, - action: Action::S3Action(S3Action::PutObjectAction), - is_owner: false, - account: "", - groups: &None, - conditions: &conditions, - object: "", - }) - .await; - - let is_public = read_only && write_only; - - let output = GetBucketPolicyStatusOutput { - policy_status: Some(PolicyStatus { - is_public: Some(is_public), - }), - }; - - Ok(S3Response::new(output)) - } - - async fn get_bucket_policy(&self, req: S3Request) -> S3Result> { - let GetBucketPolicyInput { bucket, .. } = req.input; - - let Some(store) = new_object_layer_fn() else { - return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); - }; - - store - .get_bucket_info(&bucket, &BucketOptions::default()) - .await - .map_err(ApiError::from)?; - - // Return the raw policy JSON as originally stored to preserve exact format. - // This ensures GET returns the same document that was PUT, matching S3 behavior. - let (policy_str, _) = match metadata_sys::get_bucket_policy_raw(&bucket).await { - Ok(res) => res, - Err(err) => { - if StorageError::ConfigNotFound == err { - return Err(s3_error!(NoSuchBucketPolicy)); - } - return Err(S3Error::with_message(S3ErrorCode::InternalError, err.to_string())); - } - }; - - Ok(S3Response::new(GetBucketPolicyOutput { - policy: Some(policy_str), - })) - } - - async fn put_bucket_policy(&self, req: S3Request) -> S3Result> { - let PutBucketPolicyInput { bucket, policy, .. } = req.input; - - let Some(store) = new_object_layer_fn() else { - return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); - }; - - store - .get_bucket_info(&bucket, &BucketOptions::default()) - .await - .map_err(ApiError::from)?; - - // warn!("input policy {}", &policy); - - let cfg: BucketPolicy = - serde_json::from_str(&policy).map_err(|e| s3_error!(InvalidArgument, "parse policy failed {:?}", e))?; - - if let Err(err) = cfg.is_valid() { - warn!("put_bucket_policy err input {:?}, {:?}", &policy, err); - return Err(s3_error!(InvalidPolicyDocument)); - } - - // Store the original policy string as-is to preserve the exact JSON format. - // This ensures GET returns the same document that was PUT. - let data = policy.into_bytes(); - - metadata_sys::update(&bucket, BUCKET_POLICY_CONFIG, data) - .await - .map_err(ApiError::from)?; - - Ok(S3Response::new(PutBucketPolicyOutput {})) - } - - async fn delete_bucket_policy( - &self, - req: S3Request, - ) -> S3Result> { - let DeleteBucketPolicyInput { bucket, .. } = req.input; - - let Some(store) = new_object_layer_fn() else { - return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); - }; - - store - .get_bucket_info(&bucket, &BucketOptions::default()) - .await - .map_err(ApiError::from)?; - - metadata_sys::delete(&bucket, BUCKET_POLICY_CONFIG) - .await - .map_err(ApiError::from)?; - - Ok(S3Response::new(DeleteBucketPolicyOutput {})) - } - - #[instrument(level = "debug", skip(self))] - async fn get_bucket_lifecycle_configuration( - &self, - req: S3Request, - ) -> S3Result> { - let GetBucketLifecycleConfigurationInput { bucket, .. } = req.input; - - let Some(store) = new_object_layer_fn() else { - return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); - }; - - store - .get_bucket_info(&bucket, &BucketOptions::default()) - .await - .map_err(ApiError::from)?; - - let rules = match metadata_sys::get_lifecycle_config(&bucket).await { - Ok((cfg, _)) => cfg.rules, - Err(_err) => { - // Return NoSuchLifecycleConfiguration error as expected by S3 clients - // This fixes issue #990 where Ansible S3 roles fail with KeyError: 'Rules' - return Err(s3_error!(NoSuchLifecycleConfiguration)); - } - }; - - Ok(S3Response::new(GetBucketLifecycleConfigurationOutput { - rules: Some(rules), - ..Default::default() - })) - } - - #[instrument(level = "debug", skip(self))] - async fn put_bucket_lifecycle_configuration( - &self, - req: S3Request, - ) -> S3Result> { - let PutBucketLifecycleConfigurationInput { - bucket, - lifecycle_configuration, - .. - } = req.input; - - let Some(input_cfg) = lifecycle_configuration else { return Err(s3_error!(InvalidArgument)) }; - - let rcfg = metadata_sys::get_object_lock_config(&bucket).await; - if let Ok(rcfg) = rcfg - && let Err(err) = input_cfg.validate(&rcfg.0).await - { - //return Err(S3Error::with_message(S3ErrorCode::Custom("BucketLockValidateFailed".into()), err.to_string())); - return Err(S3Error::with_message(S3ErrorCode::Custom("ValidateFailed".into()), err.to_string())); - } - - if let Err(err) = validate_transition_tier(&input_cfg).await { - //warn!("lifecycle_configuration add failed, err: {:?}", err); - return Err(S3Error::with_message(S3ErrorCode::Custom("CustomError".into()), err.to_string())); - } - - let data = try_!(serialize(&input_cfg)); - metadata_sys::update(&bucket, BUCKET_LIFECYCLE_CONFIG, data) - .await - .map_err(ApiError::from)?; - - Ok(S3Response::new(PutBucketLifecycleConfigurationOutput::default())) - } - - #[instrument(level = "debug", skip(self))] - async fn delete_bucket_lifecycle( - &self, - req: S3Request, - ) -> S3Result> { - let DeleteBucketLifecycleInput { bucket, .. } = req.input; - - let Some(store) = new_object_layer_fn() else { - return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); - }; - - store - .get_bucket_info(&bucket, &BucketOptions::default()) - .await - .map_err(ApiError::from)?; - - metadata_sys::delete(&bucket, BUCKET_LIFECYCLE_CONFIG) - .await - .map_err(ApiError::from)?; - - Ok(S3Response::new(DeleteBucketLifecycleOutput::default())) - } - - async fn get_bucket_encryption( - &self, - req: S3Request, - ) -> S3Result> { - let GetBucketEncryptionInput { bucket, .. } = req.input; - - let Some(store) = new_object_layer_fn() else { - return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); - }; - - store - .get_bucket_info(&bucket, &BucketOptions::default()) - .await - .map_err(ApiError::from)?; - - let server_side_encryption_configuration = match metadata_sys::get_sse_config(&bucket).await { - Ok((cfg, _)) => Some(cfg), - Err(err) => { - // if BucketMetadataError::BucketLifecycleNotFound.is(&err) { - // return Err(s3_error!(ErrNoSuchBucketSSEConfig)); - // } - warn!("get_sse_config err {:?}", err); - None - } - }; - - Ok(S3Response::new(GetBucketEncryptionOutput { - server_side_encryption_configuration, - })) - } - - async fn put_bucket_encryption( - &self, - req: S3Request, - ) -> S3Result> { - let PutBucketEncryptionInput { - bucket, - server_side_encryption_configuration, - .. - } = req.input; - - info!("sse_config {:?}", &server_side_encryption_configuration); - - let Some(store) = new_object_layer_fn() else { - return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); - }; - - store - .get_bucket_info(&bucket, &BucketOptions::default()) - .await - .map_err(ApiError::from)?; - - // TODO: check kms - - let data = try_!(serialize(&server_side_encryption_configuration)); - metadata_sys::update(&bucket, BUCKET_SSECONFIG, data) - .await - .map_err(ApiError::from)?; - Ok(S3Response::new(PutBucketEncryptionOutput::default())) - } - - async fn delete_bucket_encryption( - &self, - req: S3Request, - ) -> S3Result> { - let DeleteBucketEncryptionInput { bucket, .. } = req.input; - - let Some(store) = new_object_layer_fn() else { - return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); - }; - - store - .get_bucket_info(&bucket, &BucketOptions::default()) - .await - .map_err(ApiError::from)?; - metadata_sys::delete(&bucket, BUCKET_SSECONFIG) - .await - .map_err(ApiError::from)?; - - Ok(S3Response::new(DeleteBucketEncryptionOutput::default())) - } - - #[instrument(level = "debug", skip(self))] - async fn get_object_lock_configuration( - &self, - req: S3Request, - ) -> S3Result> { - let GetObjectLockConfigurationInput { bucket, .. } = req.input; - - let object_lock_configuration = match metadata_sys::get_object_lock_config(&bucket).await { - Ok((cfg, _created)) => Some(cfg), - Err(err) => { - if err == StorageError::ConfigNotFound { - return Err(S3Error::with_message( - S3ErrorCode::ObjectLockConfigurationNotFoundError, - "Object Lock configuration does not exist for this bucket".to_string(), - )); - } - warn!("get_object_lock_config err {:?}", err); - return Err(S3Error::with_message( - S3ErrorCode::InternalError, - "Failed to load Object Lock configuration".to_string(), - )); - } - }; - - // warn!("object_lock_configuration {:?}", &object_lock_configuration); - - Ok(S3Response::new(GetObjectLockConfigurationOutput { - object_lock_configuration, - })) - } - - #[instrument(level = "debug", skip(self))] - async fn put_object_lock_configuration( - &self, - req: S3Request, - ) -> S3Result> { - let PutObjectLockConfigurationInput { - bucket, - object_lock_configuration, - .. - } = req.input; - - let Some(input_cfg) = object_lock_configuration else { return Err(s3_error!(InvalidArgument)) }; - - let Some(store) = new_object_layer_fn() else { - return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); - }; - - store - .get_bucket_info(&bucket, &BucketOptions::default()) - .await - .map_err(ApiError::from)?; - - let _ = match metadata_sys::get_object_lock_config(&bucket).await { - Ok(_) => {} - Err(err) => { - if err == StorageError::ConfigNotFound { - return Err(S3Error::with_message( - S3ErrorCode::InvalidBucketState, - "Object Lock configuration cannot be enabled on existing buckets".to_string(), - )); - } - warn!("get_object_lock_config err {:?}", err); - return Err(S3Error::with_message( - S3ErrorCode::InternalError, - "Failed to get bucket ObjectLockConfiguration".to_string(), - )); - } - }; - - let data = try_!(serialize(&input_cfg)); - - metadata_sys::update(&bucket, OBJECT_LOCK_CONFIG, data) - .await - .map_err(ApiError::from)?; - - // When Object Lock is enabled, automatically enable versioning if not already enabled - // This matches AWS S3 and MinIO behavior - let versioning_config = BucketVersioningSys::get(&bucket).await.map_err(ApiError::from)?; - if !versioning_config.enabled() { - let enable_versioning_config = VersioningConfiguration { - status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)), - ..Default::default() - }; - let versioning_data = try_!(serialize(&enable_versioning_config)); - metadata_sys::update(&bucket, BUCKET_VERSIONING_CONFIG, versioning_data) - .await - .map_err(ApiError::from)?; - } - - Ok(S3Response::new(PutObjectLockConfigurationOutput::default())) - } - - async fn get_bucket_replication( - &self, - req: S3Request, - ) -> S3Result> { - let GetBucketReplicationInput { bucket, .. } = req.input; - - let Some(store) = new_object_layer_fn() else { - return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); - }; - - store - .get_bucket_info(&bucket, &BucketOptions::default()) - .await - .map_err(ApiError::from)?; - - let rcfg = match get_replication_config(&bucket).await { - Ok((cfg, _created)) => Some(cfg), - Err(err) => { - error!("get_replication_config err {:?}", err); - if err == StorageError::ConfigNotFound { - return Err(S3Error::with_message( - S3ErrorCode::ReplicationConfigurationNotFoundError, - "replication not found".to_string(), - )); - } - return Err(ApiError::from(err).into()); - } - }; - - if rcfg.is_none() { - return Err(S3Error::with_message( - S3ErrorCode::ReplicationConfigurationNotFoundError, - "replication not found".to_string(), - )); - } - - // Ok(S3Response::new(GetBucketReplicationOutput { - // replication_configuration: rcfg, - // })) - - if rcfg.is_some() { - Ok(S3Response::new(GetBucketReplicationOutput { - replication_configuration: rcfg, - })) - } else { - let rep = ReplicationConfiguration { - role: "".to_string(), - rules: vec![], - }; - Ok(S3Response::new(GetBucketReplicationOutput { - replication_configuration: Some(rep), - })) - } - } - - async fn put_bucket_replication( - &self, - req: S3Request, - ) -> S3Result> { - let PutBucketReplicationInput { - bucket, - replication_configuration, - .. - } = req.input; - warn!("put bucket replication"); - - let Some(store) = new_object_layer_fn() else { - return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); - }; - - store - .get_bucket_info(&bucket, &BucketOptions::default()) - .await - .map_err(ApiError::from)?; - - // TODO: check enable, versioning enable - let data = try_!(serialize(&replication_configuration)); - - metadata_sys::update(&bucket, BUCKET_REPLICATION_CONFIG, data) - .await - .map_err(ApiError::from)?; - - Ok(S3Response::new(PutBucketReplicationOutput::default())) - } - - async fn delete_bucket_replication( - &self, - req: S3Request, - ) -> S3Result> { - let DeleteBucketReplicationInput { bucket, .. } = req.input; - - let Some(store) = new_object_layer_fn() else { - return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); - }; - - store - .get_bucket_info(&bucket, &BucketOptions::default()) - .await - .map_err(ApiError::from)?; - metadata_sys::delete(&bucket, BUCKET_REPLICATION_CONFIG) - .await - .map_err(ApiError::from)?; - - // TODO: remove targets - error!("delete bucket"); - - Ok(S3Response::new(DeleteBucketReplicationOutput::default())) - } - - async fn get_bucket_notification_configuration( - &self, - req: S3Request, - ) -> S3Result> { - let GetBucketNotificationConfigurationInput { bucket, .. } = req.input; - - let Some(store) = new_object_layer_fn() else { - return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); - }; - - store - .get_bucket_info(&bucket, &BucketOptions::default()) - .await - .map_err(ApiError::from)?; - - let has_notification_config = metadata_sys::get_notification_config(&bucket).await.unwrap_or_else(|err| { - warn!("get_notification_config err {:?}", err); - None - }); - - // TODO: valid target list - - if let Some(NotificationConfiguration { - event_bridge_configuration, - lambda_function_configurations, - queue_configurations, - topic_configurations, - }) = has_notification_config - { - Ok(S3Response::new(GetBucketNotificationConfigurationOutput { - event_bridge_configuration, - lambda_function_configurations, - queue_configurations, - topic_configurations, - })) - } else { - Ok(S3Response::new(GetBucketNotificationConfigurationOutput::default())) - } - } - - async fn put_bucket_notification_configuration( - &self, - req: S3Request, - ) -> S3Result> { - let PutBucketNotificationConfigurationInput { - bucket, - notification_configuration, - .. - } = req.input; - - let Some(store) = new_object_layer_fn() else { - return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); - }; - - // Verify that the bucket exists - store - .get_bucket_info(&bucket, &BucketOptions::default()) - .await - .map_err(ApiError::from)?; - - // Persist the new notification configuration - let data = try_!(serialize(¬ification_configuration)); - metadata_sys::update(&bucket, BUCKET_NOTIFICATION_CONFIG, data) - .await - .map_err(ApiError::from)?; - - // Determine region (BucketInfo has no region field) -> use global region or default - let region = rustfs_ecstore::global::get_global_region().unwrap_or_else(|| req.region.clone().unwrap_or_default()); - - // Purge old rules and resolve new rules in parallel - let clear_rules = notifier_global::clear_bucket_notification_rules(&bucket); - let parse_rules = async { - let mut event_rules = Vec::new(); - - process_queue_configurations(&mut event_rules, notification_configuration.queue_configurations.clone(), |arn_str| { - ARN::parse(arn_str) - .map(|arn| arn.target_id) - .map_err(|e| TargetIDError::InvalidFormat(e.to_string())) - })?; - process_topic_configurations(&mut event_rules, notification_configuration.topic_configurations.clone(), |arn_str| { - ARN::parse(arn_str) - .map(|arn| arn.target_id) - .map_err(|e| TargetIDError::InvalidFormat(e.to_string())) - })?; - process_lambda_configurations( - &mut event_rules, - notification_configuration.lambda_function_configurations.clone(), - |arn_str| { - ARN::parse(arn_str) - .map(|arn| arn.target_id) - .map_err(|e| TargetIDError::InvalidFormat(e.to_string())) - }, - )?; - - Ok::<_, TargetIDError>(event_rules) - }; - - let (clear_result, event_rules_result) = tokio::join!(clear_rules, parse_rules); - - clear_result.map_err(|e| s3_error!(InternalError, "Failed to clear rules: {e}"))?; - let event_rules = - event_rules_result.map_err(|e| s3_error!(InvalidArgument, "Invalid ARN in notification configuration: {e}"))?; - warn!("notify event rules: {:?}", &event_rules); - - // Add a new notification rule - notifier_global::add_event_specific_rules(&bucket, ®ion, &event_rules) - .await - .map_err(|e| s3_error!(InternalError, "Failed to add rules: {e}"))?; - - Ok(S3Response::new(PutBucketNotificationConfigurationOutput {})) - } - - async fn get_bucket_acl(&self, req: S3Request) -> S3Result> { - let GetBucketAclInput { bucket, .. } = req.input; - - let Some(store) = new_object_layer_fn() else { - return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); - }; - - store - .get_bucket_info(&bucket, &BucketOptions::default()) - .await - .map_err(ApiError::from)?; - - let grants = vec![Grant { - grantee: Some(Grantee { - type_: Type::from_static(Type::CANONICAL_USER), - display_name: None, - email_address: None, - id: None, - uri: None, - }), - permission: Some(Permission::from_static(Permission::FULL_CONTROL)), - }]; - - Ok(S3Response::new(GetBucketAclOutput { - grants: Some(grants), - owner: Some(RUSTFS_OWNER.to_owned()), - })) - } - - async fn put_bucket_acl(&self, req: S3Request) -> S3Result> { - let PutBucketAclInput { - bucket, - acl, - access_control_policy, - .. - } = req.input; - - // TODO:checkRequestAuthType - - let Some(store) = new_object_layer_fn() else { - return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); - }; - - store - .get_bucket_info(&bucket, &BucketOptions::default()) - .await - .map_err(ApiError::from)?; - - if let Some(canned_acl) = acl { - if canned_acl.as_str() != BucketCannedACL::PRIVATE { - return Err(s3_error!(NotImplemented)); - } - } else { - let is_full_control = access_control_policy.is_some_and(|v| { - v.grants.is_some_and(|gs| { - // - !gs.is_empty() - && gs.first().is_some_and(|g| { - g.to_owned() - .permission - .is_some_and(|p| p.as_str() == Permission::FULL_CONTROL) - }) - }) - }); - - if !is_full_control { - return Err(s3_error!(NotImplemented)); - } - } - Ok(S3Response::new(PutBucketAclOutput::default())) - } - - async fn get_object_acl(&self, req: S3Request) -> S3Result> { - let GetObjectAclInput { bucket, key, .. } = req.input; - - let Some(store) = new_object_layer_fn() else { - return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); - }; - - if let Err(e) = store.get_object_info(&bucket, &key, &ObjectOptions::default()).await { - return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("{e}"))); - } - - let grants = vec![Grant { - grantee: Some(Grantee { - type_: Type::from_static(Type::CANONICAL_USER), - display_name: None, - email_address: None, - id: None, - uri: None, - }), - permission: Some(Permission::from_static(Permission::FULL_CONTROL)), - }]; - - Ok(S3Response::new(GetObjectAclOutput { - grants: Some(grants), - owner: Some(RUSTFS_OWNER.to_owned()), - ..Default::default() - })) - } - - async fn get_object_attributes( - &self, - req: S3Request, - ) -> S3Result> { - let mut helper = OperationHelper::new(&req, EventName::ObjectAccessedAttributes, "s3:GetObjectAttributes"); - let GetObjectAttributesInput { bucket, key, .. } = req.input.clone(); - - let Some(store) = new_object_layer_fn() else { - return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); - }; - - if let Err(e) = store - .get_object_reader(&bucket, &key, None, HeaderMap::new(), &ObjectOptions::default()) - .await - { - return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("{e}"))); - } - - let output = GetObjectAttributesOutput { - delete_marker: None, - object_parts: None, - ..Default::default() - }; - - let version_id = req.input.version_id.clone().unwrap_or_default(); - helper = helper - .object(ObjectInfo { - name: key.clone(), - bucket, - ..Default::default() - }) - .version_id(version_id); - - let result = Ok(S3Response::new(output)); - let _ = helper.complete(&result); - result - } - - async fn put_object_acl(&self, req: S3Request) -> S3Result> { - let PutObjectAclInput { - bucket, - key, - acl, - access_control_policy, - .. - } = req.input; - - let Some(store) = new_object_layer_fn() else { - return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); - }; - - if let Err(e) = store.get_object_info(&bucket, &key, &ObjectOptions::default()).await { - return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("{e}"))); - } - - if let Some(canned_acl) = acl { - if canned_acl.as_str() != BucketCannedACL::PRIVATE { - return Err(s3_error!(NotImplemented)); - } - } else { - let is_full_control = access_control_policy.is_some_and(|v| { - v.grants.is_some_and(|gs| { - // - !gs.is_empty() - && gs.first().is_some_and(|g| { - g.to_owned() - .permission - .is_some_and(|p| p.as_str() == Permission::FULL_CONTROL) - }) - }) - }); - - if !is_full_control { - return Err(s3_error!(NotImplemented)); - } - } - Ok(S3Response::new(PutObjectAclOutput::default())) - } - - async fn select_object_content( - &self, - req: S3Request, - ) -> S3Result> { - info!("handle select_object_content"); - - let input = Arc::new(req.input); - info!("{:?}", input); - - let db = get_global_db((*input).clone(), false).await.map_err(|e| { - error!("get global db failed, {}", e.to_string()); - s3_error!(InternalError, "{}", e.to_string()) - })?; - let query = Query::new(Context { input: input.clone() }, input.request.expression.clone()); - let result = db - .execute(&query) - .await - .map_err(|e| s3_error!(InternalError, "{}", e.to_string()))?; - - let results = result.result().chunk_result().await.unwrap().to_vec(); - - let mut buffer = Vec::new(); - if input.request.output_serialization.csv.is_some() { - let mut csv_writer = CsvWriterBuilder::new().with_header(false).build(&mut buffer); - for batch in results { - csv_writer - .write(&batch) - .map_err(|e| s3_error!(InternalError, "can't encode output to csv. e: {}", e.to_string()))?; - } - } else if input.request.output_serialization.json.is_some() { - let mut json_writer = JsonWriterBuilder::new() - .with_explicit_nulls(true) - .build::<_, JsonArray>(&mut buffer); - for batch in results { - json_writer - .write(&batch) - .map_err(|e| s3_error!(InternalError, "can't encode output to json. e: {}", e.to_string()))?; - } - json_writer - .finish() - .map_err(|e| s3_error!(InternalError, "writer output into json error, e: {}", e.to_string()))?; - } else { - return Err(s3_error!( - InvalidArgument, - "Unsupported output format. Supported formats are CSV and JSON" - )); - } - - let (tx, rx) = mpsc::channel::>(2); - let stream = ReceiverStream::new(rx); - tokio::spawn(async move { - let _ = tx - .send(Ok(SelectObjectContentEvent::Cont(ContinuationEvent::default()))) - .await; - let _ = tx - .send(Ok(SelectObjectContentEvent::Records(RecordsEvent { - payload: Some(Bytes::from(buffer)), - }))) - .await; - let _ = tx.send(Ok(SelectObjectContentEvent::End(EndEvent::default()))).await; - - drop(tx); - }); - - Ok(S3Response::new(SelectObjectContentOutput { - payload: Some(SelectObjectContentEventStream::new(stream)), - })) - } - async fn get_object_legal_hold( - &self, - req: S3Request, - ) -> S3Result> { - let mut helper = OperationHelper::new(&req, EventName::ObjectAccessedGetLegalHold, "s3:GetObjectLegalHold"); - let GetObjectLegalHoldInput { - bucket, key, version_id, .. - } = req.input.clone(); - - let Some(store) = new_object_layer_fn() else { - return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); - }; - - let _ = store - .get_bucket_info(&bucket, &BucketOptions::default()) - .await - .map_err(ApiError::from)?; - - // check object lock - validate_bucket_object_lock_enabled(&bucket).await?; - - let opts: ObjectOptions = get_opts(&bucket, &key, version_id, None, &req.headers) - .await - .map_err(ApiError::from)?; - - let object_info = store.get_object_info(&bucket, &key, &opts).await.map_err(|e| { - error!("get_object_info failed, {}", e.to_string()); - s3_error!(InternalError, "{}", e.to_string()) - })?; - - let legal_hold = object_info - .user_defined - .get(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER) - .map(|v| v.as_str().to_string()); - - let status = if let Some(v) = legal_hold { - v - } else { - ObjectLockLegalHoldStatus::OFF.to_string() - }; - - let output = GetObjectLegalHoldOutput { - legal_hold: Some(ObjectLockLegalHold { - status: Some(ObjectLockLegalHoldStatus::from(status)), - }), - }; - - let version_id = req.input.version_id.clone().unwrap_or_else(|| Uuid::new_v4().to_string()); - helper = helper.object(object_info).version_id(version_id); - - let result = Ok(S3Response::new(output)); - let _ = helper.complete(&result); - result - } - - async fn put_object_legal_hold( - &self, - req: S3Request, - ) -> S3Result> { - let mut helper = OperationHelper::new(&req, EventName::ObjectCreatedPutLegalHold, "s3:PutObjectLegalHold"); - let PutObjectLegalHoldInput { - bucket, - key, - legal_hold, - version_id, - .. - } = req.input.clone(); - - let Some(store) = new_object_layer_fn() else { - return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); - }; - - let _ = store - .get_bucket_info(&bucket, &BucketOptions::default()) - .await - .map_err(ApiError::from)?; - - // check object lock - validate_bucket_object_lock_enabled(&bucket).await?; - let opts: ObjectOptions = get_opts(&bucket, &key, version_id, None, &req.headers) - .await - .map_err(ApiError::from)?; - - let eval_metadata = parse_object_lock_legal_hold(legal_hold)?; - - let popts = ObjectOptions { - mod_time: opts.mod_time, - version_id: opts.version_id, - eval_metadata: Some(eval_metadata), - ..Default::default() - }; - - let info = store.put_object_metadata(&bucket, &key, &popts).await.map_err(|e| { - error!("put_object_metadata failed, {}", e.to_string()); - s3_error!(InternalError, "{}", e.to_string()) - })?; - - let output = PutObjectLegalHoldOutput { - request_charged: Some(RequestCharged::from_static(RequestCharged::REQUESTER)), - }; - let version_id = req.input.version_id.clone().unwrap_or_default(); - helper = helper.object(info).version_id(version_id); - - let result = Ok(S3Response::new(output)); - let _ = helper.complete(&result); - result - } - - async fn get_object_retention( - &self, - req: S3Request, - ) -> S3Result> { - let mut helper = OperationHelper::new(&req, EventName::ObjectAccessedGetRetention, "s3:GetObjectRetention"); - let GetObjectRetentionInput { - bucket, key, version_id, .. - } = req.input.clone(); - - let Some(store) = new_object_layer_fn() else { - return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); - }; - - // check object lock - validate_bucket_object_lock_enabled(&bucket).await?; - - let opts: ObjectOptions = get_opts(&bucket, &key, version_id, None, &req.headers) - .await - .map_err(ApiError::from)?; - - let object_info = store.get_object_info(&bucket, &key, &opts).await.map_err(|e| { - error!("get_object_info failed, {}", e.to_string()); - s3_error!(InternalError, "{}", e.to_string()) - })?; - - let mode = object_info - .user_defined - .get("x-amz-object-lock-mode") - .map(|v| ObjectLockRetentionMode::from(v.as_str().to_string())); - - let retain_until_date = object_info - .user_defined - .get("x-amz-object-lock-retain-until-date") - .and_then(|v| OffsetDateTime::parse(v.as_str(), &Rfc3339).ok()) - .map(Timestamp::from); - - let output = GetObjectRetentionOutput { - retention: Some(ObjectLockRetention { mode, retain_until_date }), - }; - let version_id = req.input.version_id.clone().unwrap_or_default(); - helper = helper.object(object_info).version_id(version_id); - - let result = Ok(S3Response::new(output)); - let _ = helper.complete(&result); - result - } - - async fn put_object_retention( - &self, - req: S3Request, - ) -> S3Result> { - let mut helper = OperationHelper::new(&req, EventName::ObjectCreatedPutRetention, "s3:PutObjectRetention"); - let PutObjectRetentionInput { - bucket, - key, - retention, - version_id, - .. - } = req.input.clone(); - - let Some(store) = new_object_layer_fn() else { - return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); - }; - - // check object lock - validate_bucket_object_lock_enabled(&bucket).await?; - - // Extract new retain_until_date for validation - let new_retain_until = retention - .as_ref() - .and_then(|r| r.retain_until_date.as_ref()) - .map(|d| OffsetDateTime::from(d.clone())); - - // Check if object already has retention and if modification is allowed - // This follows AWS S3 and MinIO behavior: - // - COMPLIANCE mode: can only extend retention period, never shorten - // - GOVERNANCE mode: requires bypass header to modify/shorten retention - // - // TODO: Known race condition (fix in future PR) - // There's a TOCTOU (time-of-check-time-of-use) window between retention check here - // and the actual update at put_object_metadata. In theory: - // Thread A: reads GOVERNANCE mode, checks bypass header - // Thread B: updates retention to COMPLIANCE mode - // Thread A: modifies retention, bypassing what is now COMPLIANCE mode - // This violates S3 spec that COMPLIANCE cannot be modified even with bypass. - // Fix options: - // 1. Pass expected retention mode to storage layer, verify before update - // 2. Use optimistic concurrency with version/etag checks - // 3. Perform check within the same lock scope as update in storage layer - // Current mitigation: Storage layer has fast_lock_manager which provides some protection - let check_opts: ObjectOptions = get_opts(&bucket, &key, version_id.clone(), None, &req.headers) - .await - .map_err(ApiError::from)?; - - if let Ok(existing_obj_info) = store.get_object_info(&bucket, &key, &check_opts).await { - let bypass_governance = has_bypass_governance_header(&req.headers); - if let Some(block_reason) = - check_retention_for_modification(&existing_obj_info.user_defined, new_retain_until, bypass_governance) - { - return Err(S3Error::with_message(S3ErrorCode::AccessDenied, block_reason.error_message())); - } - } - - let eval_metadata = parse_object_lock_retention(retention)?; - - let mut opts: ObjectOptions = get_opts(&bucket, &key, version_id, None, &req.headers) - .await - .map_err(ApiError::from)?; - opts.eval_metadata = Some(eval_metadata); - - let object_info = store.put_object_metadata(&bucket, &key, &opts).await.map_err(|e| { - error!("put_object_metadata failed, {}", e.to_string()); - s3_error!(InternalError, "{}", e.to_string()) - })?; - - let output = PutObjectRetentionOutput { - request_charged: Some(RequestCharged::from_static(RequestCharged::REQUESTER)), - }; - - let version_id = req.input.version_id.clone().unwrap_or_else(|| Uuid::new_v4().to_string()); - helper = helper.object(object_info).version_id(version_id); - - let result = Ok(S3Response::new(output)); - let _ = helper.complete(&result); - result - } -} - -/// Auxiliary functions: extract prefixes and suffixes -fn extract_prefix_suffix(filter: Option<&NotificationConfigurationFilter>) -> (String, String) { - if let Some(filter) = filter - && let Some(filter_rules) = &filter.key - { - let mut prefix = String::new(); - let mut suffix = String::new(); - if let Some(rules) = &filter_rules.filter_rules { - for rule in rules { - if let (Some(name), Some(value)) = (rule.name.as_ref(), rule.value.as_ref()) { - match name.as_str() { - "prefix" => prefix = value.clone(), - "suffix" => suffix = value.clone(), - _ => {} - } - } - } - } - return (prefix, suffix); - } - (String::new(), String::new()) -} - -/// Auxiliary functions: Handle configuration -pub(crate) fn process_queue_configurations( - event_rules: &mut Vec<(Vec, String, String, Vec)>, - configurations: Option>, - target_id_parser: F, -) -> Result<(), TargetIDError> -where - F: Fn(&str) -> Result, -{ - if let Some(configs) = configurations { - for cfg in configs { - let events = cfg.events.iter().filter_map(|e| EventName::parse(e.as_ref()).ok()).collect(); - let (prefix, suffix) = extract_prefix_suffix(cfg.filter.as_ref()); - let target_id = target_id_parser(&cfg.queue_arn)?; - event_rules.push((events, prefix, suffix, vec![target_id])); - } - } - Ok(()) -} - -pub(crate) fn process_topic_configurations( - event_rules: &mut Vec<(Vec, String, String, Vec)>, - configurations: Option>, - target_id_parser: F, -) -> Result<(), TargetIDError> -where - F: Fn(&str) -> Result, -{ - if let Some(configs) = configurations { - for cfg in configs { - let events = cfg.events.iter().filter_map(|e| EventName::parse(e.as_ref()).ok()).collect(); - let (prefix, suffix) = extract_prefix_suffix(cfg.filter.as_ref()); - let target_id = target_id_parser(&cfg.topic_arn)?; - event_rules.push((events, prefix, suffix, vec![target_id])); - } - } - Ok(()) -} - -pub(crate) fn process_lambda_configurations( - event_rules: &mut Vec<(Vec, String, String, Vec)>, - configurations: Option>, - target_id_parser: F, -) -> Result<(), TargetIDError> -where - F: Fn(&str) -> Result, -{ - if let Some(configs) = configurations { - for cfg in configs { - let events = cfg.events.iter().filter_map(|e| EventName::parse(e.as_ref()).ok()).collect(); - let (prefix, suffix) = extract_prefix_suffix(cfg.filter.as_ref()); - let target_id = target_id_parser(&cfg.lambda_function_arn)?; - event_rules.push((events, prefix, suffix, vec![target_id])); - } - } - Ok(()) -} - -pub(crate) async fn has_replication_rules(bucket: &str, objects: &[ObjectToDelete]) -> bool { - let (cfg, _created) = match get_replication_config(bucket).await { - Ok(replication_config) => replication_config, - Err(_err) => { - return false; - } - }; - - for object in objects { - if cfg.has_active_rules(&object.object_name, true) { - return true; - } - } - false -} - -#[cfg(test)] -mod tests { - use super::*; - use rustfs_config::MI_B; - use rustfs_ecstore::set_disk::DEFAULT_READ_BUFFER_SIZE; - - #[test] - fn test_fs_creation() { - let _fs = FS::new(); - - // Verify that FS struct can be created successfully - // Since it's currently empty, we just verify it doesn't panic - // The test passes if we reach this point without panicking - } - - #[test] - fn test_fs_debug_implementation() { - let fs = FS::new(); - - // Test that Debug trait is properly implemented - let debug_str = format!("{fs:?}"); - assert!(debug_str.contains("FS")); - } - - #[test] - fn test_fs_clone_implementation() { - let fs = FS::new(); - - // Test that Clone trait is properly implemented - let cloned_fs = fs.clone(); - - // Both should be equivalent (since FS is currently empty) - assert_eq!(format!("{fs:?}"), format!("{cloned_fs:?}")); - } - - #[test] - fn test_rustfs_owner_constant() { - // Test that RUSTFS_OWNER constant is properly defined - assert!(!RUSTFS_OWNER.display_name.as_ref().unwrap().is_empty()); - assert!(!RUSTFS_OWNER.id.as_ref().unwrap().is_empty()); - assert_eq!(RUSTFS_OWNER.display_name.as_ref().unwrap(), "rustfs"); - } - - // Note: Most S3 API methods require complex setup with global state, storage backend, - // and various dependencies that make unit testing challenging. For comprehensive testing - // of S3 operations, integration tests would be more appropriate. - - #[test] - fn test_list_objects_v2_key_count_includes_prefixes() { - // Test that KeyCount calculation includes both objects and common prefixes - // This verifies the fix for S3 API compatibility where KeyCount should equal - // the sum of Contents and CommonPrefixes lengths - - // Simulate the calculation logic from list_objects_v2 - let objects_count = 3_usize; - let common_prefixes_count = 2_usize; - - // KeyCount should include both objects and common prefixes per S3 API spec - let key_count = (objects_count + common_prefixes_count) as i32; - - assert_eq!(key_count, 5); - - // Edge cases: verify calculation logic - let no_objects = 0_usize; - let no_prefixes = 0_usize; - assert_eq!((no_objects + no_prefixes) as i32, 0); - - let one_object = 1_usize; - assert_eq!((one_object + no_prefixes) as i32, 1); - - let one_prefix = 1_usize; - assert_eq!((no_objects + one_prefix) as i32, 1); - } - - #[test] - fn test_s3_url_encoding_preserves_slash() { - // Test that S3 URL encoding preserves path separators (/) - // This verifies the encoding logic for EncodingType=url parameter - - use urlencoding::encode; - - // Helper function matching the implementation - let encode_s3_name = |name: &str| -> String { - name.split('/') - .map(|part| encode(part).to_string()) - .collect::>() - .join("/") - }; - - // Test cases from s3-tests - assert_eq!(encode_s3_name("asdf+b"), "asdf%2Bb"); - assert_eq!(encode_s3_name("foo+1/bar"), "foo%2B1/bar"); - assert_eq!(encode_s3_name("foo/"), "foo/"); - assert_eq!(encode_s3_name("quux ab/"), "quux%20ab/"); - - // Edge cases - assert_eq!(encode_s3_name("normal/key"), "normal/key"); - assert_eq!(encode_s3_name("key+with+plus"), "key%2Bwith%2Bplus"); - assert_eq!(encode_s3_name("key with spaces"), "key%20with%20spaces"); - } - - #[test] - fn test_s3_error_scenarios() { - // Test that we can create expected S3 errors for common validation cases - - // Test incomplete body error - let incomplete_body_error = s3_error!(IncompleteBody); - assert_eq!(incomplete_body_error.code(), &S3ErrorCode::IncompleteBody); - - // Test invalid argument error - let invalid_arg_error = s3_error!(InvalidArgument, "test message"); - assert_eq!(invalid_arg_error.code(), &S3ErrorCode::InvalidArgument); - - // Test internal error - let internal_error = S3Error::with_message(S3ErrorCode::InternalError, "test".to_string()); - assert_eq!(internal_error.code(), &S3ErrorCode::InternalError); - } - - #[test] - fn test_compression_format_usage() { - // Test that compression format detection works for common file extensions - let zip_format = CompressionFormat::from_extension("zip"); - assert_eq!(zip_format.extension(), "zip"); - - let tar_format = CompressionFormat::from_extension("tar"); - assert_eq!(tar_format.extension(), "tar"); - - let gz_format = CompressionFormat::from_extension("gz"); - assert_eq!(gz_format.extension(), "gz"); - } - - #[test] - fn test_adaptive_buffer_size_with_profile() { - const KB: i64 = 1024; - const MB: i64 = 1024 * 1024; - - // Test GeneralPurpose profile (default behavior, should match get_adaptive_buffer_size) - assert_eq!( - get_adaptive_buffer_size_with_profile(500 * KB, Some(WorkloadProfile::GeneralPurpose)), - 64 * KB as usize - ); - assert_eq!( - get_adaptive_buffer_size_with_profile(50 * MB, Some(WorkloadProfile::GeneralPurpose)), - 256 * KB as usize - ); - assert_eq!( - get_adaptive_buffer_size_with_profile(200 * MB, Some(WorkloadProfile::GeneralPurpose)), - DEFAULT_READ_BUFFER_SIZE - ); - - // Test AiTraining profile - larger buffers for large files - assert_eq!( - get_adaptive_buffer_size_with_profile(5 * MB, Some(WorkloadProfile::AiTraining)), - 512 * KB as usize - ); - assert_eq!( - get_adaptive_buffer_size_with_profile(100 * MB, Some(WorkloadProfile::AiTraining)), - 2 * MB as usize - ); - assert_eq!( - get_adaptive_buffer_size_with_profile(600 * MB, Some(WorkloadProfile::AiTraining)), - 4 * MB as usize - ); - - // Test WebWorkload profile - smaller buffers for web assets - assert_eq!( - get_adaptive_buffer_size_with_profile(100 * KB, Some(WorkloadProfile::WebWorkload)), - 32 * KB as usize - ); - assert_eq!( - get_adaptive_buffer_size_with_profile(5 * MB, Some(WorkloadProfile::WebWorkload)), - 128 * KB as usize - ); - assert_eq!( - get_adaptive_buffer_size_with_profile(50 * MB, Some(WorkloadProfile::WebWorkload)), - 256 * KB as usize - ); - - // Test SecureStorage profile - memory-constrained buffers - assert_eq!( - get_adaptive_buffer_size_with_profile(500 * KB, Some(WorkloadProfile::SecureStorage)), - 32 * KB as usize - ); - assert_eq!( - get_adaptive_buffer_size_with_profile(25 * MB, Some(WorkloadProfile::SecureStorage)), - 128 * KB as usize - ); - assert_eq!( - get_adaptive_buffer_size_with_profile(100 * MB, Some(WorkloadProfile::SecureStorage)), - 256 * KB as usize - ); - - // Test IndustrialIoT profile - low latency, moderate buffers - assert_eq!( - get_adaptive_buffer_size_with_profile(512 * KB, Some(WorkloadProfile::IndustrialIoT)), - 64 * KB as usize - ); - assert_eq!( - get_adaptive_buffer_size_with_profile(25 * MB, Some(WorkloadProfile::IndustrialIoT)), - 256 * KB as usize - ); - assert_eq!( - get_adaptive_buffer_size_with_profile(100 * MB, Some(WorkloadProfile::IndustrialIoT)), - 512 * KB as usize - ); - - // Test DataAnalytics profile - assert_eq!( - get_adaptive_buffer_size_with_profile(2 * MB, Some(WorkloadProfile::DataAnalytics)), - 128 * KB as usize - ); - assert_eq!( - get_adaptive_buffer_size_with_profile(100 * MB, Some(WorkloadProfile::DataAnalytics)), - 512 * KB as usize - ); - assert_eq!( - get_adaptive_buffer_size_with_profile(500 * MB, Some(WorkloadProfile::DataAnalytics)), - 2 * MB as usize - ); - - // Test with None (should auto-detect or use GeneralPurpose) - let result = get_adaptive_buffer_size_with_profile(50 * MB, None); - // Should be either SecureStorage (if on special OS) or GeneralPurpose - assert!(result == 128 * KB as usize || result == 256 * KB as usize); - - // Test unknown file size with different profiles - assert_eq!( - get_adaptive_buffer_size_with_profile(-1, Some(WorkloadProfile::AiTraining)), - 2 * MB as usize - ); - assert_eq!( - get_adaptive_buffer_size_with_profile(-1, Some(WorkloadProfile::WebWorkload)), - 128 * KB as usize - ); - assert_eq!( - get_adaptive_buffer_size_with_profile(-1, Some(WorkloadProfile::SecureStorage)), - 128 * KB as usize - ); - } - - #[test] - fn test_phase3_default_behavior() { - use crate::config::workload_profiles::{ - RustFSBufferConfig, WorkloadProfile, init_global_buffer_config, set_buffer_profile_enabled, - }; - - const KB: i64 = 1024; - const MB: i64 = 1024 * 1024; - - // Test Phase 3: Enabled by default with GeneralPurpose profile - set_buffer_profile_enabled(true); - init_global_buffer_config(RustFSBufferConfig::new(WorkloadProfile::GeneralPurpose)); - - // Verify GeneralPurpose profile provides consistent buffer sizes - assert_eq!(get_buffer_size_opt_in(500 * KB), 64 * KB as usize); - assert_eq!(get_buffer_size_opt_in(50 * MB), 256 * KB as usize); - assert_eq!(get_buffer_size_opt_in(200 * MB), MI_B); - assert_eq!(get_buffer_size_opt_in(-1), MI_B); // Unknown size - - // Reset for other tests - set_buffer_profile_enabled(false); - } - - #[test] - fn test_buffer_size_opt_in() { - use crate::config::workload_profiles::{is_buffer_profile_enabled, set_buffer_profile_enabled}; - - const KB: i64 = 1024; - const MB: i64 = 1024 * 1024; - - // \[1\] Default state: profile is not enabled, global configuration is not explicitly initialized - // get_buffer_size_opt_in should be equivalent to the GeneralPurpose configuration - set_buffer_profile_enabled(false); - assert!(!is_buffer_profile_enabled()); - - // GeneralPurpose rules: - // \< 1MB -> 64KB，1MB-100MB -> 256KB，\>=100MB -> 1MB - assert_eq!(get_buffer_size_opt_in(500 * KB), 64 * KB as usize); - assert_eq!(get_buffer_size_opt_in(50 * MB), 256 * KB as usize); - assert_eq!(get_buffer_size_opt_in(200 * MB), MI_B); - - // \[2\] Enable the profile switch, but the global configuration is still the default GeneralPurpose - set_buffer_profile_enabled(true); - assert!(is_buffer_profile_enabled()); - - assert_eq!(get_buffer_size_opt_in(500 * KB), 64 * KB as usize); - assert_eq!(get_buffer_size_opt_in(50 * MB), 256 * KB as usize); - assert_eq!(get_buffer_size_opt_in(200 * MB), MI_B); - - // \[3\] Close again to ensure unchanged behavior - set_buffer_profile_enabled(false); - assert!(!is_buffer_profile_enabled()); - assert_eq!(get_buffer_size_opt_in(500 * KB), 64 * KB as usize); - } - - #[test] - fn test_phase4_full_integration() { - use crate::config::workload_profiles::{ - RustFSBufferConfig, WorkloadProfile, init_global_buffer_config, set_buffer_profile_enabled, - }; - - const KB: i64 = 1024; - const MB: i64 = 1024 * 1024; - - // \[1\] During the entire test process, the global configuration is initialized only once. - // In order not to interfere with other tests, use GeneralPurpose (consistent with the default). - // If it has been initialized elsewhere, this call will be ignored by OnceLock and the behavior will still be GeneralPurpose. - init_global_buffer_config(RustFSBufferConfig::new(WorkloadProfile::GeneralPurpose)); - - // Make sure to turn off profile initially - set_buffer_profile_enabled(false); - - // \[2\] Verify behavior of get_buffer_size_opt_in in disabled profile (GeneralPurpose) - assert_eq!(get_buffer_size_opt_in(500 * KB), 64 * KB as usize); - assert_eq!(get_buffer_size_opt_in(50 * MB), 256 * KB as usize); - assert_eq!(get_buffer_size_opt_in(200 * MB), MI_B); - - // \[3\] When profile is enabled, the behavior remains consistent with the global GeneralPurpose configuration - set_buffer_profile_enabled(true); - assert_eq!(get_buffer_size_opt_in(500 * KB), 64 * KB as usize); - assert_eq!(get_buffer_size_opt_in(50 * MB), 256 * KB as usize); - assert_eq!(get_buffer_size_opt_in(200 * MB), MI_B); - - // \[4\] Complex scenes, boundary values: such as unknown size - assert_eq!(get_buffer_size_opt_in(-1), MI_B); - - set_buffer_profile_enabled(false); - } - - #[test] - fn test_validate_list_object_unordered_with_delimiter() { - // [1] Normal case: No delimiter specified. - assert!(validate_list_object_unordered_with_delimiter(None, Some("allow-unordered=true")).is_ok()); - - let delim_str = "/".to_string(); - let delimiter_some: Option<&Delimiter> = Some(&delim_str); - // [2] Normal case: Delimiter is present, but 'allow-unordered' is explicitly set to false. - assert!(validate_list_object_unordered_with_delimiter(delimiter_some, Some("allow-unordered=false")).is_ok()); - - let query_conflict = Some("allow-unordered=true"); - // [3] Conflict case: Both delimiter and 'allow-unordered=true' are present. - assert!(validate_list_object_unordered_with_delimiter(delimiter_some, query_conflict).is_err()); - - let complex_query = Some("allow-unordered=true&abc=123"); - // [4] Complex query: The validation should still trigger if 'allow-unordered=true' is part of a multi-parameter query. - assert!(validate_list_object_unordered_with_delimiter(delimiter_some, complex_query).is_err()); - - let complex_query_without_unordered = Some("abc=123&queryType=test"); - // [5] Multi-parameter query without conflict: If other parameters exist but 'allow-unordered' is missing, - assert!(validate_list_object_unordered_with_delimiter(delimiter_some, complex_query_without_unordered).is_ok()); - } - - #[test] - fn test_parse_object_lock_retention() { - use time::macros::datetime; - // [1] Normal case: No retention specified (empty metadata) - assert!(parse_object_lock_retention(None).is_ok()); - assert!(parse_object_lock_retention(None).unwrap().is_empty()); - - // [2] Normal case: Retention with valid COMPLIANCE mode (future date) - let valid_compliance_retention = ObjectLockRetention { - mode: Some(ObjectLockRetentionMode::from_static(ObjectLockRetentionMode::COMPLIANCE)), - retain_until_date: Some(datetime!(2030-01-01 00:00:00 UTC).into()), - }; - let compliance_metadata = parse_object_lock_retention(Some(valid_compliance_retention)).unwrap(); - assert_eq!(compliance_metadata.get("x-amz-object-lock-mode").unwrap(), "COMPLIANCE"); - assert_eq!( - compliance_metadata.get("x-amz-object-lock-retain-until-date").unwrap(), - "2030-01-01T00:00:00Z" - ); - assert!( - compliance_metadata.contains_key(&format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, "objectlock-retention-timestamp")) - ); - - // [3] Normal case: Retention with valid GOVERNANCE mode (future date) - let valid_governance_retention = ObjectLockRetention { - mode: Some(ObjectLockRetentionMode::from_static(ObjectLockRetentionMode::GOVERNANCE)), - retain_until_date: Some(datetime!(2030-01-01 00:00:00 UTC).into()), - }; - let governance_metadata = parse_object_lock_retention(Some(valid_governance_retention)).unwrap(); - assert_eq!(governance_metadata.get("x-amz-object-lock-mode").unwrap(), "GOVERNANCE"); - - // [4] Normal case: Retention with None mode (empty string for mode, date not validated) - let none_mode_retention = ObjectLockRetention { - mode: None, - retain_until_date: Some(datetime!(2030-01-01 00:00:00 UTC).into()), - }; - let none_mode_metadata = parse_object_lock_retention(Some(none_mode_retention)).unwrap(); - assert_eq!(none_mode_metadata.get("x-amz-object-lock-mode").unwrap(), ""); - - // [5] Normal case: Retention with None retain_until_date (empty string for date) - let none_date_retention = ObjectLockRetention { - mode: Some(ObjectLockRetentionMode::from_static(ObjectLockRetentionMode::COMPLIANCE)), - retain_until_date: None, - }; - let none_date_metadata = parse_object_lock_retention(Some(none_date_retention)).unwrap(); - assert_eq!(none_date_metadata.get("x-amz-object-lock-retain-until-date").unwrap(), ""); - - // [6] Error case: Retention with invalid mode (non COMPLIANCE/GOVERNANCE) - let invalid_mode_retention = ObjectLockRetention { - mode: Some(ObjectLockRetentionMode::from_static("INVALID_MODE")), - retain_until_date: Some(datetime!(2030-01-01 00:00:00 UTC).into()), - }; - let err = parse_object_lock_retention(Some(invalid_mode_retention)).unwrap_err(); - assert_eq!(err.code().as_str(), S3ErrorCode::MalformedXML.as_str()); - assert_eq!( - err.message(), - Some("The XML you provided was not well-formed or did not validate against our published schema") - ); - - // [7] Error case: Retention with past date should fail - let past_date_retention = ObjectLockRetention { - mode: Some(ObjectLockRetentionMode::from_static(ObjectLockRetentionMode::COMPLIANCE)), - retain_until_date: Some(datetime!(2020-01-01 00:00:00 UTC).into()), - }; - let err = parse_object_lock_retention(Some(past_date_retention)).unwrap_err(); - assert_eq!(err.code().as_str(), S3ErrorCode::InvalidArgument.as_str()); - assert_eq!(err.message(), Some("The retain until date must be in the future")); - } - - #[test] - fn test_parse_object_lock_legal_hold() { - // [1] Normal case: No legal hold specified (empty metadata) - assert!(parse_object_lock_legal_hold(None).is_ok()); - assert!(parse_object_lock_legal_hold(None).unwrap().is_empty()); - - // [2] Normal case: Legal hold with valid ON status - let valid_on_legal_hold = ObjectLockLegalHold { - status: Some(ObjectLockLegalHoldStatus::from_static(ObjectLockLegalHoldStatus::ON)), - }; - let on_metadata = parse_object_lock_legal_hold(Some(valid_on_legal_hold)).unwrap(); - assert_eq!(on_metadata.get(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER).unwrap(), "ON"); - assert!(on_metadata.contains_key(&format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, "objectlock-legalhold-timestamp"))); - - // [3] Normal case: Legal hold with valid OFF status - let valid_off_legal_hold = ObjectLockLegalHold { - status: Some(ObjectLockLegalHoldStatus::from_static(ObjectLockLegalHoldStatus::OFF)), - }; - let off_metadata = parse_object_lock_legal_hold(Some(valid_off_legal_hold)).unwrap(); - assert_eq!(off_metadata.get(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER).unwrap(), "OFF"); - - // [4] Normal case: Legal hold with None status (empty string for status) - let none_status_legal_hold = ObjectLockLegalHold { status: None }; - let none_status_metadata = parse_object_lock_legal_hold(Some(none_status_legal_hold)).unwrap(); - assert_eq!(none_status_metadata.get(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER).unwrap(), ""); - - // [5] Error case: Legal hold with invalid status (non ON/OFF) - let invalid_status_legal_hold = ObjectLockLegalHold { - status: Some(ObjectLockLegalHoldStatus::from_static("INVALID_STATUS")), - }; - let err = parse_object_lock_legal_hold(Some(invalid_status_legal_hold)).unwrap_err(); - assert_eq!(err.code().as_str(), S3ErrorCode::MalformedXML.as_str()); - assert_eq!( - err.message(), - Some("The XML you provided was not well-formed or did not validate against our published schema") - ); - } - - #[tokio::test] - async fn test_validate_bucket_object_lock_enabled() { - use rustfs_ecstore::bucket::metadata::BucketMetadata; - use rustfs_ecstore::bucket::metadata_sys::set_bucket_metadata; - use s3s::dto::{ObjectLockConfiguration, ObjectLockEnabled}; - use time::OffsetDateTime; - - if rustfs_ecstore::bucket::metadata_sys::GLOBAL_BucketMetadataSys.get().is_none() { - eprintln!("Skipping test: GLOBAL_BucketMetadataSys not initialized"); - return; - } - - let test_bucket = "test-bucket-object-lock"; - - let mut bm = BucketMetadata::new(test_bucket); - bm.object_lock_config = Some(ObjectLockConfiguration { - object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)), - rule: None, - }); - bm.object_lock_config_updated_at = OffsetDateTime::now_utc(); - set_bucket_metadata(test_bucket.to_string(), bm).await.unwrap(); - assert!(validate_bucket_object_lock_enabled(test_bucket).await.is_ok()); - - let mut bm = BucketMetadata::new(test_bucket); - bm.object_lock_config = Some(ObjectLockConfiguration { - object_lock_enabled: None, - rule: None, - }); - bm.object_lock_config_updated_at = OffsetDateTime::now_utc(); - set_bucket_metadata(test_bucket.to_string(), bm).await.unwrap(); - let err = validate_bucket_object_lock_enabled(test_bucket).await.unwrap_err(); - assert_eq!(err.code().as_str(), S3ErrorCode::InvalidRequest.as_str()); - assert_eq!(err.message(), Some("Object Lock is not enabled for this bucket")); - - let non_exist_bucket = "non-exist-bucket-object-lock"; - let err = validate_bucket_object_lock_enabled(non_exist_bucket).await.unwrap_err(); - assert_eq!(err.code().as_str(), S3ErrorCode::InvalidRequest.as_str()); - assert_eq!(err.message(), Some("Bucket is missing ObjectLockConfiguration")); - } - - #[test] - fn test_is_etag_equal() { - // [1] Header ETag is "*", should return true (match any object ETag) - assert!(is_etag_equal("\"d41d8cd98f00b204e9800998ecf8427e\"", "*")); - - // [2] Exact match (both with double quotes) - assert!(is_etag_equal( - "\"d41d8cd98f00b204e9800998ecf8427e\"", - "\"d41d8cd98f00b204e9800998ecf8427e\"" - )); - - // [3] Exact match (object ETag with quotes, header ETag without) - assert!(is_etag_equal("\"d41d8cd98f00b204e9800998ecf8427e\"", "d41d8cd98f00b204e9800998ecf8427e")); - - // [4] Header ETag has multiple values (comma-separated), one matches - assert!(is_etag_equal("\"12345\"", "\"67890\", \"12345\", \"abcde\"")); - - // [5] Header ETag has multiple values with spaces, one matches after trim - assert!(is_etag_equal("\"12345\"", " \"67890\" , \"12345\" , \"abcde\" ")); - - // [6] No match (different ETag) - assert!(!is_etag_equal("\"12345\"", "\"67890\"")); - - // [7] No match in multiple values - assert!(!is_etag_equal("\"12345\"", "\"67890\", \"abcde\"")); - } - - #[test] - fn test_check_preconditions() { - let valid_mod_time = OffsetDateTime::from_unix_timestamp(1700000000).unwrap(); - let valid_etag = "\"d41d8cd98f00b204e9800998ecf8427e\""; - let wrong_etag = "\"wrong-etag-123456\""; - - // [1] Both mod_time and etag are None → early return Ok() - let info1 = ObjectInfo { - mod_time: None, - etag: None, - ..Default::default() - }; - let headers1 = HeaderMap::new(); - assert!(check_preconditions(&headers1, &info1).is_ok()); - - // [2] No conditional headers with etag=None → Ok() - let info2 = ObjectInfo { - mod_time: Some(valid_mod_time), - etag: None, - ..Default::default() - }; - let headers2 = HeaderMap::new(); - assert!(check_preconditions(&headers2, &info2).is_ok()); - - // [3] If-None-Match matches → return Err(S3Error::NotModified) - let mut headers3 = HeaderMap::new(); - headers3.insert("if-none-match", HeaderValue::from_str(valid_etag).unwrap()); - let info3 = ObjectInfo { - mod_time: Some(valid_mod_time), - etag: Some(valid_etag.to_string()), - ..Default::default() - }; - let result3 = check_preconditions(&headers3, &info3); - assert!(result3.is_err()); - let err3 = result3.unwrap_err(); - assert_eq!(err3.code(), &S3ErrorCode::NotModified); - assert_eq!(err3.message(), Some("Not Modified")); - assert_eq!(err3.status_code(), Some(StatusCode::NOT_MODIFIED)); - - // [4] If-None-Match does not match → return Ok() - let mut headers4 = HeaderMap::new(); - headers4.insert("if-none-match", HeaderValue::from_str("\"wrong-etag\"").unwrap()); - let info4 = info3.clone(); - assert!(check_preconditions(&headers4, &info4).is_ok()); - - // [5] If-Modified-Since >= mod_time → return Err(S3Error::NotModified) - let mut headers5 = HeaderMap::new(); - headers5.insert( - "if-modified-since", - HeaderValue::from_str(&valid_mod_time.format(&Rfc2822).unwrap()).unwrap(), - ); - let info5 = info3.clone(); - let result5 = check_preconditions(&headers5, &info5); - assert!(result5.is_err()); - let err5 = result5.unwrap_err(); - assert_eq!(err5.code(), &S3ErrorCode::NotModified); - - // [6] If-Modified-Since < mod_time → return Ok() - let earlier_time = valid_mod_time - time::Duration::hours(1); - let mut headers6 = HeaderMap::new(); - headers6.insert( - "if-modified-since", - HeaderValue::from_str(&earlier_time.format(&Rfc2822).unwrap()).unwrap(), - ); - let info6 = info3.clone(); - assert!(check_preconditions(&headers6, &info6).is_ok()); - - // [7] If-Match does not match → return Err(S3Error::PreconditionFailed) - let mut headers7 = HeaderMap::new(); - headers7.insert("if-match", HeaderValue::from_str("\"wrong-etag\"").unwrap()); - let info7 = info3.clone(); - let result7 = check_preconditions(&headers7, &info7); - assert!(result7.is_err()); - assert_eq!(result7.unwrap_err().code(), &S3ErrorCode::PreconditionFailed); - - // [8] If-Match matches → return Ok() - let mut headers8 = HeaderMap::new(); - headers8.insert("if-match", HeaderValue::from_str(valid_etag).unwrap()); - let info8 = info3.clone(); - assert!(check_preconditions(&headers8, &info8).is_ok()); - - // [9] If-Unmodified-Since < mod_time (no If-Match) → return Err(S3Error::PreconditionFailed) - let mut headers9 = HeaderMap::new(); - headers9.insert( - "if-unmodified-since", - HeaderValue::from_str(&earlier_time.format(&Rfc2822).unwrap()).unwrap(), - ); - let info9 = info3.clone(); - let result9 = check_preconditions(&headers9, &info9); - assert!(result9.is_err()); - assert_eq!(result9.unwrap_err().code(), &S3ErrorCode::PreconditionFailed); - - // [10] If-Unmodified-Since >= mod_time (no If-Match) → return Ok() - let mut headers10 = HeaderMap::new(); - headers10.insert( - "if-unmodified-since", - HeaderValue::from_str(&valid_mod_time.format(&Rfc2822).unwrap()).unwrap(), - ); - let info10 = info3.clone(); - assert!(check_preconditions(&headers10, &info10).is_ok()); - - // [11] If-Match (mismatch) + If-None-Match (match) → return Err(S3Error::PreconditionFailed) - let mut headers11 = HeaderMap::new(); - headers11.insert("if-match", HeaderValue::from_str(wrong_etag).unwrap()); - headers11.insert("if-none-match", HeaderValue::from_str(valid_etag).unwrap()); - let info11 = info3.clone(); - let result11 = check_preconditions(&headers11, &info11); - assert!(result11.is_err()); - let err11 = result11.unwrap_err(); - assert_eq!(err11.code(), &S3ErrorCode::PreconditionFailed); - assert_ne!(err11.code(), &S3ErrorCode::NotModified); - - // [12] If-Match (match) + If-None-Match (match) → return Err(S3Error::NotModified) - let mut headers12 = HeaderMap::new(); - headers12.insert("if-match", HeaderValue::from_str(valid_etag).unwrap()); - headers12.insert("if-none-match", HeaderValue::from_str(valid_etag).unwrap()); - let info12 = info3.clone(); - let result12 = check_preconditions(&headers12, &info12); - assert!(result12.is_err()); - let err12 = result12.unwrap_err(); - assert_eq!(err12.code(), &S3ErrorCode::NotModified); - assert_ne!(err12.code(), &S3ErrorCode::PreconditionFailed); - - // [13] If-None-Match (match) + If-Modified-Since → NotModified (If-Modified-Since ignored) - let mut headers13 = HeaderMap::new(); - headers13.insert("if-none-match", HeaderValue::from_str(valid_etag).unwrap()); - headers13.insert( - "if-modified-since", - HeaderValue::from_str(&earlier_time.format(&Rfc2822).unwrap()).unwrap(), - ); - let info13 = info3.clone(); - let result13 = check_preconditions(&headers13, &info13); - assert!(result13.is_err()); - assert_eq!(result13.unwrap_err().code(), &S3ErrorCode::NotModified); - - // [14] If-None-Match (no match) + If-Modified-Since → Ok (If-Modified-Since ignored) - let mut headers14 = HeaderMap::new(); - headers14.insert("if-none-match", HeaderValue::from_str("\"wrong-etag\"").unwrap()); - headers14.insert( - "if-modified-since", - HeaderValue::from_str(&valid_mod_time.format(&Rfc2822).unwrap()).unwrap(), - ); - let info14 = info3.clone(); - assert!(check_preconditions(&headers14, &info14).is_ok()); - - // [15] If-Match with no ETag → PreconditionFailed - let mut headers15 = HeaderMap::new(); - headers15.insert("if-match", HeaderValue::from_str(valid_etag).unwrap()); - let info15 = ObjectInfo { - mod_time: Some(valid_mod_time), - etag: None, - ..Default::default() - }; - let result15 = check_preconditions(&headers15, &info15); - assert!(result15.is_err()); - assert_eq!(result15.unwrap_err().code(), &S3ErrorCode::PreconditionFailed); - - // [16] If-None-Match with no ETag → Ok (no match possible) - let mut headers16 = HeaderMap::new(); - headers16.insert("if-none-match", HeaderValue::from_str(valid_etag).unwrap()); - let info16 = ObjectInfo { - mod_time: Some(valid_mod_time), - etag: None, - ..Default::default() - }; - assert!(check_preconditions(&headers16, &info16).is_ok()); - - // [17] mod_time None, etag exists, If-Match matches → Ok - let mut headers17 = HeaderMap::new(); - headers17.insert("if-match", HeaderValue::from_str(valid_etag).unwrap()); - let info17 = ObjectInfo { - mod_time: None, - etag: Some(valid_etag.to_string()), - ..Default::default() - }; - assert!(check_preconditions(&headers17, &info17).is_ok()); - } - - #[test] - fn test_parse_etag() { - // [1] ETag without quotes → adds quotes - let result1 = parse_etag("d41d8cd98f00b204e9800998ecf8427e").unwrap(); - assert_eq!(result1.to_str().unwrap(), "\"d41d8cd98f00b204e9800998ecf8427e\""); - - // [2] ETag with quotes → normalizes to single set of quotes - let result2 = parse_etag("\"d41d8cd98f00b204e9800998ecf8427e\"").unwrap(); - assert_eq!(result2.to_str().unwrap(), "\"d41d8cd98f00b204e9800998ecf8427e\""); - - // [3] ETag with multiple quotes → removes all and adds single set - let result3 = parse_etag("\"\"\"d41d8cd98f00b204e9800998ecf8427e\"\"\"").unwrap(); - assert_eq!(result3.to_str().unwrap(), "\"d41d8cd98f00b204e9800998ecf8427e\""); - - // [4] Empty string → returns error - let result4 = parse_etag(""); - assert!(result4.is_err()); - assert_eq!(result4.unwrap_err().code(), &S3ErrorCode::InvalidArgument); - - // [5] String with only quotes → returns error - let result5 = parse_etag("\"\""); - assert!(result5.is_err()); - assert_eq!(result5.unwrap_err().code(), &S3ErrorCode::InvalidArgument); - - // [6] ETag with quotes only at start → removes and adds quotes - let result6 = parse_etag("\"d41d8cd98f00b204e9800998ecf8427e").unwrap(); - assert_eq!(result6.to_str().unwrap(), "\"d41d8cd98f00b204e9800998ecf8427e\""); - - // [7] ETag with quotes only at end → removes and adds quotes - let result7 = parse_etag("d41d8cd98f00b204e9800998ecf8427e\"").unwrap(); - assert_eq!(result7.to_str().unwrap(), "\"d41d8cd98f00b204e9800998ecf8427e\""); - - // [8] ETag with newline character → returns error (invalid header value) - let result8 = parse_etag("d41d8cd98f00b204e9800998ecf8427e\n"); - assert!(result8.is_err()); - assert_eq!(result8.unwrap_err().code(), &S3ErrorCode::InvalidArgument); - - // [9] ETag with null byte → returns error (invalid header value) - let result9 = parse_etag("d41d8cd98f00b204e9800998ecf8427e\0"); - assert!(result9.is_err()); - assert_eq!(result9.unwrap_err().code(), &S3ErrorCode::InvalidArgument); - - // [10] ETag with carriage return → returns error (invalid header value) - let result10 = parse_etag("d41d8cd98f00b204e9800998ecf8427e\r"); - assert!(result10.is_err()); - assert_eq!(result10.unwrap_err().code(), &S3ErrorCode::InvalidArgument); - - // [11] ETag with whitespace → preserves content, adds quotes - let result11 = parse_etag(" d41d8cd98f00b204e9800998ecf8427e ").unwrap(); - assert_eq!(result11.to_str().unwrap(), "\" d41d8cd98f00b204e9800998ecf8427e \""); - - // [12] ETag with only whitespace (no quotes) → returns error - let result12 = parse_etag(" "); - assert!(result12.is_err()); - assert_eq!(result12.unwrap_err().code(), &S3ErrorCode::InvalidArgument); - - // [13] ETag with only whitespace (with quotes) → returns error - let result13 = parse_etag("\" \""); - assert!(result13.is_err()); - assert_eq!(result13.unwrap_err().code(), &S3ErrorCode::InvalidArgument); - } - - // Note: S3Request structure is complex and requires many fields. - // For real testing, we would need proper integration test setup. - // Removing this test as it requires too much S3 infrastructure setup. - - // Note: Testing actual S3 operations like put_object, get_object, etc. requires: - // 1. Initialized storage backend (ECStore) - // 2. Global configuration setup - // 3. Valid credentials and authorization - // 4. Bucket and object metadata systems - // 5. Network and disk I/O capabilities - // - // These are better suited for integration tests rather than unit tests. - // The current tests focus on the testable parts without external dependencies. - - /// Test that next_key_marker and next_version_id_marker are filtered correctly - /// AWS S3 API requires these fields to be omitted when empty, not set to None or "" - #[test] - fn test_next_marker_filtering() { - // Test filter behavior for empty strings - let empty_string = Some(String::new()); - let filtered = empty_string.filter(|v| !v.is_empty()); - assert!(filtered.is_none(), "Empty string should be filtered to None"); - - // Test filter behavior for non-empty strings - let non_empty = Some("some-marker".to_string()); - let filtered = non_empty.filter(|v| !v.is_empty()); - assert!(filtered.is_some(), "Non-empty string should not be filtered"); - assert_eq!(filtered.unwrap(), "some-marker"); - - // Test filter behavior for None - let none_value: Option = None; - let filtered = none_value.filter(|v| !v.is_empty()); - assert!(filtered.is_none(), "None should remain None"); - } - - /// Test version_id handling for ListObjectVersions response - /// Per AWS S3 API spec: - /// - Versioned objects: version_id is a UUID string - /// - Non-versioned objects: version_id should be "null" string - #[test] - fn test_version_id_formatting() { - use uuid::Uuid; - - // Non-versioned object: version_id is None, should format as "null" - let version_id: Option = None; - let formatted = version_id.map(|v| v.to_string()).unwrap_or_else(|| "null".to_string()); - assert_eq!(formatted, "null"); - - // Versioned object: version_id is Some(UUID), should format as UUID string - let uuid = Uuid::parse_str("550e8400-e29b-41d4-a716-446655440000").unwrap(); - let version_id: Option = Some(uuid); - let formatted = version_id.map(|v| v.to_string()).unwrap_or_else(|| "null".to_string()); - assert_eq!(formatted, "550e8400-e29b-41d4-a716-446655440000"); - } - - /// Test that ListObjectVersionsOutput markers are correctly set - /// This verifies the fix for boto3 ParamValidationError - #[test] - fn test_list_object_versions_markers_handling() { - // Simulate the marker filtering logic from list_object_versions - - // Case 1: Both markers have values (truncated result with versioned object) - let next_marker = Some("object-key".to_string()); - let next_version_idmarker = Some("550e8400-e29b-41d4-a716-446655440000".to_string()); - - let filtered_key_marker = next_marker.filter(|v| !v.is_empty()); - let filtered_version_marker = next_version_idmarker.filter(|v| !v.is_empty()); - - assert!(filtered_key_marker.is_some()); - assert!(filtered_version_marker.is_some()); - - // Case 2: Markers are empty strings (non-truncated result) - let next_marker = Some(String::new()); - let next_version_idmarker = Some(String::new()); - - let filtered_key_marker = next_marker.filter(|v| !v.is_empty()); - let filtered_version_marker = next_version_idmarker.filter(|v| !v.is_empty()); - - assert!(filtered_key_marker.is_none(), "Empty key marker should be filtered to None"); - assert!(filtered_version_marker.is_none(), "Empty version marker should be filtered to None"); - - // Case 3: Truncated result with non-versioned object (version_id is "null") - let next_marker = Some("object-key".to_string()); - let next_version_idmarker = Some("null".to_string()); - - let filtered_key_marker = next_marker.filter(|v| !v.is_empty()); - let filtered_version_marker = next_version_idmarker.filter(|v| !v.is_empty()); - - assert!(filtered_key_marker.is_some()); - assert!(filtered_version_marker.is_some()); - assert_eq!(filtered_version_marker.unwrap(), "null"); - } - - #[test] - fn test_matches_origin_pattern_exact_match() { - // Test exact match - assert!(matches_origin_pattern("https://example.com", "https://example.com")); - assert!(matches_origin_pattern("http://localhost:3000", "http://localhost:3000")); - assert!(!matches_origin_pattern("https://example.com", "https://other.com")); - } - - #[test] - fn test_matches_origin_pattern_wildcard() { - // Test wildcard pattern matching (S3 CORS supports * as subdomain wildcard) - assert!(matches_origin_pattern("https://*.example.com", "https://app.example.com")); - assert!(matches_origin_pattern("https://*.example.com", "https://api.example.com")); - assert!(matches_origin_pattern("https://*.example.com", "https://subdomain.example.com")); - - // Test wildcard at start (matches any domain) - assert!(matches_origin_pattern("https://*", "https://example.com")); - assert!(matches_origin_pattern("https://*", "https://any-domain.com")); - - // Test wildcard at end (matches any protocol) - assert!(matches_origin_pattern("*://example.com", "https://example.com")); - assert!(matches_origin_pattern("*://example.com", "http://example.com")); - - // Test invalid wildcard patterns (should not match) - assert!(!matches_origin_pattern("https://*.*.com", "https://app.example.com")); // Multiple wildcards (invalid pattern) - // Note: "https://*example.com" actually matches "https://app.example.com" with our current implementation - // because it splits on * and checks starts_with/ends_with. This is a limitation but acceptable - // for S3 CORS which typically uses patterns like "https://*.example.com" - } - - #[test] - fn test_matches_origin_pattern_no_wildcard() { - // Test patterns without wildcards - assert!(matches_origin_pattern("https://example.com", "https://example.com")); - assert!(!matches_origin_pattern("https://example.com", "https://example.org")); - assert!(!matches_origin_pattern("http://example.com", "https://example.com")); // Different protocol - } - - #[test] - fn test_matches_origin_pattern_edge_cases() { - // Test edge cases - assert!(!matches_origin_pattern("", "https://example.com")); // Empty pattern - assert!(!matches_origin_pattern("https://example.com", "")); // Empty origin - assert!(matches_origin_pattern("", "")); // Both empty - assert!(!matches_origin_pattern("https://example.com", "http://example.com")); // Protocol mismatch - } - - #[test] - fn test_cors_headers_validation() { - use http::HeaderMap; - - // Test case 1: Validate header name case-insensitivity - let mut headers = HeaderMap::new(); - headers.insert("access-control-request-headers", "Content-Type,X-Custom-Header".parse().unwrap()); - - let req_headers_str = headers - .get("access-control-request-headers") - .and_then(|v| v.to_str().ok()) - .unwrap(); - let req_headers: Vec = req_headers_str.split(',').map(|s| s.trim().to_lowercase()).collect(); - - // Headers should be lowercased for comparison - assert_eq!(req_headers, vec!["content-type", "x-custom-header"]); - - // Test case 2: Wildcard matching - let allowed_headers = ["*".to_string()]; - let all_allowed = req_headers.iter().all(|req_header| { - allowed_headers - .iter() - .any(|allowed| allowed.to_lowercase() == "*" || allowed.to_lowercase() == *req_header) - }); - assert!(all_allowed, "Wildcard should allow all headers"); - - // Test case 3: Specific header matching - let allowed_headers = ["content-type".to_string(), "x-custom-header".to_string()]; - let all_allowed = req_headers - .iter() - .all(|req_header| allowed_headers.iter().any(|allowed| allowed.to_lowercase() == *req_header)); - assert!(all_allowed, "All requested headers should be allowed"); - - // Test case 4: Disallowed header - let req_headers = ["content-type".to_string(), "x-forbidden-header".to_string()]; - let allowed_headers = ["content-type".to_string()]; - let all_allowed = req_headers - .iter() - .all(|req_header| allowed_headers.iter().any(|allowed| allowed.to_lowercase() == *req_header)); - assert!(!all_allowed, "Forbidden header should not be allowed"); - } - - #[test] - fn test_cors_response_headers_structure() { - use http::{HeaderMap, HeaderValue}; - - let mut cors_headers = HeaderMap::new(); - - // Simulate building CORS response headers - let origin = "https://example.com"; - let methods = ["GET", "PUT", "POST"]; - let allowed_headers = ["Content-Type", "Authorization"]; - let expose_headers = ["ETag", "x-amz-version-id"]; - let max_age = 3600; - - // Add headers - cors_headers.insert("access-control-allow-origin", HeaderValue::from_str(origin).unwrap()); - cors_headers.insert("vary", HeaderValue::from_static("Origin")); - - let methods_str = methods.join(", "); - cors_headers.insert("access-control-allow-methods", HeaderValue::from_str(&methods_str).unwrap()); - - let headers_str = allowed_headers.join(", "); - cors_headers.insert("access-control-allow-headers", HeaderValue::from_str(&headers_str).unwrap()); - - let expose_str = expose_headers.join(", "); - cors_headers.insert("access-control-expose-headers", HeaderValue::from_str(&expose_str).unwrap()); - - cors_headers.insert("access-control-max-age", HeaderValue::from_str(&max_age.to_string()).unwrap()); - - // Verify all headers are present - assert_eq!(cors_headers.get("access-control-allow-origin").unwrap(), origin); - assert_eq!(cors_headers.get("vary").unwrap(), "Origin"); - assert_eq!(cors_headers.get("access-control-allow-methods").unwrap(), "GET, PUT, POST"); - assert_eq!(cors_headers.get("access-control-allow-headers").unwrap(), "Content-Type, Authorization"); - assert_eq!(cors_headers.get("access-control-expose-headers").unwrap(), "ETag, x-amz-version-id"); - assert_eq!(cors_headers.get("access-control-max-age").unwrap(), "3600"); - } - - #[test] - fn test_cors_preflight_vs_actual_request() { - use http::Method; - - // Test that we can distinguish preflight from actual requests - let preflight_method = Method::OPTIONS; - let actual_method = Method::PUT; - - assert_eq!(preflight_method, Method::OPTIONS); - assert_ne!(actual_method, Method::OPTIONS); - - // Preflight should check Access-Control-Request-Method - // Actual request should use the actual method - let is_preflight_1 = preflight_method == Method::OPTIONS; - let is_preflight_2 = actual_method == Method::OPTIONS; - - assert!(is_preflight_1); - assert!(!is_preflight_2); - } - - #[tokio::test] - async fn test_apply_cors_headers_no_origin() { - // Test when no Origin header is present - let headers = HeaderMap::new(); - let method = http::Method::GET; - - // Should return None when no origin header - let result = apply_cors_headers("test-bucket", &method, &headers).await; - assert!(result.is_none(), "Should return None when no Origin header"); - } - - #[tokio::test] - async fn test_apply_cors_headers_no_cors_config() { - // Test when bucket has no CORS configuration - let mut headers = HeaderMap::new(); - headers.insert("origin", "https://example.com".parse().unwrap()); - let method = http::Method::GET; - - // Should return None when no CORS config exists - // Note: This test may fail if test-bucket actually has CORS config - // In a real scenario, we'd use a mock or ensure the bucket doesn't exist - let _result = apply_cors_headers("non-existent-bucket-for-testing", &method, &headers).await; - // Result depends on whether bucket exists and has CORS config - // This is expected behavior - we just verify it doesn't panic - } - - #[tokio::test] - async fn test_apply_cors_headers_unsupported_method() { - // Test with unsupported HTTP method - let mut headers = HeaderMap::new(); - headers.insert("origin", "https://example.com".parse().unwrap()); - let method = http::Method::PATCH; // Unsupported method - - let result = apply_cors_headers("test-bucket", &method, &headers).await; - assert!(result.is_none(), "Should return None for unsupported methods"); - } - - #[test] - fn test_matches_origin_pattern_complex_wildcards() { - // Test more complex wildcard scenarios - assert!(matches_origin_pattern("https://*.example.com", "https://sub.example.com")); - // Note: "https://*.example.com" matches "https://api.sub.example.com" with our implementation - // because it only checks starts_with and ends_with. Real S3 might be more strict. - - // Test wildcard in middle position - // Our implementation allows this, but it's not standard S3 CORS pattern - // The pattern "https://example.*.com" splits to ["https://example.", ".com"] - // and "https://example.sub.com" matches because it starts with "https://example." and ends with ".com" - // This is acceptable for our use case as S3 CORS typically uses "https://*.example.com" format - } - - // === Notification Configuration Error Propagation Tests === - - #[test] - fn test_process_queue_configurations_propagates_error_on_invalid_arn() { - use rustfs_targets::arn::{ARN, TargetIDError}; - - let mut event_rules = Vec::new(); - let invalid_arn = "arn:minio:sqs::1:webhook"; // Wrong prefix, should fail - - let result = process_queue_configurations( - &mut event_rules, - Some(vec![QueueConfiguration { - events: vec!["s3:ObjectCreated:*".to_string().into()], - queue_arn: invalid_arn.to_string(), - filter: None, - id: None, - }]), - |arn_str| { - ARN::parse(arn_str) - .map(|arn| arn.target_id) - .map_err(|e| TargetIDError::InvalidFormat(e.to_string())) - }, - ); - - assert!(result.is_err(), "Should return error for invalid ARN prefix"); - assert!(event_rules.is_empty(), "Should not add rules when ARN is invalid"); - } - - #[test] - fn test_process_topic_configurations_propagates_error_on_invalid_arn() { - use rustfs_targets::arn::{ARN, TargetIDError}; - - let mut event_rules = Vec::new(); - let invalid_arn = "arn:aws:sns:us-east-1:123:topic"; // Wrong prefix, should fail - - let result = process_topic_configurations( - &mut event_rules, - Some(vec![TopicConfiguration { - events: vec!["s3:ObjectCreated:*".to_string().into()], - topic_arn: invalid_arn.to_string(), - filter: None, - id: None, - }]), - |arn_str| { - ARN::parse(arn_str) - .map(|arn| arn.target_id) - .map_err(|e| TargetIDError::InvalidFormat(e.to_string())) - }, - ); - - assert!(result.is_err(), "Should return error for invalid ARN prefix"); - assert!(event_rules.is_empty(), "Should not add rules when ARN is invalid"); - } - - #[test] - fn test_process_lambda_configurations_propagates_error_on_invalid_arn() { - use rustfs_targets::arn::{ARN, TargetIDError}; - - let mut event_rules = Vec::new(); - let invalid_arn = "arn:aws:lambda:us-east-1:123:function"; // Wrong prefix, should fail - - let result = process_lambda_configurations( - &mut event_rules, - Some(vec![LambdaFunctionConfiguration { - events: vec!["s3:ObjectCreated:*".to_string().into()], - lambda_function_arn: invalid_arn.to_string(), - filter: None, - id: None, - }]), - |arn_str| { - ARN::parse(arn_str) - .map(|arn| arn.target_id) - .map_err(|e| TargetIDError::InvalidFormat(e.to_string())) - }, - ); - - assert!(result.is_err(), "Should return error for invalid ARN prefix"); - assert!(event_rules.is_empty(), "Should not add rules when ARN is invalid"); - } - - #[test] - fn test_process_queue_configurations_succeeds_with_valid_arn() { - use rustfs_targets::arn::{ARN, TargetIDError}; - - let mut event_rules = Vec::new(); - let valid_arn = "arn:rustfs:sqs:us-east-1:1:webhook"; // Correct prefix - - let result = process_queue_configurations( - &mut event_rules, - Some(vec![QueueConfiguration { - events: vec!["s3:ObjectCreated:*".to_string().into()], - queue_arn: valid_arn.to_string(), - filter: None, - id: None, - }]), - |arn_str| { - ARN::parse(arn_str) - .map(|arn| arn.target_id) - .map_err(|e| TargetIDError::InvalidFormat(e.to_string())) - }, - ); - - assert!(result.is_ok(), "Should succeed with valid ARN"); - assert_eq!(event_rules.len(), 1, "Should add one rule"); - } } diff --git a/rustfs/src/storage/ecfs_extend.rs b/rustfs/src/storage/ecfs_extend.rs new file mode 100644 index 000000000..f521e32cb --- /dev/null +++ b/rustfs/src/storage/ecfs_extend.rs @@ -0,0 +1,1100 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::config::workload_profiles::{ + RustFSBufferConfig, WorkloadProfile, get_global_buffer_config, is_buffer_profile_enabled, +}; +use crate::error::ApiError; +use crate::server::cors; +use crate::storage::ecfs::{InMemoryAsyncReader, ListObjectUnorderedQuery}; +use axum::body::Body; +use http::{HeaderMap, HeaderValue, StatusCode}; +use metrics::counter; +use rustfs_ecstore::bucket::metadata_sys; +use rustfs_ecstore::bucket::metadata_sys::get_replication_config; +use rustfs_ecstore::bucket::replication::ReplicationConfigurationExt; +use rustfs_ecstore::error::StorageError; +use rustfs_ecstore::store_api::{BucketOptions, ObjectInfo, ObjectToDelete}; +use rustfs_ecstore::{StorageAPI, new_object_layer_fn}; +use rustfs_filemeta::ObjectPartInfo; +use rustfs_kms::{EncryptionMetadata, ObjectEncryptionContext, get_global_encryption_service}; +use rustfs_rio::{DecryptReader, Reader, WarpReader}; +use rustfs_targets::EventName; +use rustfs_targets::arn::{TargetID, TargetIDError}; +use rustfs_utils::http::{ + AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER, + RESERVED_METADATA_PREFIX_LOWER, +}; +use s3s::dto::{ + Delimiter, LambdaFunctionConfiguration, NotificationConfigurationFilter, ObjectLockEnabled, ObjectLockLegalHold, + ObjectLockLegalHoldStatus, ObjectLockRetention, ObjectLockRetentionMode, QueueConfiguration, ServerSideEncryption, + TopicConfiguration, +}; +use s3s::{S3Error, S3ErrorCode, S3Response, S3Result}; +use serde_urlencoded::from_bytes; +use std::collections::HashMap; +use std::ops::Add; +use std::sync::Arc; +use time::OffsetDateTime; +use time::format_description::well_known::{Rfc2822, Rfc3339}; +use tokio::io::AsyncRead; +use tracing::{debug, warn}; + +/// ======================= +/// Presigned POST helpers +/// ======================= +/// +/// AWS S3 RESTObjectPOST (HTML form upload) semantics: +/// - Default success response is 204 (No Content) +/// - If `success_action_status` is specified, it may be 200, 201, or 204 +/// - If `success_action_redirect` is specified, respond with 303 and Location header. +/// +/// Reference: +/// https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectPOST.html +#[allow(dead_code)] +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub(crate) enum PostObjectSuccessAction { + #[default] + NoContent204, + Ok200, + Created201, + Redirect303 { + location: String, + }, +} + +/// Parse success action from Presigned POST form fields. +/// +/// Integration point (manual): +/// - In the PostPolicy handler, after parsing form fields, call this function to determine the desired success action. +/// +/// # Arguments +/// * `fields` - HashMap of form fields from the POST request +/// +/// # Returns +/// * `S3Result` - Parsed success action or error +/// +/// Notes: +/// - Follows AWS S3 behavior: `success_action_redirect` takes precedence over `success_action_status`. +/// - Validates `success_action_status` values; invalid values result in MalformedPOSTRequest error. +/// +#[allow(dead_code)] +pub(crate) fn parse_success_action_from_form_fields(fields: &HashMap) -> S3Result { + // 1) success_action_redirect wins over success_action_status (AWS compatible behavior). + if let Some(loc) = fields + .get("success_action_redirect") + .map(|s| s.trim()) + .filter(|s| !s.is_empty()) + { + return Ok(PostObjectSuccessAction::Redirect303 { + location: loc.to_string(), + }); + } + + // 2) success_action_status is optional; default is 204. + let Some(status_str) = fields + .get("success_action_status") + .map(|s| s.trim()) + .filter(|s| !s.is_empty()) + else { + return Ok(PostObjectSuccessAction::NoContent204); + }; + + // AWS allows only 200/201/204 for POST form success_action_status. + // Treat invalid values as MalformedPOSTRequest to match S3 strictness. + match status_str { + "200" => Ok(PostObjectSuccessAction::Ok200), + "201" => Ok(PostObjectSuccessAction::Created201), + "204" => Ok(PostObjectSuccessAction::NoContent204), + _ => Err(S3Error::with_message( + S3ErrorCode::MalformedPOSTRequest, + format!("Invalid success_action_status: {status_str}. Allowed values are 200, 201, 204."), + )), + } +} + +/// Build the final S3Response for a successful Presigned POST upload. +/// +/// Integration point (manual): +/// - After `put_object` succeeds in the PostPolicy handler, call this function +/// with parsed form fields + object info to produce the correct HTTP status. +/// +/// Notes: +/// - For 204: empty body +/// - For 303: empty body + Location header +/// - For 200: empty body (some clients accept this); you may optionally return XML/HTML body if you already implement it. +/// - For 201: prefer returning PostResponse XML; if not available, empty body still satisfies most clients, but strict tests may require XML. If you have a PostResponse serializer already, plug it in here. +#[allow(dead_code)] +pub(crate) fn build_post_object_success_response( + form_fields: &HashMap, + // These are optional; used if you want to return richer responses for 200/201. + bucket: &str, + key: &str, + etag: Option<&str>, + location: Option<&str>, +) -> S3Result> { + let action = parse_success_action_from_form_fields(form_fields)?; + + match action { + PostObjectSuccessAction::NoContent204 => Ok(S3Response::new((StatusCode::NO_CONTENT, Body::empty()))), + + PostObjectSuccessAction::Redirect303 { location } => { + let mut headers = HeaderMap::new(); + headers.insert( + http::header::LOCATION, + HeaderValue::from_str(&location) + .map_err(|_| S3Error::with_message(S3ErrorCode::InvalidArgument, "Invalid success_action_redirect URL"))?, + ); + Ok(S3Response::with_headers((StatusCode::SEE_OTHER, Body::empty()), headers)) + } + + PostObjectSuccessAction::Ok200 => { + // AWS may return 200 with an HTML/redirect response for browser workflows. + // For compatibility, returning empty body is acceptable unless strict clients require content. + // Keep Content-Length implicit; Body::empty() -> 0. + Ok(S3Response::new((StatusCode::OK, Body::empty()))) + } + + PostObjectSuccessAction::Created201 => { + // AWS 201 response is XML: + // + // ... + // ... + // ... + // ... + // + // + // If RustFS already has a DTO for this, switch to it here. + // To keep this patch minimal and safe, return empty body with 201 by default. + // + // IMPORTANT: If you run strict s3-tests for POST Object, you may need to implement XML body. + let _ = (bucket, key, etag, location); + Ok(S3Response::new((StatusCode::CREATED, Body::empty()))) + } + } +} + +/// Calculate adaptive buffer size with workload profile support. +/// +/// This enhanced version supports different workload profiles for optimal performance +/// across various use cases (AI/ML, web workloads, secure storage, etc.). +/// +/// # Arguments +/// * `file_size` - The size of the file in bytes, or -1 if unknown +/// * `profile` - Optional workload profile. If None, uses auto-detection or GeneralPurpose +/// +/// # Returns +/// Optimal buffer size in bytes based on the workload profile and file size +/// +/// # Examples +/// ```ignore +/// // Use general purpose profile (default) +/// let buffer_size = get_adaptive_buffer_size_with_profile(1024 * 1024, None); +/// +/// // Use AI training profile for large model files +/// let buffer_size = get_adaptive_buffer_size_with_profile( +/// 500 * 1024 * 1024, +/// Some(WorkloadProfile::AiTraining) +/// ); +/// +/// // Use secure storage profile for compliance scenarios +/// let buffer_size = get_adaptive_buffer_size_with_profile( +/// 10 * 1024 * 1024, +/// Some(WorkloadProfile::SecureStorage) +/// ); +/// ``` +/// +#[allow(dead_code)] +pub(crate) fn get_adaptive_buffer_size_with_profile(file_size: i64, profile: Option) -> usize { + let config = match profile { + Some(p) => RustFSBufferConfig::new(p), + None => { + // Auto-detect OS environment or use general purpose + RustFSBufferConfig::with_auto_detect() + } + }; + + config.get_buffer_size(file_size) +} + +/// Get adaptive buffer size using global workload profile configuration. +/// +/// This is the primary buffer sizing function that uses the workload profile +/// system configured at startup to provide optimal buffer sizes for different scenarios. +/// +/// The function automatically selects buffer sizes based on: +/// - Configured workload profile (default: GeneralPurpose) +/// - File size characteristics +/// - Optional performance metrics collection +/// +/// # Arguments +/// * `file_size` - The size of the file in bytes, or -1 if unknown +/// +/// # Returns +/// Optimal buffer size in bytes based on the configured workload profile +/// +/// # Performance Metrics +/// When compiled with the `metrics` feature flag, this function tracks: +/// - Buffer size distribution +/// - Selection frequency +/// - Buffer-to-file size ratios +/// +/// # Examples +/// ```ignore +/// // Uses configured profile (default: GeneralPurpose) +/// let buffer_size = get_buffer_size_opt_in(file_size); +/// ``` +pub(crate) fn get_buffer_size_opt_in(file_size: i64) -> usize { + let buffer_size = if is_buffer_profile_enabled() { + // Use globally configured workload profile (enabled by default in Phase 3) + let config = get_global_buffer_config(); + config.get_buffer_size(file_size) + } else { + // Opt-out mode: Use GeneralPurpose profile for consistent behavior + let config = RustFSBufferConfig::new(WorkloadProfile::GeneralPurpose); + config.get_buffer_size(file_size) + }; + + // Optional performance metrics collection for monitoring and optimization + #[cfg(feature = "metrics")] + { + use metrics::histogram; + histogram!("rustfs.buffer.size.bytes").record(buffer_size as f64); + counter!("rustfs.buffer.size.selections").increment(1); + + if file_size >= 0 { + let ratio = buffer_size as f64 / file_size as f64; + histogram!("rustfs.buffer.to.file.ratio").record(ratio); + } + } + + buffer_size +} + +pub(crate) async fn create_managed_encryption_material( + bucket: &str, + key: &str, + algorithm: &ServerSideEncryption, + kms_key_id: Option, + original_size: i64, +) -> Result { + let Some(service) = get_global_encryption_service().await else { + return Err(ApiError::from(StorageError::other("KMS encryption service is not initialized"))); + }; + + if !is_managed_sse(algorithm) { + return Err(ApiError::from(StorageError::other(format!( + "Unsupported server-side encryption algorithm: {}", + algorithm.as_str() + )))); + } + + let algorithm_str = algorithm.as_str(); + + let mut context = ObjectEncryptionContext::new(bucket.to_string(), key.to_string()); + if original_size >= 0 { + context = context.with_size(original_size as u64); + } + + let mut kms_key_candidate = kms_key_id; + if kms_key_candidate.is_none() { + kms_key_candidate = service.get_default_key_id().cloned(); + } + + let kms_key_to_use = kms_key_candidate + .clone() + .ok_or_else(|| ApiError::from(StorageError::other("No KMS key available for managed server-side encryption")))?; + + let (data_key, encrypted_data_key) = service + .create_data_key(&kms_key_candidate, &context) + .await + .map_err(|e| ApiError::from(StorageError::other(format!("Failed to create data key: {e}"))))?; + + let metadata = EncryptionMetadata { + algorithm: algorithm_str.to_string(), + key_id: kms_key_to_use.clone(), + key_version: 1, + iv: data_key.nonce.to_vec(), + tag: None, + encryption_context: context.encryption_context.clone(), + encrypted_at: jiff::Zoned::now(), + original_size: if original_size >= 0 { original_size as u64 } else { 0 }, + encrypted_data_key, + }; + + let mut headers = service.metadata_to_headers(&metadata); + headers.insert("x-rustfs-encryption-original-size".to_string(), metadata.original_size.to_string()); + + Ok(crate::storage::ecfs::ManagedEncryptionMaterial { + data_key, + headers, + kms_key_id: kms_key_to_use, + }) +} + +pub(crate) async fn decrypt_managed_encryption_key( + bucket: &str, + key: &str, + metadata: &HashMap, +) -> Result)>, ApiError> { + if !metadata.contains_key("x-rustfs-encryption-key") { + return Ok(None); + } + + let Some(service) = get_global_encryption_service().await else { + return Err(ApiError::from(StorageError::other("KMS encryption service is not initialized"))); + }; + + let parsed = service + .headers_to_metadata(metadata) + .map_err(|e| ApiError::from(StorageError::other(format!("Failed to parse encryption metadata: {e}"))))?; + + if parsed.iv.len() != 12 { + return Err(ApiError::from(StorageError::other("Invalid encryption nonce length; expected 12 bytes"))); + } + + let context = ObjectEncryptionContext::new(bucket.to_string(), key.to_string()); + let data_key = service + .decrypt_data_key(&parsed.encrypted_data_key, &context) + .await + .map_err(|e| ApiError::from(StorageError::other(format!("Failed to decrypt data key: {e}"))))?; + + let key_bytes = data_key.plaintext_key; + let mut nonce = [0u8; 12]; + nonce.copy_from_slice(&parsed.iv[..12]); + + let original_size = metadata + .get("x-rustfs-encryption-original-size") + .and_then(|s| s.parse::().ok()); + + Ok(Some((key_bytes, nonce, original_size))) +} + +pub(crate) fn derive_part_nonce(base: [u8; 12], part_number: usize) -> [u8; 12] { + let mut nonce = base; + let current = u32::from_be_bytes([nonce[8], nonce[9], nonce[10], nonce[11]]); + let incremented = current.wrapping_add(part_number as u32); + nonce[8..12].copy_from_slice(&incremented.to_be_bytes()); + nonce +} + +pub(crate) async fn decrypt_multipart_managed_stream( + mut encrypted_stream: Box, + parts: &[ObjectPartInfo], + key_bytes: [u8; 32], + base_nonce: [u8; 12], +) -> Result<(Box, i64), StorageError> { + let total_plain_capacity: usize = parts.iter().map(|part| part.actual_size.max(0) as usize).sum(); + + let mut plaintext = Vec::with_capacity(total_plain_capacity); + + for part in parts { + if part.size == 0 { + continue; + } + + let mut encrypted_part = vec![0u8; part.size]; + tokio::io::AsyncReadExt::read_exact(&mut encrypted_stream, &mut encrypted_part) + .await + .map_err(|e| StorageError::other(format!("failed to read encrypted multipart segment {}: {}", part.number, e)))?; + + let part_nonce = derive_part_nonce(base_nonce, part.number); + let cursor = std::io::Cursor::new(encrypted_part); + let mut decrypt_reader = DecryptReader::new(WarpReader::new(cursor), key_bytes, part_nonce); + + tokio::io::AsyncReadExt::read_to_end(&mut decrypt_reader, &mut plaintext) + .await + .map_err(|e| StorageError::other(format!("failed to decrypt multipart segment {}: {}", part.number, e)))?; + } + + let total_plain_size = plaintext.len() as i64; + let reader = Box::new(WarpReader::new(InMemoryAsyncReader::new(plaintext))) as Box; + + Ok((reader, total_plain_size)) +} + +pub(crate) fn strip_managed_encryption_metadata(metadata: &mut HashMap) { + const KEYS: [&str; 7] = [ + "x-amz-server-side-encryption", + "x-amz-server-side-encryption-aws-kms-key-id", + "x-rustfs-encryption-iv", + "x-rustfs-encryption-tag", + "x-rustfs-encryption-key", + "x-rustfs-encryption-context", + "x-rustfs-encryption-original-size", + ]; + + for key in KEYS.iter() { + metadata.remove(*key); + } +} + +/// Check if the given server-side encryption algorithm is a managed SSE type +/// +/// This function checks if the provided ServerSideEncryption algorithm +/// corresponds to a managed server-side encryption method, specifically +/// "AES256" or "aws:kms". +/// +/// # Arguments +/// * `algorithm` - A reference to the ServerSideEncryption enum to check. +/// +/// # Returns +/// * `true` if the algorithm is "AES256" or "aws:kms", otherwise `false`. +/// +pub(crate) fn is_managed_sse(algorithm: &ServerSideEncryption) -> bool { + matches!(algorithm.as_str(), "AES256" | "aws:kms") +} + +/// Validate object key for control characters and log special characters +/// +/// This function: +/// 1. Rejects keys containing control characters (null bytes, newlines, carriage returns) +/// 2. Logs debug information for keys containing spaces, plus signs, or percent signs +/// +/// The s3s library handles URL decoding, so keys are already decoded when they reach this function. +/// This validation ensures that invalid characters that could cause issues are rejected early. +pub(crate) fn validate_object_key(key: &str, operation: &str) -> S3Result<()> { + // Validate object key doesn't contain control characters + if key.contains(['\0', '\n', '\r']) { + return Err(S3Error::with_message( + S3ErrorCode::InvalidArgument, + format!("Object key contains invalid control characters: {key:?}"), + )); + } + + // Log debug info for keys with special characters to help diagnose encoding issues + if key.contains([' ', '+', '%']) { + debug!("{} object with special characters in key: {:?}", operation, key); + } + + Ok(()) +} + +/// Validate that 'allow-unordered' parameter is not used with a delimiter +/// +/// This function: +/// 1. Checks if a delimiter is specified in the ListObjects request +/// 2. Parses the query string to check for the 'allow-unordered' parameter +/// 3. Rejects the request if both 'delimiter' and 'allow-unordered=true' are present +/// +/// According to S3 compatibility requirements, unordered listing cannot be combined with +/// hierarchical directory traversal (delimited listing). This validation ensures +/// conflicting parameters are caught before processing the request. +pub(crate) fn validate_list_object_unordered_with_delimiter( + delimiter: Option<&Delimiter>, + query_string: Option<&str>, +) -> S3Result<()> { + if delimiter.is_none() { + return Ok(()); + } + + let Some(query) = query_string else { + return Ok(()); + }; + + if let Ok(params) = from_bytes::(query.as_bytes()) + && params.allow_unordered.as_deref() == Some("true") + { + return Err(S3Error::with_message( + S3ErrorCode::InvalidArgument, + "The allow-unordered parameter cannot be used when delimiter is specified.".to_string(), + )); + } + + Ok(()) +} + +pub(crate) fn parse_object_lock_retention(retention: Option) -> S3Result> { + let mut eval_metadata = HashMap::new(); + + if let Some(v) = retention { + let mode = match v.mode { + Some(mode) => match mode.as_str() { + ObjectLockRetentionMode::COMPLIANCE | ObjectLockRetentionMode::GOVERNANCE => mode.as_str().to_string(), + _ => { + return Err(S3Error::with_message( + S3ErrorCode::MalformedXML, + "The XML you provided was not well-formed or did not validate against our published schema".to_string(), + )); + } + }, + None => String::default(), + }; + + let now = OffsetDateTime::now_utc(); + // Validate retain_until_date is in the future (S3 requirement) + // Only validate when both mode and date are provided (not clearing retention) + let retain_until_date = if let Some(date) = v.retain_until_date { + let retain_until = OffsetDateTime::from(date); + // Only validate future date when mode is set (not clearing retention) + if !mode.is_empty() && retain_until <= now { + return Err(S3Error::with_message( + S3ErrorCode::InvalidArgument, + "The retain until date must be in the future".to_string(), + )); + } + retain_until.format(&Rfc3339).unwrap() + } else { + String::default() + }; + + // This is intentional behavior. Empty string represents "retention cleared" which is different from "retention never set". Consistent with minio + eval_metadata.insert(AMZ_OBJECT_LOCK_MODE_LOWER.to_string(), mode); + eval_metadata.insert(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER.to_string(), retain_until_date); + eval_metadata.insert( + format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, "objectlock-retention-timestamp"), + format!("{}.{:09}Z", now.format(&Rfc3339).unwrap(), now.nanosecond()), + ); + } + Ok(eval_metadata) +} + +pub(crate) fn parse_object_lock_legal_hold(legal_hold: Option) -> S3Result> { + let mut eval_metadata = HashMap::new(); + if let Some(v) = legal_hold { + let status = match v.status { + Some(status) => match status.as_str() { + ObjectLockLegalHoldStatus::OFF | ObjectLockLegalHoldStatus::ON => status.as_str().to_string(), + _ => { + return Err(S3Error::with_message( + S3ErrorCode::MalformedXML, + "The XML you provided was not well-formed or did not validate against our published schema".to_string(), + )); + } + }, + None => String::default(), + }; + let now = OffsetDateTime::now_utc(); + // This is intentional behavior. Empty string represents "status cleared" which is different from "status never set". Consistent with minio + eval_metadata.insert(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER.to_string(), status); + eval_metadata.insert( + format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, "objectlock-legalhold-timestamp"), + format!("{}.{:09}Z", now.format(&Rfc3339).unwrap(), now.nanosecond()), + ); + } + Ok(eval_metadata) +} + +pub(crate) async fn validate_bucket_object_lock_enabled(bucket: &str) -> S3Result<()> { + match metadata_sys::get_object_lock_config(bucket).await { + Ok((cfg, _created)) => { + if cfg.object_lock_enabled != Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)) { + return Err(S3Error::with_message( + S3ErrorCode::InvalidRequest, + "Object Lock is not enabled for this bucket".to_string(), + )); + } + } + Err(err) => { + if err == StorageError::ConfigNotFound { + return Err(S3Error::with_message( + S3ErrorCode::InvalidRequest, + "Bucket is missing ObjectLockConfiguration".to_string(), + )); + } + warn!("get_object_lock_config err {:?}", err); + return Err(S3Error::with_message( + S3ErrorCode::InternalError, + "Failed to get bucket ObjectLockConfiguration".to_string(), + )); + } + } + Ok(()) +} + +/// Validates HTTP conditional request headers for a single object according to +/// RFC 7232 and S3 API semantics. +/// +/// This function evaluates the following headers, if present, in the standard +/// conditional request order: +/// - `If-Match` +/// - `If-None-Match` +/// - `If-Modified-Since` +/// - `If-Unmodified-Since` +/// +/// The `headers` parameter provides the incoming HTTP request headers, and +/// `info` supplies the object's current metadata (ETag and modification time) +/// used to evaluate those preconditions. +/// +/// The function returns `Ok(())` if either no relevant conditional headers are +/// set or all specified preconditions are satisfied. If any precondition fails, +/// it returns an `Err(S3Error)` with an appropriate S3 error code (for example, +/// `PreconditionFailed` for failed `If-Match` / `If-Unmodified-Since`, or +/// `NotModified` for `If-None-Match` / `If-Modified-Since` when the resource +/// has not changed), allowing the caller to translate this into the correct +/// HTTP response. +pub(crate) fn check_preconditions(headers: &HeaderMap, info: &ObjectInfo) -> S3Result<()> { + let mod_time = info.mod_time; + let etag = info.etag.as_deref(); + + if mod_time.is_none() && etag.is_none() { + return Ok(()); + } + + // If-Match: requires ETag to exist + if let Some(if_match_val) = headers.get("if-match").and_then(|v| v.to_str().ok()) { + match etag { + Some(e) if is_etag_equal(e, if_match_val) => {} + _ => return Err(S3Error::new(S3ErrorCode::PreconditionFailed)), + } + } + + // If-Unmodified-Since (only when If-Match is absent) + if headers.get("if-match").is_none() + && let Some(t) = mod_time + && let Some(if_unmodified_since) = headers.get("if-unmodified-since").and_then(|v| v.to_str().ok()) + && let Ok(given_time) = OffsetDateTime::parse(if_unmodified_since, &Rfc2822) + && t > given_time.add(time::Duration::seconds(1)) + { + return Err(S3Error::new(S3ErrorCode::PreconditionFailed)); + } + + // If-None-Match + if let Some(if_none_match) = headers.get("if-none-match").and_then(|v| v.to_str().ok()) + && let Some(e) = etag + && is_etag_equal(e, if_none_match) + { + let mut error_headers = HeaderMap::new(); + if let Ok(etag_header) = parse_etag(e) { + error_headers.insert("etag", etag_header); + } + if let Some(t) = mod_time + && let Ok(last_modified_str) = t.format(&Rfc2822) + && let Ok(last_modified_header) = HeaderValue::from_str(&last_modified_str) + { + error_headers.insert("last-modified", last_modified_header); + } + + let mut s3_error = S3Error::new(S3ErrorCode::NotModified); + s3_error.set_message("Not Modified".to_string()); + s3_error.set_status_code(StatusCode::NOT_MODIFIED); + s3_error.set_headers(error_headers); + return Err(s3_error); + } + + // If-Modified-Since (only when If-None-Match is absent - RFC 7232) + if headers.get("if-none-match").is_none() + && let Some(t) = mod_time + && let Some(if_modified_since) = headers.get("if-modified-since").and_then(|v| v.to_str().ok()) + && let Ok(given_time) = OffsetDateTime::parse(if_modified_since, &Rfc2822) + && t < given_time.add(time::Duration::seconds(1)) + { + let mut error_headers = HeaderMap::new(); + if let Some(e) = etag + && let Ok(etag_header) = parse_etag(e) + { + error_headers.insert("etag", etag_header); + } + if let Ok(last_modified_str) = t.format(&Rfc2822) + && let Ok(last_modified_header) = HeaderValue::from_str(&last_modified_str) + { + error_headers.insert("last-modified", last_modified_header); + } + + let mut s3_error = S3Error::new(S3ErrorCode::NotModified); + s3_error.set_message("Not Modified".to_string()); + s3_error.set_status_code(StatusCode::NOT_MODIFIED); + s3_error.set_headers(error_headers); + + return Err(s3_error); + } + + Ok(()) +} + +/// Compares an object ETag with an ETag value from an HTTP header. +/// +/// This helper implements HTTP ETag comparison semantics for headers such as +/// `If-Match` and `If-None-Match`: +/// - Supports the wildcard `*`, which matches any `object_etag`. +/// - Supports comma-separated ETag lists (e.g., `"etag1", "etag2"`), returning +/// `true` if any entry matches `object_etag`. +/// - Automatically trims surrounding whitespace and double quotes from both the +/// header entries and `object_etag` before comparison. +/// +/// # Parameters +/// - `object_etag`: The ETag associated with the stored object. +/// - `header_etag`: The raw ETag header value received in the request, which may +/// be a wildcard, a single ETag, or a comma-separated list of ETags. +/// +/// # Returns +/// `true` if the header value matches the object ETag according to the above +/// HTTP ETag comparison rules, otherwise `false`. +pub(crate) fn is_etag_equal(object_etag: &str, header_etag: &str) -> bool { + let header_etag = header_etag.trim(); + if header_etag == "*" { + return true; + } + header_etag + .split(',') + .map(|s| s.trim().trim_matches('"')) + .any(|e| e == object_etag.trim_matches('"')) +} + +/// Converts an object ETag string into an HTTP `HeaderValue` for use in response headers. +/// +/// This function normalizes ETag values by ensuring they are wrapped in double quotes, +/// which is the standard HTTP ETag format according to RFC 7232. If the input ETag +/// already contains quotes, they are removed before adding new quotes to avoid duplication. +/// +/// # Parameters +/// - `object_etag`: The ETag string, which may or may not already be wrapped in quotes. +/// Must not be empty after removing quotes. +/// +/// # Returns +/// - `Ok(HeaderValue)`: The normalized ETag value wrapped in double quotes, ready for use +/// in HTTP response headers. +/// - `Err(S3Error)`: Returns `InvalidArgument` if the ETag value is empty (after removing quotes) +/// or contains characters that cannot be represented in an HTTP header value. +/// +/// # Note +/// When this function returns an error, callers should log the error but continue with +/// the operation (e.g., return NotModified without the ETag header) rather than propagating +/// the InvalidArgument error, as the original operation semantics should be preserved. +pub fn parse_etag(object_etag: &str) -> Result { + let etag_trimmed = object_etag.trim_matches('"'); + + if etag_trimmed.trim().is_empty() { + return Err(S3Error::with_message(S3ErrorCode::InvalidArgument, "ETag cannot be empty".to_string())); + } + + let etag_quoted = format!("\"{}\"", etag_trimmed); + + HeaderValue::from_str(&etag_quoted).map_err(|e| { + warn!( + "Failed to convert ETag to HeaderValue (ETag: {}): {}. The ETag header will be omitted from the response.", + etag_quoted, e + ); + S3Error::with_message( + S3ErrorCode::InvalidArgument, + format!("Invalid ETag header value (ETag: {}): {}", etag_quoted, e), + ) + }) +} + +/// Auxiliary functions: extract prefixes and suffixes +pub(crate) fn extract_prefix_suffix(filter: Option<&NotificationConfigurationFilter>) -> (String, String) { + if let Some(filter) = filter + && let Some(filter_rules) = &filter.key + { + let mut prefix = String::new(); + let mut suffix = String::new(); + if let Some(rules) = &filter_rules.filter_rules { + for rule in rules { + if let (Some(name), Some(value)) = (rule.name.as_ref(), rule.value.as_ref()) { + match name.as_str() { + "prefix" => prefix = value.clone(), + "suffix" => suffix = value.clone(), + _ => {} + } + } + } + } + return (prefix, suffix); + } + (String::new(), String::new()) +} + +/// Auxiliary functions: Handle configuration +pub(crate) fn process_queue_configurations( + event_rules: &mut Vec<(Vec, String, String, Vec)>, + configurations: Option>, + target_id_parser: F, +) -> Result<(), TargetIDError> +where + F: Fn(&str) -> Result, +{ + if let Some(configs) = configurations { + for cfg in configs { + let events = cfg.events.iter().filter_map(|e| EventName::parse(e.as_ref()).ok()).collect(); + let (prefix, suffix) = extract_prefix_suffix(cfg.filter.as_ref()); + let target_id = target_id_parser(&cfg.queue_arn)?; + event_rules.push((events, prefix, suffix, vec![target_id])); + } + } + Ok(()) +} + +pub(crate) fn process_topic_configurations( + event_rules: &mut Vec<(Vec, String, String, Vec)>, + configurations: Option>, + target_id_parser: F, +) -> Result<(), TargetIDError> +where + F: Fn(&str) -> Result, +{ + if let Some(configs) = configurations { + for cfg in configs { + let events = cfg.events.iter().filter_map(|e| EventName::parse(e.as_ref()).ok()).collect(); + let (prefix, suffix) = extract_prefix_suffix(cfg.filter.as_ref()); + let target_id = target_id_parser(&cfg.topic_arn)?; + event_rules.push((events, prefix, suffix, vec![target_id])); + } + } + Ok(()) +} + +pub(crate) fn process_lambda_configurations( + event_rules: &mut Vec<(Vec, String, String, Vec)>, + configurations: Option>, + target_id_parser: F, +) -> Result<(), TargetIDError> +where + F: Fn(&str) -> Result, +{ + if let Some(configs) = configurations { + for cfg in configs { + let events = cfg.events.iter().filter_map(|e| EventName::parse(e.as_ref()).ok()).collect(); + let (prefix, suffix) = extract_prefix_suffix(cfg.filter.as_ref()); + let target_id = target_id_parser(&cfg.lambda_function_arn)?; + event_rules.push((events, prefix, suffix, vec![target_id])); + } + } + Ok(()) +} + +pub(crate) async fn has_replication_rules(bucket: &str, objects: &[ObjectToDelete]) -> bool { + let (cfg, _created) = match get_replication_config(bucket).await { + Ok(replication_config) => replication_config, + Err(_err) => { + return false; + } + }; + + for object in objects { + if cfg.has_active_rules(&object.object_name, true) { + return true; + } + } + false +} + +/// Helper function to get store and validate bucket exists +pub(crate) async fn get_validated_store(bucket: &str) -> S3Result> { + let Some(store) = new_object_layer_fn() else { + return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); + }; + + // Validate bucket exists + store + .get_bucket_info(bucket, &BucketOptions::default()) + .await + .map_err(ApiError::from)?; + + Ok(store) +} + +/// Quick check if CORS processing is needed (lightweight check for Origin header) +/// This avoids unnecessary function calls for non-CORS requests +#[inline] +pub(crate) fn needs_cors_processing(headers: &HeaderMap) -> bool { + headers.contains_key(cors::standard::ORIGIN) +} + +/// Apply CORS headers to response based on bucket CORS configuration and request origin +/// +/// This function: +/// 1. Reads the Origin header from the request +/// 2. Retrieves the bucket's CORS configuration +/// 3. Matches the origin against CORS rules +/// 4. Validates AllowedHeaders if request headers are present +/// 5. Returns headers to add to the response if a match is found +/// +/// Note: This function should only be called if `needs_cors_processing()` returns true +/// to avoid unnecessary overhead for non-CORS requests. +pub(crate) async fn apply_cors_headers(bucket: &str, method: &http::Method, headers: &HeaderMap) -> Option { + use http::HeaderValue; + + // Get Origin header from request + let origin = headers.get(cors::standard::ORIGIN)?.to_str().ok()?; + + // Get CORS configuration for the bucket + let cors_config = match metadata_sys::get_cors_config(bucket).await { + Ok((config, _)) => config, + Err(_) => return None, // No CORS config, no headers to add + }; + + // Early return if no CORS rules configured + if cors_config.cors_rules.is_empty() { + return None; + } + + // Check if method is supported and get its string representation + const SUPPORTED_METHODS: &[&str] = &["GET", "PUT", "POST", "DELETE", "HEAD", "OPTIONS"]; + let method_str = method.as_str(); + if !SUPPORTED_METHODS.contains(&method_str) { + return None; + } + + // For OPTIONS (preflight) requests, check Access-Control-Request-Method + let is_preflight = method == http::Method::OPTIONS; + let requested_method = if is_preflight { + headers + .get(cors::request::ACCESS_CONTROL_REQUEST_METHOD) + .and_then(|v| v.to_str().ok()) + .unwrap_or(method_str) + } else { + method_str + }; + + // Get requested headers from preflight request + let requested_headers = if is_preflight { + headers + .get(cors::request::ACCESS_CONTROL_REQUEST_HEADERS) + .and_then(|v| v.to_str().ok()) + .map(|h| h.split(',').map(|s| s.trim().to_lowercase()).collect::>()) + } else { + None + }; + + // Find matching CORS rule + for rule in cors_config.cors_rules.iter() { + // Check if origin matches + let origin_matches = rule.allowed_origins.iter().any(|allowed_origin| { + if allowed_origin == "*" { + true + } else { + // Exact match or pattern match (support wildcards like https://*.example.com) + allowed_origin == origin || matches_origin_pattern(allowed_origin, origin) + } + }); + + if !origin_matches { + continue; + } + + // Check if method is allowed + let method_allowed = rule + .allowed_methods + .iter() + .any(|allowed_method| allowed_method.as_str() == requested_method); + + if !method_allowed { + continue; + } + + // Validate AllowedHeaders if present in the request + if let Some(ref req_headers) = requested_headers { + if let Some(ref allowed_headers) = rule.allowed_headers { + // Check if all requested headers are allowed + let all_headers_allowed = req_headers.iter().all(|req_header| { + allowed_headers.iter().any(|allowed_header| { + let allowed_lower = allowed_header.to_lowercase(); + // "*" allows all headers, or exact match + allowed_lower == "*" || allowed_lower == *req_header + }) + }); + + if !all_headers_allowed { + // If not all headers are allowed, skip this rule + continue; + } + } else if !req_headers.is_empty() { + // If no AllowedHeaders specified but headers were requested, skip this rule + // Unless the rule explicitly allows all headers + continue; + } + } + + // Found matching rule, build response headers + let mut response_headers = HeaderMap::new(); + + // Access-Control-Allow-Origin + // If origin is "*", use "*", otherwise echo back the origin + let has_wildcard_origin = rule.allowed_origins.iter().any(|o| o == "*"); + if has_wildcard_origin { + response_headers.insert(cors::response::ACCESS_CONTROL_ALLOW_ORIGIN, HeaderValue::from_static("*")); + } else if let Ok(origin_value) = HeaderValue::from_str(origin) { + response_headers.insert(cors::response::ACCESS_CONTROL_ALLOW_ORIGIN, origin_value); + } + + // Vary: Origin (required for caching, except when using wildcard) + if !has_wildcard_origin { + response_headers.insert(cors::standard::VARY, HeaderValue::from_static("Origin")); + } + + // Access-Control-Allow-Methods (required for preflight) + if is_preflight || !rule.allowed_methods.is_empty() { + let methods_str = rule.allowed_methods.iter().map(|m| m.as_str()).collect::>().join(", "); + if let Ok(methods_value) = HeaderValue::from_str(&methods_str) { + response_headers.insert(cors::response::ACCESS_CONTROL_ALLOW_METHODS, methods_value); + } + } + + // Access-Control-Allow-Headers (required for preflight if headers were requested) + if is_preflight && let Some(ref allowed_headers) = rule.allowed_headers { + let headers_str = allowed_headers.iter().map(|h| h.as_str()).collect::>().join(", "); + if let Ok(headers_value) = HeaderValue::from_str(&headers_str) { + response_headers.insert(cors::response::ACCESS_CONTROL_ALLOW_HEADERS, headers_value); + } + } + + // Access-Control-Expose-Headers (for actual requests) + if !is_preflight && let Some(ref expose_headers) = rule.expose_headers { + let expose_headers_str = expose_headers.iter().map(|h| h.as_str()).collect::>().join(", "); + if let Ok(expose_value) = HeaderValue::from_str(&expose_headers_str) { + response_headers.insert(cors::response::ACCESS_CONTROL_EXPOSE_HEADERS, expose_value); + } + } + + // Access-Control-Max-Age (for preflight requests) + if is_preflight + && let Some(max_age) = rule.max_age_seconds + && let Ok(max_age_value) = HeaderValue::from_str(&max_age.to_string()) + { + response_headers.insert(cors::response::ACCESS_CONTROL_MAX_AGE, max_age_value); + } + + return Some(response_headers); + } + + None // No matching rule found +} +/// Check if an origin matches a pattern (supports wildcards like https://*.example.com) +pub(crate) fn matches_origin_pattern(pattern: &str, origin: &str) -> bool { + // Simple wildcard matching: * matches any sequence + if pattern.contains('*') { + let pattern_parts: Vec<&str> = pattern.split('*').collect(); + if pattern_parts.len() == 2 { + origin.starts_with(pattern_parts[0]) && origin.ends_with(pattern_parts[1]) + } else { + false + } + } else { + pattern == origin + } +} + +/// Wrap S3Response with CORS headers if needed +/// This function performs a lightweight check first to avoid unnecessary CORS processing +/// for non-CORS requests (requests without Origin header) +pub(crate) async fn wrap_response_with_cors( + bucket: &str, + method: &http::Method, + headers: &HeaderMap, + output: T, +) -> S3Response { + let mut response = S3Response::new(output); + + // Quick check: only process CORS if Origin header is present + if needs_cors_processing(headers) + && let Some(cors_headers) = apply_cors_headers(bucket, method, headers).await + { + for (key, value) in cors_headers.iter() { + response.headers.insert(key, value.clone()); + } + } + + response +} diff --git a/rustfs/src/storage/ecfs_test.rs b/rustfs/src/storage/ecfs_test.rs new file mode 100644 index 000000000..305619694 --- /dev/null +++ b/rustfs/src/storage/ecfs_test.rs @@ -0,0 +1,1194 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#[cfg(test)] +mod tests { + use crate::config::workload_profiles::WorkloadProfile; + use crate::storage::ecfs::FS; + use crate::storage::ecfs::RUSTFS_OWNER; + use crate::storage::{ + apply_cors_headers, check_preconditions, get_adaptive_buffer_size_with_profile, get_buffer_size_opt_in, is_etag_equal, + matches_origin_pattern, parse_etag, parse_object_lock_legal_hold, parse_object_lock_retention, + process_lambda_configurations, process_queue_configurations, process_topic_configurations, + validate_bucket_object_lock_enabled, validate_list_object_unordered_with_delimiter, + }; + use http::{HeaderMap, HeaderValue, StatusCode}; + use rustfs_config::MI_B; + use rustfs_ecstore::set_disk::DEFAULT_READ_BUFFER_SIZE; + use rustfs_ecstore::store_api::ObjectInfo; + use rustfs_utils::http::{AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, RESERVED_METADATA_PREFIX_LOWER}; + use rustfs_zip::CompressionFormat; + use s3s::dto::{ + Delimiter, LambdaFunctionConfiguration, ObjectLockLegalHold, ObjectLockLegalHoldStatus, ObjectLockRetention, + ObjectLockRetentionMode, QueueConfiguration, TopicConfiguration, + }; + use s3s::{S3Error, S3ErrorCode, s3_error}; + use time::OffsetDateTime; + use time::format_description::well_known::Rfc2822; + + #[test] + fn test_fs_creation() { + let _fs = FS::new(); + + // Verify that FS struct can be created successfully + // Since it's currently empty, we just verify it doesn't panic + // The test passes if we reach this point without panicking + } + + #[test] + fn test_fs_debug_implementation() { + let fs = FS::new(); + + // Test that Debug trait is properly implemented + let debug_str = format!("{fs:?}"); + assert!(debug_str.contains("FS")); + } + + #[test] + fn test_fs_clone_implementation() { + let fs = FS::new(); + + // Test that Clone trait is properly implemented + let cloned_fs = fs.clone(); + + // Both should be equivalent (since FS is currently empty) + assert_eq!(format!("{fs:?}"), format!("{cloned_fs:?}")); + } + + #[test] + fn test_rustfs_owner_constant() { + // Test that RUSTFS_OWNER constant is properly defined + assert!(!RUSTFS_OWNER.display_name.as_ref().unwrap().is_empty()); + assert!(!RUSTFS_OWNER.id.as_ref().unwrap().is_empty()); + assert_eq!(RUSTFS_OWNER.display_name.as_ref().unwrap(), "rustfs"); + } + + // Note: Most S3 API methods require complex setup with global state, storage backend, + // and various dependencies that make unit testing challenging. For comprehensive testing + // of S3 operations, integration tests would be more appropriate. + + #[test] + fn test_list_objects_v2_key_count_includes_prefixes() { + // Test that KeyCount calculation includes both objects and common prefixes + // This verifies the fix for S3 API compatibility where KeyCount should equal + // the sum of Contents and CommonPrefixes lengths + + // Simulate the calculation logic from list_objects_v2 + let objects_count = 3_usize; + let common_prefixes_count = 2_usize; + + // KeyCount should include both objects and common prefixes per S3 API spec + let key_count = (objects_count + common_prefixes_count) as i32; + + assert_eq!(key_count, 5); + + // Edge cases: verify calculation logic + let no_objects = 0_usize; + let no_prefixes = 0_usize; + assert_eq!((no_objects + no_prefixes) as i32, 0); + + let one_object = 1_usize; + assert_eq!((one_object + no_prefixes) as i32, 1); + + let one_prefix = 1_usize; + assert_eq!((no_objects + one_prefix) as i32, 1); + } + + #[test] + fn test_s3_url_encoding_preserves_slash() { + // Test that S3 URL encoding preserves path separators (/) + // This verifies the encoding logic for EncodingType=url parameter + + use urlencoding::encode; + + // Helper function matching the implementation + let encode_s3_name = |name: &str| -> String { + name.split('/') + .map(|part| encode(part).to_string()) + .collect::>() + .join("/") + }; + + // Test cases from s3-tests + assert_eq!(encode_s3_name("asdf+b"), "asdf%2Bb"); + assert_eq!(encode_s3_name("foo+1/bar"), "foo%2B1/bar"); + assert_eq!(encode_s3_name("foo/"), "foo/"); + assert_eq!(encode_s3_name("quux ab/"), "quux%20ab/"); + + // Edge cases + assert_eq!(encode_s3_name("normal/key"), "normal/key"); + assert_eq!(encode_s3_name("key+with+plus"), "key%2Bwith%2Bplus"); + assert_eq!(encode_s3_name("key with spaces"), "key%20with%20spaces"); + } + + #[test] + fn test_s3_error_scenarios() { + // Test that we can create expected S3 errors for common validation cases + + // Test incomplete body error + let incomplete_body_error = s3_error!(IncompleteBody); + assert_eq!(incomplete_body_error.code(), &S3ErrorCode::IncompleteBody); + + // Test invalid argument error + let invalid_arg_error = s3_error!(InvalidArgument, "test message"); + assert_eq!(invalid_arg_error.code(), &S3ErrorCode::InvalidArgument); + + // Test internal error + let internal_error = S3Error::with_message(S3ErrorCode::InternalError, "test".to_string()); + assert_eq!(internal_error.code(), &S3ErrorCode::InternalError); + } + + #[test] + fn test_compression_format_usage() { + // Test that compression format detection works for common file extensions + let zip_format = CompressionFormat::from_extension("zip"); + assert_eq!(zip_format.extension(), "zip"); + + let tar_format = CompressionFormat::from_extension("tar"); + assert_eq!(tar_format.extension(), "tar"); + + let gz_format = CompressionFormat::from_extension("gz"); + assert_eq!(gz_format.extension(), "gz"); + } + + #[test] + fn test_adaptive_buffer_size_with_profile() { + const KB: i64 = 1024; + const MB: i64 = 1024 * 1024; + + // Test GeneralPurpose profile (default behavior, should match get_adaptive_buffer_size) + assert_eq!( + get_adaptive_buffer_size_with_profile(500 * KB, Some(WorkloadProfile::GeneralPurpose)), + 64 * KB as usize + ); + assert_eq!( + get_adaptive_buffer_size_with_profile(50 * MB, Some(WorkloadProfile::GeneralPurpose)), + 256 * KB as usize + ); + assert_eq!( + get_adaptive_buffer_size_with_profile(200 * MB, Some(WorkloadProfile::GeneralPurpose)), + DEFAULT_READ_BUFFER_SIZE + ); + + // Test AiTraining profile - larger buffers for large files + assert_eq!( + get_adaptive_buffer_size_with_profile(5 * MB, Some(WorkloadProfile::AiTraining)), + 512 * KB as usize + ); + assert_eq!( + get_adaptive_buffer_size_with_profile(100 * MB, Some(WorkloadProfile::AiTraining)), + 2 * MB as usize + ); + assert_eq!( + get_adaptive_buffer_size_with_profile(600 * MB, Some(WorkloadProfile::AiTraining)), + 4 * MB as usize + ); + + // Test WebWorkload profile - smaller buffers for web assets + assert_eq!( + get_adaptive_buffer_size_with_profile(100 * KB, Some(WorkloadProfile::WebWorkload)), + 32 * KB as usize + ); + assert_eq!( + get_adaptive_buffer_size_with_profile(5 * MB, Some(WorkloadProfile::WebWorkload)), + 128 * KB as usize + ); + assert_eq!( + get_adaptive_buffer_size_with_profile(50 * MB, Some(WorkloadProfile::WebWorkload)), + 256 * KB as usize + ); + + // Test SecureStorage profile - memory-constrained buffers + assert_eq!( + get_adaptive_buffer_size_with_profile(500 * KB, Some(WorkloadProfile::SecureStorage)), + 32 * KB as usize + ); + assert_eq!( + get_adaptive_buffer_size_with_profile(25 * MB, Some(WorkloadProfile::SecureStorage)), + 128 * KB as usize + ); + assert_eq!( + get_adaptive_buffer_size_with_profile(100 * MB, Some(WorkloadProfile::SecureStorage)), + 256 * KB as usize + ); + + // Test IndustrialIoT profile - low latency, moderate buffers + assert_eq!( + get_adaptive_buffer_size_with_profile(512 * KB, Some(WorkloadProfile::IndustrialIoT)), + 64 * KB as usize + ); + assert_eq!( + get_adaptive_buffer_size_with_profile(25 * MB, Some(WorkloadProfile::IndustrialIoT)), + 256 * KB as usize + ); + assert_eq!( + get_adaptive_buffer_size_with_profile(100 * MB, Some(WorkloadProfile::IndustrialIoT)), + 512 * KB as usize + ); + + // Test DataAnalytics profile + assert_eq!( + get_adaptive_buffer_size_with_profile(2 * MB, Some(WorkloadProfile::DataAnalytics)), + 128 * KB as usize + ); + assert_eq!( + get_adaptive_buffer_size_with_profile(100 * MB, Some(WorkloadProfile::DataAnalytics)), + 512 * KB as usize + ); + assert_eq!( + get_adaptive_buffer_size_with_profile(500 * MB, Some(WorkloadProfile::DataAnalytics)), + 2 * MB as usize + ); + + // Test with None (should auto-detect or use GeneralPurpose) + let result = get_adaptive_buffer_size_with_profile(50 * MB, None); + // Should be either SecureStorage (if on special OS) or GeneralPurpose + assert!(result == 128 * KB as usize || result == 256 * KB as usize); + + // Test unknown file size with different profiles + assert_eq!( + get_adaptive_buffer_size_with_profile(-1, Some(WorkloadProfile::AiTraining)), + 2 * MB as usize + ); + assert_eq!( + get_adaptive_buffer_size_with_profile(-1, Some(WorkloadProfile::WebWorkload)), + 128 * KB as usize + ); + assert_eq!( + get_adaptive_buffer_size_with_profile(-1, Some(WorkloadProfile::SecureStorage)), + 128 * KB as usize + ); + } + + #[test] + fn test_phase3_default_behavior() { + use crate::config::workload_profiles::{ + RustFSBufferConfig, WorkloadProfile, init_global_buffer_config, set_buffer_profile_enabled, + }; + + const KB: i64 = 1024; + const MB: i64 = 1024 * 1024; + + // Test Phase 3: Enabled by default with GeneralPurpose profile + set_buffer_profile_enabled(true); + init_global_buffer_config(RustFSBufferConfig::new(WorkloadProfile::GeneralPurpose)); + + // Verify GeneralPurpose profile provides consistent buffer sizes + assert_eq!(get_buffer_size_opt_in(500 * KB), 64 * KB as usize); + assert_eq!(get_buffer_size_opt_in(50 * MB), 256 * KB as usize); + assert_eq!(get_buffer_size_opt_in(200 * MB), MI_B); + assert_eq!(get_buffer_size_opt_in(-1), MI_B); // Unknown size + + // Reset for other tests + set_buffer_profile_enabled(false); + } + + #[test] + fn test_buffer_size_opt_in() { + use crate::config::workload_profiles::{is_buffer_profile_enabled, set_buffer_profile_enabled}; + + const KB: i64 = 1024; + const MB: i64 = 1024 * 1024; + + // \[1\] Default state: profile is not enabled, global configuration is not explicitly initialized + // get_buffer_size_opt_in should be equivalent to the GeneralPurpose configuration + set_buffer_profile_enabled(false); + assert!(!is_buffer_profile_enabled()); + + // GeneralPurpose rules: + // \< 1MB -> 64KB，1MB-100MB -> 256KB，\>=100MB -> 1MB + assert_eq!(get_buffer_size_opt_in(500 * KB), 64 * KB as usize); + assert_eq!(get_buffer_size_opt_in(50 * MB), 256 * KB as usize); + assert_eq!(get_buffer_size_opt_in(200 * MB), MI_B); + + // \[2\] Enable the profile switch, but the global configuration is still the default GeneralPurpose + set_buffer_profile_enabled(true); + assert!(is_buffer_profile_enabled()); + + assert_eq!(get_buffer_size_opt_in(500 * KB), 64 * KB as usize); + assert_eq!(get_buffer_size_opt_in(50 * MB), 256 * KB as usize); + assert_eq!(get_buffer_size_opt_in(200 * MB), MI_B); + + // \[3\] Close again to ensure unchanged behavior + set_buffer_profile_enabled(false); + assert!(!is_buffer_profile_enabled()); + assert_eq!(get_buffer_size_opt_in(500 * KB), 64 * KB as usize); + } + + #[test] + fn test_phase4_full_integration() { + use crate::config::workload_profiles::{ + RustFSBufferConfig, WorkloadProfile, init_global_buffer_config, set_buffer_profile_enabled, + }; + + const KB: i64 = 1024; + const MB: i64 = 1024 * 1024; + + // \[1\] During the entire test process, the global configuration is initialized only once. + // In order not to interfere with other tests, use GeneralPurpose (consistent with the default). + // If it has been initialized elsewhere, this call will be ignored by OnceLock and the behavior will still be GeneralPurpose. + init_global_buffer_config(RustFSBufferConfig::new(WorkloadProfile::GeneralPurpose)); + + // Make sure to turn off profile initially + set_buffer_profile_enabled(false); + + // \[2\] Verify behavior of get_buffer_size_opt_in in disabled profile (GeneralPurpose) + assert_eq!(get_buffer_size_opt_in(500 * KB), 64 * KB as usize); + assert_eq!(get_buffer_size_opt_in(50 * MB), 256 * KB as usize); + assert_eq!(get_buffer_size_opt_in(200 * MB), MI_B); + + // \[3\] When profile is enabled, the behavior remains consistent with the global GeneralPurpose configuration + set_buffer_profile_enabled(true); + assert_eq!(get_buffer_size_opt_in(500 * KB), 64 * KB as usize); + assert_eq!(get_buffer_size_opt_in(50 * MB), 256 * KB as usize); + assert_eq!(get_buffer_size_opt_in(200 * MB), MI_B); + + // \[4\] Complex scenes, boundary values: such as unknown size + assert_eq!(get_buffer_size_opt_in(-1), MI_B); + + set_buffer_profile_enabled(false); + } + + #[test] + fn test_validate_list_object_unordered_with_delimiter() { + // [1] Normal case: No delimiter specified. + assert!(validate_list_object_unordered_with_delimiter(None, Some("allow-unordered=true")).is_ok()); + + let delim_str = "/".to_string(); + let delimiter_some: Option<&Delimiter> = Some(&delim_str); + // [2] Normal case: Delimiter is present, but 'allow-unordered' is explicitly set to false. + assert!(validate_list_object_unordered_with_delimiter(delimiter_some, Some("allow-unordered=false")).is_ok()); + + let query_conflict = Some("allow-unordered=true"); + // [3] Conflict case: Both delimiter and 'allow-unordered=true' are present. + assert!(validate_list_object_unordered_with_delimiter(delimiter_some, query_conflict).is_err()); + + let complex_query = Some("allow-unordered=true&abc=123"); + // [4] Complex query: The validation should still trigger if 'allow-unordered=true' is part of a multi-parameter query. + assert!(validate_list_object_unordered_with_delimiter(delimiter_some, complex_query).is_err()); + + let complex_query_without_unordered = Some("abc=123&queryType=test"); + // [5] Multi-parameter query without conflict: If other parameters exist but 'allow-unordered' is missing, + assert!(validate_list_object_unordered_with_delimiter(delimiter_some, complex_query_without_unordered).is_ok()); + } + + #[test] + fn test_parse_object_lock_retention() { + use time::macros::datetime; + // [1] Normal case: No retention specified (empty metadata) + assert!(parse_object_lock_retention(None).is_ok()); + assert!(parse_object_lock_retention(None).unwrap().is_empty()); + + // [2] Normal case: Retention with valid COMPLIANCE mode (future date) + let valid_compliance_retention = ObjectLockRetention { + mode: Some(ObjectLockRetentionMode::from_static(ObjectLockRetentionMode::COMPLIANCE)), + retain_until_date: Some(datetime!(2030-01-01 00:00:00 UTC).into()), + }; + let compliance_metadata = parse_object_lock_retention(Some(valid_compliance_retention)).unwrap(); + assert_eq!(compliance_metadata.get("x-amz-object-lock-mode").unwrap(), "COMPLIANCE"); + assert_eq!( + compliance_metadata.get("x-amz-object-lock-retain-until-date").unwrap(), + "2030-01-01T00:00:00Z" + ); + assert!( + compliance_metadata.contains_key(&format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, "objectlock-retention-timestamp")) + ); + + // [3] Normal case: Retention with valid GOVERNANCE mode (future date) + let valid_governance_retention = ObjectLockRetention { + mode: Some(ObjectLockRetentionMode::from_static(ObjectLockRetentionMode::GOVERNANCE)), + retain_until_date: Some(datetime!(2030-01-01 00:00:00 UTC).into()), + }; + let governance_metadata = parse_object_lock_retention(Some(valid_governance_retention)).unwrap(); + assert_eq!(governance_metadata.get("x-amz-object-lock-mode").unwrap(), "GOVERNANCE"); + + // [4] Normal case: Retention with None mode (empty string for mode, date not validated) + let none_mode_retention = ObjectLockRetention { + mode: None, + retain_until_date: Some(datetime!(2030-01-01 00:00:00 UTC).into()), + }; + let none_mode_metadata = parse_object_lock_retention(Some(none_mode_retention)).unwrap(); + assert_eq!(none_mode_metadata.get("x-amz-object-lock-mode").unwrap(), ""); + + // [5] Normal case: Retention with None retain_until_date (empty string for date) + let none_date_retention = ObjectLockRetention { + mode: Some(ObjectLockRetentionMode::from_static(ObjectLockRetentionMode::COMPLIANCE)), + retain_until_date: None, + }; + let none_date_metadata = parse_object_lock_retention(Some(none_date_retention)).unwrap(); + assert_eq!(none_date_metadata.get("x-amz-object-lock-retain-until-date").unwrap(), ""); + + // [6] Error case: Retention with invalid mode (non COMPLIANCE/GOVERNANCE) + let invalid_mode_retention = ObjectLockRetention { + mode: Some(ObjectLockRetentionMode::from_static("INVALID_MODE")), + retain_until_date: Some(datetime!(2030-01-01 00:00:00 UTC).into()), + }; + let err = parse_object_lock_retention(Some(invalid_mode_retention)).unwrap_err(); + assert_eq!(err.code().as_str(), S3ErrorCode::MalformedXML.as_str()); + assert_eq!( + err.message(), + Some("The XML you provided was not well-formed or did not validate against our published schema") + ); + + // [7] Error case: Retention with past date should fail + let past_date_retention = ObjectLockRetention { + mode: Some(ObjectLockRetentionMode::from_static(ObjectLockRetentionMode::COMPLIANCE)), + retain_until_date: Some(datetime!(2020-01-01 00:00:00 UTC).into()), + }; + let err = parse_object_lock_retention(Some(past_date_retention)).unwrap_err(); + assert_eq!(err.code().as_str(), S3ErrorCode::InvalidArgument.as_str()); + assert_eq!(err.message(), Some("The retain until date must be in the future")); + } + + #[test] + fn test_parse_object_lock_legal_hold() { + // [1] Normal case: No legal hold specified (empty metadata) + assert!(parse_object_lock_legal_hold(None).is_ok()); + assert!(parse_object_lock_legal_hold(None).unwrap().is_empty()); + + // [2] Normal case: Legal hold with valid ON status + let valid_on_legal_hold = ObjectLockLegalHold { + status: Some(ObjectLockLegalHoldStatus::from_static(ObjectLockLegalHoldStatus::ON)), + }; + let on_metadata = parse_object_lock_legal_hold(Some(valid_on_legal_hold)).unwrap(); + assert_eq!(on_metadata.get(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER).unwrap(), "ON"); + assert!(on_metadata.contains_key(&format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, "objectlock-legalhold-timestamp"))); + + // [3] Normal case: Legal hold with valid OFF status + let valid_off_legal_hold = ObjectLockLegalHold { + status: Some(ObjectLockLegalHoldStatus::from_static(ObjectLockLegalHoldStatus::OFF)), + }; + let off_metadata = parse_object_lock_legal_hold(Some(valid_off_legal_hold)).unwrap(); + assert_eq!(off_metadata.get(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER).unwrap(), "OFF"); + + // [4] Normal case: Legal hold with None status (empty string for status) + let none_status_legal_hold = ObjectLockLegalHold { status: None }; + let none_status_metadata = parse_object_lock_legal_hold(Some(none_status_legal_hold)).unwrap(); + assert_eq!(none_status_metadata.get(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER).unwrap(), ""); + + // [5] Error case: Legal hold with invalid status (non ON/OFF) + let invalid_status_legal_hold = ObjectLockLegalHold { + status: Some(ObjectLockLegalHoldStatus::from_static("INVALID_STATUS")), + }; + let err = parse_object_lock_legal_hold(Some(invalid_status_legal_hold)).unwrap_err(); + assert_eq!(err.code().as_str(), S3ErrorCode::MalformedXML.as_str()); + assert_eq!( + err.message(), + Some("The XML you provided was not well-formed or did not validate against our published schema") + ); + } + + #[tokio::test] + async fn test_validate_bucket_object_lock_enabled() { + use rustfs_ecstore::bucket::metadata::BucketMetadata; + use rustfs_ecstore::bucket::metadata_sys::set_bucket_metadata; + use s3s::dto::{ObjectLockConfiguration, ObjectLockEnabled}; + use time::OffsetDateTime; + + if rustfs_ecstore::bucket::metadata_sys::GLOBAL_BucketMetadataSys.get().is_none() { + eprintln!("Skipping test: GLOBAL_BucketMetadataSys not initialized"); + return; + } + + let test_bucket = "test-bucket-object-lock"; + + let mut bm = BucketMetadata::new(test_bucket); + bm.object_lock_config = Some(ObjectLockConfiguration { + object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)), + rule: None, + }); + bm.object_lock_config_updated_at = OffsetDateTime::now_utc(); + set_bucket_metadata(test_bucket.to_string(), bm).await.unwrap(); + assert!(validate_bucket_object_lock_enabled(test_bucket).await.is_ok()); + + let mut bm = BucketMetadata::new(test_bucket); + bm.object_lock_config = Some(ObjectLockConfiguration { + object_lock_enabled: None, + rule: None, + }); + bm.object_lock_config_updated_at = OffsetDateTime::now_utc(); + set_bucket_metadata(test_bucket.to_string(), bm).await.unwrap(); + let err = validate_bucket_object_lock_enabled(test_bucket).await.unwrap_err(); + assert_eq!(err.code().as_str(), S3ErrorCode::InvalidRequest.as_str()); + assert_eq!(err.message(), Some("Object Lock is not enabled for this bucket")); + + let non_exist_bucket = "non-exist-bucket-object-lock"; + let err = validate_bucket_object_lock_enabled(non_exist_bucket).await.unwrap_err(); + assert_eq!(err.code().as_str(), S3ErrorCode::InvalidRequest.as_str()); + assert_eq!(err.message(), Some("Bucket is missing ObjectLockConfiguration")); + } + + #[test] + fn test_is_etag_equal() { + // [1] Header ETag is "*", should return true (match any object ETag) + assert!(is_etag_equal("\"d41d8cd98f00b204e9800998ecf8427e\"", "*")); + + // [2] Exact match (both with double quotes) + assert!(is_etag_equal( + "\"d41d8cd98f00b204e9800998ecf8427e\"", + "\"d41d8cd98f00b204e9800998ecf8427e\"" + )); + + // [3] Exact match (object ETag with quotes, header ETag without) + assert!(is_etag_equal("\"d41d8cd98f00b204e9800998ecf8427e\"", "d41d8cd98f00b204e9800998ecf8427e")); + + // [4] Header ETag has multiple values (comma-separated), one matches + assert!(is_etag_equal("\"12345\"", "\"67890\", \"12345\", \"abcde\"")); + + // [5] Header ETag has multiple values with spaces, one matches after trim + assert!(is_etag_equal("\"12345\"", " \"67890\" , \"12345\" , \"abcde\" ")); + + // [6] No match (different ETag) + assert!(!is_etag_equal("\"12345\"", "\"67890\"")); + + // [7] No match in multiple values + assert!(!is_etag_equal("\"12345\"", "\"67890\", \"abcde\"")); + } + + #[test] + fn test_check_preconditions() { + let valid_mod_time = OffsetDateTime::from_unix_timestamp(1700000000).unwrap(); + let valid_etag = "\"d41d8cd98f00b204e9800998ecf8427e\""; + let wrong_etag = "\"wrong-etag-123456\""; + + // [1] Both mod_time and etag are None → early return Ok() + let info1 = ObjectInfo { + mod_time: None, + etag: None, + ..Default::default() + }; + let headers1 = HeaderMap::new(); + assert!(check_preconditions(&headers1, &info1).is_ok()); + + // [2] No conditional headers with etag=None → Ok() + let info2 = ObjectInfo { + mod_time: Some(valid_mod_time), + etag: None, + ..Default::default() + }; + let headers2 = HeaderMap::new(); + assert!(check_preconditions(&headers2, &info2).is_ok()); + + // [3] If-None-Match matches → return Err(S3Error::NotModified) + let mut headers3 = HeaderMap::new(); + headers3.insert("if-none-match", HeaderValue::from_str(valid_etag).unwrap()); + let info3 = ObjectInfo { + mod_time: Some(valid_mod_time), + etag: Some(valid_etag.to_string()), + ..Default::default() + }; + let result3 = check_preconditions(&headers3, &info3); + assert!(result3.is_err()); + let err3 = result3.unwrap_err(); + assert_eq!(err3.code(), &S3ErrorCode::NotModified); + assert_eq!(err3.message(), Some("Not Modified")); + assert_eq!(err3.status_code(), Some(StatusCode::NOT_MODIFIED)); + + // [4] If-None-Match does not match → return Ok() + let mut headers4 = HeaderMap::new(); + headers4.insert("if-none-match", HeaderValue::from_str("\"wrong-etag\"").unwrap()); + let info4 = info3.clone(); + assert!(check_preconditions(&headers4, &info4).is_ok()); + + // [5] If-Modified-Since >= mod_time → return Err(S3Error::NotModified) + let mut headers5 = HeaderMap::new(); + headers5.insert( + "if-modified-since", + HeaderValue::from_str(&valid_mod_time.format(&Rfc2822).unwrap()).unwrap(), + ); + let info5 = info3.clone(); + let result5 = check_preconditions(&headers5, &info5); + assert!(result5.is_err()); + let err5 = result5.unwrap_err(); + assert_eq!(err5.code(), &S3ErrorCode::NotModified); + + // [6] If-Modified-Since < mod_time → return Ok() + let earlier_time = valid_mod_time - time::Duration::hours(1); + let mut headers6 = HeaderMap::new(); + headers6.insert( + "if-modified-since", + HeaderValue::from_str(&earlier_time.format(&Rfc2822).unwrap()).unwrap(), + ); + let info6 = info3.clone(); + assert!(check_preconditions(&headers6, &info6).is_ok()); + + // [7] If-Match does not match → return Err(S3Error::PreconditionFailed) + let mut headers7 = HeaderMap::new(); + headers7.insert("if-match", HeaderValue::from_str("\"wrong-etag\"").unwrap()); + let info7 = info3.clone(); + let result7 = check_preconditions(&headers7, &info7); + assert!(result7.is_err()); + assert_eq!(result7.unwrap_err().code(), &S3ErrorCode::PreconditionFailed); + + // [8] If-Match matches → return Ok() + let mut headers8 = HeaderMap::new(); + headers8.insert("if-match", HeaderValue::from_str(valid_etag).unwrap()); + let info8 = info3.clone(); + assert!(check_preconditions(&headers8, &info8).is_ok()); + + // [9] If-Unmodified-Since < mod_time (no If-Match) → return Err(S3Error::PreconditionFailed) + let mut headers9 = HeaderMap::new(); + headers9.insert( + "if-unmodified-since", + HeaderValue::from_str(&earlier_time.format(&Rfc2822).unwrap()).unwrap(), + ); + let info9 = info3.clone(); + let result9 = check_preconditions(&headers9, &info9); + assert!(result9.is_err()); + assert_eq!(result9.unwrap_err().code(), &S3ErrorCode::PreconditionFailed); + + // [10] If-Unmodified-Since >= mod_time (no If-Match) → return Ok() + let mut headers10 = HeaderMap::new(); + headers10.insert( + "if-unmodified-since", + HeaderValue::from_str(&valid_mod_time.format(&Rfc2822).unwrap()).unwrap(), + ); + let info10 = info3.clone(); + assert!(check_preconditions(&headers10, &info10).is_ok()); + + // [11] If-Match (mismatch) + If-None-Match (match) → return Err(S3Error::PreconditionFailed) + let mut headers11 = HeaderMap::new(); + headers11.insert("if-match", HeaderValue::from_str(wrong_etag).unwrap()); + headers11.insert("if-none-match", HeaderValue::from_str(valid_etag).unwrap()); + let info11 = info3.clone(); + let result11 = check_preconditions(&headers11, &info11); + assert!(result11.is_err()); + let err11 = result11.unwrap_err(); + assert_eq!(err11.code(), &S3ErrorCode::PreconditionFailed); + assert_ne!(err11.code(), &S3ErrorCode::NotModified); + + // [12] If-Match (match) + If-None-Match (match) → return Err(S3Error::NotModified) + let mut headers12 = HeaderMap::new(); + headers12.insert("if-match", HeaderValue::from_str(valid_etag).unwrap()); + headers12.insert("if-none-match", HeaderValue::from_str(valid_etag).unwrap()); + let info12 = info3.clone(); + let result12 = check_preconditions(&headers12, &info12); + assert!(result12.is_err()); + let err12 = result12.unwrap_err(); + assert_eq!(err12.code(), &S3ErrorCode::NotModified); + assert_ne!(err12.code(), &S3ErrorCode::PreconditionFailed); + + // [13] If-None-Match (match) + If-Modified-Since → NotModified (If-Modified-Since ignored) + let mut headers13 = HeaderMap::new(); + headers13.insert("if-none-match", HeaderValue::from_str(valid_etag).unwrap()); + headers13.insert( + "if-modified-since", + HeaderValue::from_str(&earlier_time.format(&Rfc2822).unwrap()).unwrap(), + ); + let info13 = info3.clone(); + let result13 = check_preconditions(&headers13, &info13); + assert!(result13.is_err()); + assert_eq!(result13.unwrap_err().code(), &S3ErrorCode::NotModified); + + // [14] If-None-Match (no match) + If-Modified-Since → Ok (If-Modified-Since ignored) + let mut headers14 = HeaderMap::new(); + headers14.insert("if-none-match", HeaderValue::from_str("\"wrong-etag\"").unwrap()); + headers14.insert( + "if-modified-since", + HeaderValue::from_str(&valid_mod_time.format(&Rfc2822).unwrap()).unwrap(), + ); + let info14 = info3.clone(); + assert!(check_preconditions(&headers14, &info14).is_ok()); + + // [15] If-Match with no ETag → PreconditionFailed + let mut headers15 = HeaderMap::new(); + headers15.insert("if-match", HeaderValue::from_str(valid_etag).unwrap()); + let info15 = ObjectInfo { + mod_time: Some(valid_mod_time), + etag: None, + ..Default::default() + }; + let result15 = check_preconditions(&headers15, &info15); + assert!(result15.is_err()); + assert_eq!(result15.unwrap_err().code(), &S3ErrorCode::PreconditionFailed); + + // [16] If-None-Match with no ETag → Ok (no match possible) + let mut headers16 = HeaderMap::new(); + headers16.insert("if-none-match", HeaderValue::from_str(valid_etag).unwrap()); + let info16 = ObjectInfo { + mod_time: Some(valid_mod_time), + etag: None, + ..Default::default() + }; + assert!(check_preconditions(&headers16, &info16).is_ok()); + + // [17] mod_time None, etag exists, If-Match matches → Ok + let mut headers17 = HeaderMap::new(); + headers17.insert("if-match", HeaderValue::from_str(valid_etag).unwrap()); + let info17 = ObjectInfo { + mod_time: None, + etag: Some(valid_etag.to_string()), + ..Default::default() + }; + assert!(check_preconditions(&headers17, &info17).is_ok()); + } + + #[test] + fn test_parse_etag() { + // [1] ETag without quotes → adds quotes + let result1 = parse_etag("d41d8cd98f00b204e9800998ecf8427e").unwrap(); + assert_eq!(result1.to_str().unwrap(), "\"d41d8cd98f00b204e9800998ecf8427e\""); + + // [2] ETag with quotes → normalizes to single set of quotes + let result2 = parse_etag("\"d41d8cd98f00b204e9800998ecf8427e\"").unwrap(); + assert_eq!(result2.to_str().unwrap(), "\"d41d8cd98f00b204e9800998ecf8427e\""); + + // [3] ETag with multiple quotes → removes all and adds single set + let result3 = parse_etag("\"\"\"d41d8cd98f00b204e9800998ecf8427e\"\"\"").unwrap(); + assert_eq!(result3.to_str().unwrap(), "\"d41d8cd98f00b204e9800998ecf8427e\""); + + // [4] Empty string → returns error + let result4 = parse_etag(""); + assert!(result4.is_err()); + assert_eq!(result4.unwrap_err().code(), &S3ErrorCode::InvalidArgument); + + // [5] String with only quotes → returns error + let result5 = parse_etag("\"\""); + assert!(result5.is_err()); + assert_eq!(result5.unwrap_err().code(), &S3ErrorCode::InvalidArgument); + + // [6] ETag with quotes only at start → removes and adds quotes + let result6 = parse_etag("\"d41d8cd98f00b204e9800998ecf8427e").unwrap(); + assert_eq!(result6.to_str().unwrap(), "\"d41d8cd98f00b204e9800998ecf8427e\""); + + // [7] ETag with quotes only at end → removes and adds quotes + let result7 = parse_etag("d41d8cd98f00b204e9800998ecf8427e\"").unwrap(); + assert_eq!(result7.to_str().unwrap(), "\"d41d8cd98f00b204e9800998ecf8427e\""); + + // [8] ETag with newline character → returns error (invalid header value) + let result8 = parse_etag("d41d8cd98f00b204e9800998ecf8427e\n"); + assert!(result8.is_err()); + assert_eq!(result8.unwrap_err().code(), &S3ErrorCode::InvalidArgument); + + // [9] ETag with null byte → returns error (invalid header value) + let result9 = parse_etag("d41d8cd98f00b204e9800998ecf8427e\0"); + assert!(result9.is_err()); + assert_eq!(result9.unwrap_err().code(), &S3ErrorCode::InvalidArgument); + + // [10] ETag with carriage return → returns error (invalid header value) + let result10 = parse_etag("d41d8cd98f00b204e9800998ecf8427e\r"); + assert!(result10.is_err()); + assert_eq!(result10.unwrap_err().code(), &S3ErrorCode::InvalidArgument); + + // [11] ETag with whitespace → preserves content, adds quotes + let result11 = parse_etag(" d41d8cd98f00b204e9800998ecf8427e ").unwrap(); + assert_eq!(result11.to_str().unwrap(), "\" d41d8cd98f00b204e9800998ecf8427e \""); + + // [12] ETag with only whitespace (no quotes) → returns error + let result12 = parse_etag(" "); + assert!(result12.is_err()); + assert_eq!(result12.unwrap_err().code(), &S3ErrorCode::InvalidArgument); + + // [13] ETag with only whitespace (with quotes) → returns error + let result13 = parse_etag("\" \""); + assert!(result13.is_err()); + assert_eq!(result13.unwrap_err().code(), &S3ErrorCode::InvalidArgument); + } + + // Note: S3Request structure is complex and requires many fields. + // For real testing, we would need proper integration test setup. + // Removing this test as it requires too much S3 infrastructure setup. + + // Note: Testing actual S3 operations like put_object, get_object, etc. requires: + // 1. Initialized storage backend (ECStore) + // 2. Global configuration setup + // 3. Valid credentials and authorization + // 4. Bucket and object metadata systems + // 5. Network and disk I/O capabilities + // + // These are better suited for integration tests rather than unit tests. + // The current tests focus on the testable parts without external dependencies. + + /// Test that next_key_marker and next_version_id_marker are filtered correctly + /// AWS S3 API requires these fields to be omitted when empty, not set to None or "" + #[test] + fn test_next_marker_filtering() { + // Test filter behavior for empty strings + let empty_string = Some(String::new()); + let filtered = empty_string.filter(|v| !v.is_empty()); + assert!(filtered.is_none(), "Empty string should be filtered to None"); + + // Test filter behavior for non-empty strings + let non_empty = Some("some-marker".to_string()); + let filtered = non_empty.filter(|v| !v.is_empty()); + assert!(filtered.is_some(), "Non-empty string should not be filtered"); + assert_eq!(filtered.unwrap(), "some-marker"); + + // Test filter behavior for None + let none_value: Option = None; + let filtered = none_value.filter(|v| !v.is_empty()); + assert!(filtered.is_none(), "None should remain None"); + } + + /// Test version_id handling for ListObjectVersions response + /// Per AWS S3 API spec: + /// - Versioned objects: version_id is a UUID string + /// - Non-versioned objects: version_id should be "null" string + #[test] + fn test_version_id_formatting() { + use uuid::Uuid; + + // Non-versioned object: version_id is None, should format as "null" + let version_id: Option = None; + let formatted = version_id.map(|v| v.to_string()).unwrap_or_else(|| "null".to_string()); + assert_eq!(formatted, "null"); + + // Versioned object: version_id is Some(UUID), should format as UUID string + let uuid = Uuid::parse_str("550e8400-e29b-41d4-a716-446655440000").unwrap(); + let version_id: Option = Some(uuid); + let formatted = version_id.map(|v| v.to_string()).unwrap_or_else(|| "null".to_string()); + assert_eq!(formatted, "550e8400-e29b-41d4-a716-446655440000"); + } + + /// Test that ListObjectVersionsOutput markers are correctly set + /// This verifies the fix for boto3 ParamValidationError + #[test] + fn test_list_object_versions_markers_handling() { + // Simulate the marker filtering logic from list_object_versions + + // Case 1: Both markers have values (truncated result with versioned object) + let next_marker = Some("object-key".to_string()); + let next_version_idmarker = Some("550e8400-e29b-41d4-a716-446655440000".to_string()); + + let filtered_key_marker = next_marker.filter(|v| !v.is_empty()); + let filtered_version_marker = next_version_idmarker.filter(|v| !v.is_empty()); + + assert!(filtered_key_marker.is_some()); + assert!(filtered_version_marker.is_some()); + + // Case 2: Markers are empty strings (non-truncated result) + let next_marker = Some(String::new()); + let next_version_idmarker = Some(String::new()); + + let filtered_key_marker = next_marker.filter(|v| !v.is_empty()); + let filtered_version_marker = next_version_idmarker.filter(|v| !v.is_empty()); + + assert!(filtered_key_marker.is_none(), "Empty key marker should be filtered to None"); + assert!(filtered_version_marker.is_none(), "Empty version marker should be filtered to None"); + + // Case 3: Truncated result with non-versioned object (version_id is "null") + let next_marker = Some("object-key".to_string()); + let next_version_idmarker = Some("null".to_string()); + + let filtered_key_marker = next_marker.filter(|v| !v.is_empty()); + let filtered_version_marker = next_version_idmarker.filter(|v| !v.is_empty()); + + assert!(filtered_key_marker.is_some()); + assert!(filtered_version_marker.is_some()); + assert_eq!(filtered_version_marker.unwrap(), "null"); + } + + #[test] + fn test_matches_origin_pattern_exact_match() { + // Test exact match + assert!(matches_origin_pattern("https://example.com", "https://example.com")); + assert!(matches_origin_pattern("http://localhost:3000", "http://localhost:3000")); + assert!(!matches_origin_pattern("https://example.com", "https://other.com")); + } + + #[test] + fn test_matches_origin_pattern_wildcard() { + // Test wildcard pattern matching (S3 CORS supports * as subdomain wildcard) + assert!(matches_origin_pattern("https://*.example.com", "https://app.example.com")); + assert!(matches_origin_pattern("https://*.example.com", "https://api.example.com")); + assert!(matches_origin_pattern("https://*.example.com", "https://subdomain.example.com")); + + // Test wildcard at start (matches any domain) + assert!(matches_origin_pattern("https://*", "https://example.com")); + assert!(matches_origin_pattern("https://*", "https://any-domain.com")); + + // Test wildcard at end (matches any protocol) + assert!(matches_origin_pattern("*://example.com", "https://example.com")); + assert!(matches_origin_pattern("*://example.com", "http://example.com")); + + // Test invalid wildcard patterns (should not match) + assert!(!matches_origin_pattern("https://*.*.com", "https://app.example.com")); // Multiple wildcards (invalid pattern) + // Note: "https://*example.com" actually matches "https://app.example.com" with our current implementation + // because it splits on * and checks starts_with/ends_with. This is a limitation but acceptable + // for S3 CORS which typically uses patterns like "https://*.example.com" + } + + #[test] + fn test_matches_origin_pattern_no_wildcard() { + // Test patterns without wildcards + assert!(matches_origin_pattern("https://example.com", "https://example.com")); + assert!(!matches_origin_pattern("https://example.com", "https://example.org")); + assert!(!matches_origin_pattern("http://example.com", "https://example.com")); // Different protocol + } + + #[test] + fn test_matches_origin_pattern_edge_cases() { + // Test edge cases + assert!(!matches_origin_pattern("", "https://example.com")); // Empty pattern + assert!(!matches_origin_pattern("https://example.com", "")); // Empty origin + assert!(matches_origin_pattern("", "")); // Both empty + assert!(!matches_origin_pattern("https://example.com", "http://example.com")); // Protocol mismatch + } + + #[test] + fn test_cors_headers_validation() { + use http::HeaderMap; + + // Test case 1: Validate header name case-insensitivity + let mut headers = HeaderMap::new(); + headers.insert("access-control-request-headers", "Content-Type,X-Custom-Header".parse().unwrap()); + + let req_headers_str = headers + .get("access-control-request-headers") + .and_then(|v| v.to_str().ok()) + .unwrap(); + let req_headers: Vec = req_headers_str.split(',').map(|s| s.trim().to_lowercase()).collect(); + + // Headers should be lowercased for comparison + assert_eq!(req_headers, vec!["content-type", "x-custom-header"]); + + // Test case 2: Wildcard matching + let allowed_headers = ["*".to_string()]; + let all_allowed = req_headers.iter().all(|req_header| { + allowed_headers + .iter() + .any(|allowed| allowed.to_lowercase() == "*" || allowed.to_lowercase() == *req_header) + }); + assert!(all_allowed, "Wildcard should allow all headers"); + + // Test case 3: Specific header matching + let allowed_headers = ["content-type".to_string(), "x-custom-header".to_string()]; + let all_allowed = req_headers + .iter() + .all(|req_header| allowed_headers.iter().any(|allowed| allowed.to_lowercase() == *req_header)); + assert!(all_allowed, "All requested headers should be allowed"); + + // Test case 4: Disallowed header + let req_headers = ["content-type".to_string(), "x-forbidden-header".to_string()]; + let allowed_headers = ["content-type".to_string()]; + let all_allowed = req_headers + .iter() + .all(|req_header| allowed_headers.iter().any(|allowed| allowed.to_lowercase() == *req_header)); + assert!(!all_allowed, "Forbidden header should not be allowed"); + } + + #[test] + fn test_cors_response_headers_structure() { + use http::{HeaderMap, HeaderValue}; + + let mut cors_headers = HeaderMap::new(); + + // Simulate building CORS response headers + let origin = "https://example.com"; + let methods = ["GET", "PUT", "POST"]; + let allowed_headers = ["Content-Type", "Authorization"]; + let expose_headers = ["ETag", "x-amz-version-id"]; + let max_age = 3600; + + // Add headers + cors_headers.insert("access-control-allow-origin", HeaderValue::from_str(origin).unwrap()); + cors_headers.insert("vary", HeaderValue::from_static("Origin")); + + let methods_str = methods.join(", "); + cors_headers.insert("access-control-allow-methods", HeaderValue::from_str(&methods_str).unwrap()); + + let headers_str = allowed_headers.join(", "); + cors_headers.insert("access-control-allow-headers", HeaderValue::from_str(&headers_str).unwrap()); + + let expose_str = expose_headers.join(", "); + cors_headers.insert("access-control-expose-headers", HeaderValue::from_str(&expose_str).unwrap()); + + cors_headers.insert("access-control-max-age", HeaderValue::from_str(&max_age.to_string()).unwrap()); + + // Verify all headers are present + assert_eq!(cors_headers.get("access-control-allow-origin").unwrap(), origin); + assert_eq!(cors_headers.get("vary").unwrap(), "Origin"); + assert_eq!(cors_headers.get("access-control-allow-methods").unwrap(), "GET, PUT, POST"); + assert_eq!(cors_headers.get("access-control-allow-headers").unwrap(), "Content-Type, Authorization"); + assert_eq!(cors_headers.get("access-control-expose-headers").unwrap(), "ETag, x-amz-version-id"); + assert_eq!(cors_headers.get("access-control-max-age").unwrap(), "3600"); + } + + #[test] + fn test_cors_preflight_vs_actual_request() { + use http::Method; + + // Test that we can distinguish preflight from actual requests + let preflight_method = Method::OPTIONS; + let actual_method = Method::PUT; + + assert_eq!(preflight_method, Method::OPTIONS); + assert_ne!(actual_method, Method::OPTIONS); + + // Preflight should check Access-Control-Request-Method + // Actual request should use the actual method + let is_preflight_1 = preflight_method == Method::OPTIONS; + let is_preflight_2 = actual_method == Method::OPTIONS; + + assert!(is_preflight_1); + assert!(!is_preflight_2); + } + + #[tokio::test] + async fn test_apply_cors_headers_no_origin() { + // Test when no Origin header is present + let headers = HeaderMap::new(); + let method = http::Method::GET; + + // Should return None when no origin header + let result = apply_cors_headers("test-bucket", &method, &headers).await; + assert!(result.is_none(), "Should return None when no Origin header"); + } + + #[tokio::test] + async fn test_apply_cors_headers_no_cors_config() { + // Test when bucket has no CORS configuration + let mut headers = HeaderMap::new(); + headers.insert("origin", "https://example.com".parse().unwrap()); + let method = http::Method::GET; + + // Should return None when no CORS config exists + // Note: This test may fail if test-bucket actually has CORS config + // In a real scenario, we'd use a mock or ensure the bucket doesn't exist + let _result = apply_cors_headers("non-existent-bucket-for-testing", &method, &headers).await; + // Result depends on whether bucket exists and has CORS config + // This is expected behavior - we just verify it doesn't panic + } + + #[tokio::test] + async fn test_apply_cors_headers_unsupported_method() { + // Test with unsupported HTTP method + let mut headers = HeaderMap::new(); + headers.insert("origin", "https://example.com".parse().unwrap()); + let method = http::Method::PATCH; // Unsupported method + + let result = apply_cors_headers("test-bucket", &method, &headers).await; + assert!(result.is_none(), "Should return None for unsupported methods"); + } + + #[test] + fn test_matches_origin_pattern_complex_wildcards() { + // Test more complex wildcard scenarios + assert!(matches_origin_pattern("https://*.example.com", "https://sub.example.com")); + // Note: "https://*.example.com" matches "https://api.sub.example.com" with our implementation + // because it only checks starts_with and ends_with. Real S3 might be more strict. + + // Test wildcard in middle position + // Our implementation allows this, but it's not standard S3 CORS pattern + // The pattern "https://example.*.com" splits to ["https://example.", ".com"] + // and "https://example.sub.com" matches because it starts with "https://example." and ends with ".com" + // This is acceptable for our use case as S3 CORS typically uses "https://*.example.com" format + } + + // === Notification Configuration Error Propagation Tests === + + #[test] + fn test_process_queue_configurations_propagates_error_on_invalid_arn() { + use rustfs_targets::arn::{ARN, TargetIDError}; + + let mut event_rules = Vec::new(); + let invalid_arn = "arn:minio:sqs::1:webhook"; // Wrong prefix, should fail + + let result = process_queue_configurations( + &mut event_rules, + Some(vec![QueueConfiguration { + events: vec!["s3:ObjectCreated:*".to_string().into()], + queue_arn: invalid_arn.to_string(), + filter: None, + id: None, + }]), + |arn_str| { + ARN::parse(arn_str) + .map(|arn| arn.target_id) + .map_err(|e| TargetIDError::InvalidFormat(e.to_string())) + }, + ); + + assert!(result.is_err(), "Should return error for invalid ARN prefix"); + assert!(event_rules.is_empty(), "Should not add rules when ARN is invalid"); + } + + #[test] + fn test_process_topic_configurations_propagates_error_on_invalid_arn() { + use rustfs_targets::arn::{ARN, TargetIDError}; + + let mut event_rules = Vec::new(); + let invalid_arn = "arn:aws:sns:us-east-1:123:topic"; // Wrong prefix, should fail + + let result = process_topic_configurations( + &mut event_rules, + Some(vec![TopicConfiguration { + events: vec!["s3:ObjectCreated:*".to_string().into()], + topic_arn: invalid_arn.to_string(), + filter: None, + id: None, + }]), + |arn_str| { + ARN::parse(arn_str) + .map(|arn| arn.target_id) + .map_err(|e| TargetIDError::InvalidFormat(e.to_string())) + }, + ); + + assert!(result.is_err(), "Should return error for invalid ARN prefix"); + assert!(event_rules.is_empty(), "Should not add rules when ARN is invalid"); + } + + #[test] + fn test_process_lambda_configurations_propagates_error_on_invalid_arn() { + use rustfs_targets::arn::{ARN, TargetIDError}; + + let mut event_rules = Vec::new(); + let invalid_arn = "arn:aws:lambda:us-east-1:123:function"; // Wrong prefix, should fail + + let result = process_lambda_configurations( + &mut event_rules, + Some(vec![LambdaFunctionConfiguration { + events: vec!["s3:ObjectCreated:*".to_string().into()], + lambda_function_arn: invalid_arn.to_string(), + filter: None, + id: None, + }]), + |arn_str| { + ARN::parse(arn_str) + .map(|arn| arn.target_id) + .map_err(|e| TargetIDError::InvalidFormat(e.to_string())) + }, + ); + + assert!(result.is_err(), "Should return error for invalid ARN prefix"); + assert!(event_rules.is_empty(), "Should not add rules when ARN is invalid"); + } + + #[test] + fn test_process_queue_configurations_succeeds_with_valid_arn() { + use rustfs_targets::arn::{ARN, TargetIDError}; + + let mut event_rules = Vec::new(); + let valid_arn = "arn:rustfs:sqs:us-east-1:1:webhook"; // Correct prefix + + let result = process_queue_configurations( + &mut event_rules, + Some(vec![QueueConfiguration { + events: vec!["s3:ObjectCreated:*".to_string().into()], + queue_arn: valid_arn.to_string(), + filter: None, + id: None, + }]), + |arn_str| { + ARN::parse(arn_str) + .map(|arn| arn.target_id) + .map_err(|e| TargetIDError::InvalidFormat(e.to_string())) + }, + ); + + assert!(result.is_ok(), "Should succeed with valid ARN"); + assert_eq!(event_rules.len(), 1, "Should add one rule"); + } +} diff --git a/rustfs/src/storage/mod.rs b/rustfs/src/storage/mod.rs index 4af9eb5d1..ca44fa5a4 100644 --- a/rustfs/src/storage/mod.rs +++ b/rustfs/src/storage/mod.rs @@ -22,3 +22,8 @@ pub mod tonic_service; #[cfg(test)] mod concurrent_get_object_test; +mod ecfs_extend; +#[cfg(test)] +mod ecfs_test; + +pub(crate) use ecfs_extend::*;