diff --git a/.vscode/launch.json b/.vscode/launch.json index 28578442c..0fb590e86 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -20,33 +20,19 @@ } }, "env": { - "RUST_LOG": "rustfs=debug,ecstore=info,s3s=debug", - "RUSTFS_VOLUMES": "./target/volume/test{0...3}", - "RUSTFS_ADDRESS": "[::]:9000", - "RUSTFS_CONSOLE_ENABLE": "true", - "RUSTFS_CONSOLE_ADDRESS": "[::]:9002", - "RUSTFS_SERVER_DOMAINS": "localhost:9000", - "RUSTFS_TLS_PATH": "./deploy/certs", - "RUSTFS_OBS_CONFIG": "./deploy/config/obs.example.toml", - "RUSTFS__OBSERVABILITY__ENDPOINT": "http://localhost:4317", - "RUSTFS__OBSERVABILITY__USE_STDOUT": "true", - "RUSTFS__OBSERVABILITY__SAMPLE_RATIO": "2.0", - "RUSTFS__OBSERVABILITY__METER_INTERVAL": "30", - "RUSTFS__OBSERVABILITY__SERVICE_NAME": "rustfs", - "RUSTFS__OBSERVABILITY__SERVICE_VERSION": "0.1.0", - "RUSTFS__OBSERVABILITY__ENVIRONMENT": "develop", - "RUSTFS__OBSERVABILITY__LOGGER_LEVEL": "info", - "RUSTFS__SINKS__FILE__ENABLED": "true", - "RUSTFS__SINKS__FILE__PATH": "./deploy/logs/rustfs.log", - "RUSTFS__SINKS__WEBHOOK__ENABLED": "false", - "RUSTFS__SINKS__WEBHOOK__ENDPOINT": "", - "RUSTFS__SINKS__WEBHOOK__AUTH_TOKEN": "", - "RUSTFS__SINKS__KAFKA__ENABLED": "false", - "RUSTFS__SINKS__KAFKA__BOOTSTRAP_SERVERS": "", - "RUSTFS__SINKS__KAFKA__TOPIC": "", - "RUSTFS__LOGGER__QUEUE_CAPACITY": "10" - + "RUST_LOG": "rustfs=debug,ecstore=info,s3s=debug" }, + "args": [ + "--access-key", + "AKEXAMPLERUSTFS", + "--secret-key", + "SKEXAMPLERUSTFS", + "--address", + "0.0.0.0:9010", + "--domain-name", + "127.0.0.1:9010", + "./target/volume/test{0...4}" + ], "cwd": "${workspaceFolder}" }, { @@ -86,6 +72,19 @@ }, "args": [], "cwd": "${workspaceFolder}" + }, + { + "name": "Debug executable target/debug/rustfs", + "type": "lldb", + "request": "launch", + "program": "${workspaceFolder}/target/debug/rustfs", + "args": [], + "cwd": "${workspaceFolder}", + //"stopAtEntry": false, + //"preLaunchTask": "cargo build", + "sourceLanguages": [ + "rust" + ], } ] } \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index 2ec45197f..f47221129 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -13,9 +13,9 @@ dependencies = [ [[package]] name = "adler2" -version = "2.0.1" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" +checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" [[package]] name = "aead" @@ -52,6 +52,17 @@ dependencies = [ "subtle", ] +[[package]] +name = "ahash" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891477e0c6a8957309ee5c45a6368af3ae14bb510732d2684ffa19af310920f9" +dependencies = [ + "getrandom 0.2.16", + "once_cell", + "version_check", +] + [[package]] name = "ahash" version = "0.8.12" @@ -237,6 +248,12 @@ dependencies = [ "password-hash", ] +[[package]] +name = "arraydeque" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d902e3d592a523def97af8f317b08ce16b7ab854c1985a0c671e6f15cebc236" + [[package]] name = "arrayref" version = "0.3.9" @@ -290,7 +307,7 @@ version = "54.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a12fcdb3f1d03f69d3ec26ac67645a8fe3f878d77b5ebb0b15d64a116c212985" dependencies = [ - "ahash", + "ahash 0.8.12", "arrow-buffer", "arrow-data", "arrow-schema", @@ -435,7 +452,7 @@ version = "54.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69efcd706420e52cd44f5c4358d279801993846d1c2a8e52111853d61d55a619" dependencies = [ - "ahash", + "ahash 0.8.12", "arrow-array", "arrow-buffer", "arrow-data", @@ -593,7 +610,7 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2", "quote", - "syn 2.0.103", + "syn 2.0.101", ] [[package]] @@ -633,7 +650,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.103", + "syn 2.0.101", ] [[package]] @@ -650,7 +667,7 @@ checksum = "e539d3fca749fcee5236ab05e93a52867dd549cc157c8cb7f99595f3cedffdb5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.103", + "syn 2.0.101", ] [[package]] @@ -713,14 +730,14 @@ checksum = "99e1aca718ea7b89985790c94aad72d77533063fe00bc497bb79a7c2dae6a661" dependencies = [ "proc-macro2", "quote", - "syn 2.0.103", + "syn 2.0.101", ] [[package]] name = "autocfg" -version = "1.5.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] name = "aws-credential-types" @@ -759,9 +776,9 @@ dependencies = [ [[package]] name = "aws-runtime" -version = "1.5.8" +version = "1.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f6c68419d8ba16d9a7463671593c54f81ba58cab466e9b759418da606dcc2e2" +checksum = "6c4063282c69991e57faab9e5cb21ae557e59f5b0fb285c196335243df8dc25c" dependencies = [ "aws-credential-types", "aws-sigv4", @@ -784,9 +801,9 @@ dependencies = [ [[package]] name = "aws-sdk-s3" -version = "1.93.0" +version = "1.91.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16b9734dc8145b417a3c22eae8769a2879851690982dba718bdc52bd28ad04ce" +checksum = "10c7d58f9c99e7d33e5a9b288ec84db24de046add7ba4c1e98baf6b3a5b37fde" dependencies = [ "aws-credential-types", "aws-runtime", @@ -808,7 +825,7 @@ dependencies = [ "http 0.2.12", "http 1.3.1", "http-body 0.4.6", - "lru", + "lru 0.12.5", "percent-encoding", "regex-lite", "sha2 0.10.9", @@ -818,9 +835,9 @@ dependencies = [ [[package]] name = "aws-sigv4" -version = "1.3.3" +version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddfb9021f581b71870a17eac25b52335b82211cdc092e02b6876b2bcefa61666" +checksum = "3734aecf9ff79aa401a6ca099d076535ab465ff76b46440cf567c8e70b65dc13" dependencies = [ "aws-credential-types", "aws-smithy-eventstream", @@ -877,9 +894,9 @@ dependencies = [ [[package]] name = "aws-smithy-eventstream" -version = "0.60.9" +version = "0.60.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "338a3642c399c0a5d157648426110e199ca7fd1c689cc395676b81aa563700c4" +checksum = "7c45d3dddac16c5c59d553ece225a88870cf81b7b813c9cc17b78cf4685eac7a" dependencies = [ "aws-smithy-types", "bytes", @@ -909,9 +926,9 @@ dependencies = [ [[package]] name = "aws-smithy-http-client" -version = "1.0.5" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f491388e741b7ca73b24130ff464c1478acc34d5b331b7dd0a2ee4643595a15" +checksum = "073d330f94bdf1f47bb3e0f5d45dda1e372a54a553c39ab6e9646902c8c81594" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", @@ -928,7 +945,7 @@ dependencies = [ "hyper-util", "pin-project-lite", "rustls 0.21.12", - "rustls 0.23.28", + "rustls 0.23.27", "rustls-native-certs 0.8.1", "rustls-pki-types", "tokio", @@ -938,9 +955,9 @@ dependencies = [ [[package]] name = "aws-smithy-json" -version = "0.61.4" +version = "0.61.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a16e040799d29c17412943bdbf488fd75db04112d0c0d4b9290bacf5ae0014b9" +checksum = "92144e45819cae7dc62af23eac5a038a58aa544432d2102609654376a900bd07" dependencies = [ "aws-smithy-types", ] @@ -980,9 +997,9 @@ dependencies = [ [[package]] name = "aws-smithy-runtime-api" -version = "1.8.1" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd8531b6d8882fd8f48f82a9754e682e29dd44cff27154af51fa3eb730f59efb" +checksum = "a1e5d9e3a80a18afa109391fb5ad09c3daf887b516c6fd805a157c6ea7994a57" dependencies = [ "aws-smithy-async", "aws-smithy-types", @@ -997,9 +1014,9 @@ dependencies = [ [[package]] name = "aws-smithy-types" -version = "1.3.2" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d498595448e43de7f4296b7b7a18a8a02c61ec9349128c80a368f7c3b4ab11a8" +checksum = "40076bd09fadbc12d5e026ae080d0930defa606856186e31d83ccc6a255eeaf3" dependencies = [ "base64-simd", "bytes", @@ -1023,9 +1040,9 @@ dependencies = [ [[package]] name = "aws-smithy-xml" -version = "0.60.10" +version = "0.60.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3db87b96cb1b16c024980f133968d52882ca0daaee3a086c6decc500f6c99728" +checksum = "ab0b0166827aa700d3dc519f72f8b3a91c35d0b8d042dc5d643a91e6f80648fc" dependencies = [ "xmlparser", ] @@ -1134,7 +1151,7 @@ dependencies = [ "hyper 1.6.0", "hyper-util", "pin-project-lite", - "rustls 0.23.28", + "rustls 0.23.27", "rustls-pemfile 2.2.0", "rustls-pki-types", "tokio", @@ -1174,15 +1191,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "349a06037c7bf932dd7e7d1f653678b2038b9ad46a74102f1fc7bd7872678cce" -[[package]] -name = "base62" -version = "2.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10e52a7bcb1d6beebee21fb5053af9e3cbb7a7ed1a4909e534040e676437ab1f" -dependencies = [ - "rustversion", -] - [[package]] name = "base64" version = "0.21.7" @@ -1243,7 +1251,7 @@ dependencies = [ "regex", "rustc-hash 1.1.0", "shlex", - "syn 2.0.103", + "syn 2.0.101", "which", ] @@ -1381,16 +1389,6 @@ dependencies = [ "alloc-stdlib", ] -[[package]] -name = "bstr" -version = "1.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "234113d19d0d7d613b40e86fb654acf958910802bcceab913a4f9e7cda03b1a4" -dependencies = [ - "memchr", - "serde", -] - [[package]] name = "bumpalo" version = "3.18.1" @@ -1408,9 +1406,6 @@ name = "bytes" version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" -dependencies = [ - "serde", -] [[package]] name = "bytes-utils" @@ -1492,38 +1487,21 @@ dependencies = [ [[package]] name = "cargo-platform" -version = "0.2.0" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84982c6c0ae343635a3a4ee6dedef965513735c8b183caa7289fa6e27399ebd4" +checksum = "e35af189006b9c0f00a064685c727031e3ed2d8020f7ba284d78cc2671bd36ea" dependencies = [ "serde", ] -[[package]] -name = "cargo-util-schemas" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e63d2780ac94487eb9f1fea7b0d56300abc9eb488800854ca217f102f5caccca" -dependencies = [ - "semver", - "serde", - "serde-untagged", - "serde-value", - "thiserror 1.0.69", - "toml", - "unicode-xid", - "url", -] - [[package]] name = "cargo_metadata" -version = "0.20.0" +version = "0.19.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f7835cfc6135093070e95eb2b53e5d9b5c403dc3a6be6040ee026270aa82502" +checksum = "dd5eb614ed4c27c5d706420e4320fbe3216ab31fa1c33cd8246ac36dae4479ba" dependencies = [ "camino", "cargo-platform", - "cargo-util-schemas", "semver", "serde", "serde_json", @@ -1538,9 +1516,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.2.27" +version = "1.2.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d487aa071b5f64da6f19a3e848e3578944b726ee5a4854b82172f02aa876bfdc" +checksum = "956a5e21988b87f372569b66183b78babf23ebc2e744b733e4350a752c4dafac" dependencies = [ "jobserver", "libc", @@ -1585,9 +1563,9 @@ dependencies = [ [[package]] name = "cfg-if" -version = "1.0.1" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9555578bc9e57714c812a1f84e4fc5b4d21fcb063490c624de019f7464c91268" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "cfg_aliases" @@ -1735,14 +1713,14 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.103", + "syn 2.0.101", ] [[package]] name = "clap_lex" -version = "0.7.5" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b94f61472cee1439c0b966b47e3aca9ae07e45d070759512cd390ea2bebc6675" +checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6" [[package]] name = "cmake" @@ -1859,6 +1837,25 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "config" +version = "0.15.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "595aae20e65c3be792d05818e8c63025294ac3cb7e200f11459063a352a6ef80" +dependencies = [ + "async-trait", + "convert_case 0.6.0", + "json5", + "pathdiff", + "ron", + "rust-ini", + "serde", + "serde_json", + "toml", + "winnow 0.7.10", + "yaml-rust2", +] + [[package]] name = "console_error_panic_hook" version = "0.1.7" @@ -1919,7 +1916,7 @@ checksum = "04382d0d9df7434af6b1b49ea1a026ef39df1b0738b1cc373368cf175354f6eb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.103", + "syn 2.0.101", ] [[package]] @@ -1939,7 +1936,7 @@ checksum = "f0d1c4c3cb85e5856b34e829af0035d7154f8c2889b15bbf43c8a6c6786dcab5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.103", + "syn 2.0.101", ] [[package]] @@ -2083,10 +2080,11 @@ checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" [[package]] name = "crc-fast" -version = "1.3.0" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bf62af4cc77d8fe1c22dde4e721d87f2f54056139d8c412e1366b740305f56f" +checksum = "68fcb2be5386ffb77e30bf10820934cb89a628bcb976e7cc632dcd88c059ebea" dependencies = [ + "cc", "crc", "digest 0.10.7", "libc", @@ -2292,7 +2290,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13b588ba4ac1a99f7f2964d24b3d896ddc6bf847ee3855dbd4366f058cfcd331" dependencies = [ "quote", - "syn 2.0.103", + "syn 2.0.101", ] [[package]] @@ -2346,7 +2344,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.103", + "syn 2.0.101", ] [[package]] @@ -2357,7 +2355,7 @@ checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ "darling_core", "quote", - "syn 2.0.103", + "syn 2.0.101", ] [[package]] @@ -2370,7 +2368,7 @@ dependencies = [ "hashbrown 0.14.5", "lock_api", "once_cell", - "parking_lot_core", + "parking_lot_core 0.9.11", ] [[package]] @@ -2384,7 +2382,7 @@ dependencies = [ "hashbrown 0.14.5", "lock_api", "once_cell", - "parking_lot_core", + "parking_lot_core 0.9.11", ] [[package]] @@ -2431,7 +2429,7 @@ dependencies = [ "itertools 0.14.0", "log", "object_store", - "parking_lot", + "parking_lot 0.12.4", "parquet", "rand 0.8.5", "regex", @@ -2461,7 +2459,7 @@ dependencies = [ "futures", "itertools 0.14.0", "log", - "parking_lot", + "parking_lot 0.12.4", ] [[package]] @@ -2492,7 +2490,7 @@ version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f53d7ec508e1b3f68bd301cee3f649834fad51eff9240d898a4b2614cfd0a7a" dependencies = [ - "ahash", + "ahash 0.8.12", "arrow", "arrow-ipc", "base64 0.22.1", @@ -2573,7 +2571,7 @@ dependencies = [ "futures", "log", "object_store", - "parking_lot", + "parking_lot 0.12.4", "rand 0.8.5", "tempfile", "url", @@ -2648,7 +2646,7 @@ version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "adfc2d074d5ee4d9354fdcc9283d5b2b9037849237ddecb8942a29144b77ca05" dependencies = [ - "ahash", + "ahash 0.8.12", "arrow", "datafusion-common", "datafusion-doc", @@ -2669,7 +2667,7 @@ version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1cbceba0f98d921309a9121b702bcd49289d383684cccabf9a92cda1602f3bbb" dependencies = [ - "ahash", + "ahash 0.8.12", "arrow", "datafusion-common", "datafusion-expr-common", @@ -2709,7 +2707,7 @@ dependencies = [ "datafusion-common", "datafusion-expr", "datafusion-physical-plan", - "parking_lot", + "parking_lot 0.12.4", "paste", ] @@ -2748,7 +2746,7 @@ checksum = "4800e1ff7ecf8f310887e9b54c9c444b8e215ccbc7b21c2f244cfae373b1ece7" dependencies = [ "datafusion-expr", "quote", - "syn 2.0.103", + "syn 2.0.101", ] [[package]] @@ -2776,7 +2774,7 @@ version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e1447c2c6bc8674a16be4786b4abf528c302803fafa186aa6275692570e64d85" dependencies = [ - "ahash", + "ahash 0.8.12", "arrow", "datafusion-common", "datafusion-expr", @@ -2798,7 +2796,7 @@ version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69f8c25dcd069073a75b3d2840a79d0f81e64bdd2c05f2d3d18939afb36a7dcb" dependencies = [ - "ahash", + "ahash 0.8.12", "arrow", "datafusion-common", "datafusion-expr-common", @@ -2831,7 +2829,7 @@ version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "88cc160df00e413e370b3b259c8ea7bfbebc134d32de16325950e9e923846b7f" dependencies = [ - "ahash", + "ahash 0.8.12", "arrow", "arrow-ord", "arrow-schema", @@ -2850,7 +2848,7 @@ dependencies = [ "indexmap 2.9.0", "itertools 0.14.0", "log", - "parking_lot", + "parking_lot 0.12.4", "pin-project-lite", "tokio", ] @@ -2941,7 +2939,7 @@ checksum = "30542c1ad912e0e3d22a1935c290e12e8a29d704a420177a31faad4a601a0800" dependencies = [ "proc-macro2", "quote", - "syn 2.0.103", + "syn 2.0.101", ] [[package]] @@ -2962,7 +2960,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.103", + "syn 2.0.101", ] [[package]] @@ -2972,7 +2970,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" dependencies = [ "derive_builder_core", - "syn 2.0.103", + "syn 2.0.101", ] [[package]] @@ -2985,7 +2983,7 @@ dependencies = [ "proc-macro2", "quote", "rustc_version", - "syn 2.0.103", + "syn 2.0.101", ] [[package]] @@ -3088,7 +3086,7 @@ dependencies = [ "dioxus-rsx", "proc-macro2", "quote", - "syn 2.0.103", + "syn 2.0.101", ] [[package]] @@ -3282,7 +3280,7 @@ dependencies = [ "convert_case 0.6.0", "proc-macro2", "quote", - "syn 2.0.103", + "syn 2.0.101", ] [[package]] @@ -3375,7 +3373,7 @@ dependencies = [ "proc-macro2", "quote", "slab", - "syn 2.0.103", + "syn 2.0.101", ] [[package]] @@ -3387,7 +3385,7 @@ dependencies = [ "proc-macro2", "proc-macro2-diagnostics", "quote", - "syn 2.0.103", + "syn 2.0.101", ] [[package]] @@ -3401,7 +3399,7 @@ dependencies = [ "futures-util", "generational-box", "once_cell", - "parking_lot", + "parking_lot 0.12.4", "rustc-hash 1.1.0", "tracing", "warnings", @@ -3448,7 +3446,7 @@ dependencies = [ "proc-macro2", "quote", "server_fn_macro", - "syn 2.0.103", + "syn 2.0.101", ] [[package]] @@ -3469,7 +3467,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.60.2", + "windows-sys 0.59.0", ] [[package]] @@ -3508,7 +3506,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.103", + "syn 2.0.101", ] [[package]] @@ -3531,9 +3529,24 @@ checksum = "788160fb30de9cdd857af31c6a2675904b16ece8fc2737b2c7127ba368c9d0f4" dependencies = [ "proc-macro2", "quote", - "syn 2.0.103", + "syn 2.0.101", ] +[[package]] +name = "dlv-list" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "442039f5147480ba31067cb00ada1adae6892028e40e45fc5de7b7df6dcc1b5f" +dependencies = [ + "const-random", +] + +[[package]] +name = "dotenvy" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" + [[package]] name = "dpi" version = "0.1.2" @@ -3565,7 +3578,6 @@ checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" name = "e2e_test" version = "0.0.1" dependencies = [ - "bytes", "common", "ecstore", "flatbuffers 25.2.10", @@ -3600,10 +3612,10 @@ dependencies = [ name = "ecstore" version = "0.0.1" dependencies = [ + "async-channel", "async-trait", "aws-sdk-s3", "backon", - "base64 0.22.1", "base64-simd", "blake2", "byteorder", @@ -3613,20 +3625,25 @@ dependencies = [ "common", "crc32fast", "criterion", + "enumset", "flatbuffers 25.2.10", "futures", - "futures-util", "glob", "hex-simd", "highway", "hmac 0.12.1", "http 1.3.1", + "http-body-util", + "hyper 1.6.0", + "hyper-rustls 0.27.7", + "hyper-util", "lazy_static", "lock", "madmin", "md-5", "netif", "nix 0.30.1", + "num", "num_cpus", "once_cell", "path-absolutize", @@ -3635,6 +3652,8 @@ dependencies = [ "policy", "protos", "rand 0.9.1", + "reader", + "reed-solomon-erasure", "reed-solomon-simd", "regex", "reqwest", @@ -3645,15 +3664,18 @@ dependencies = [ "rustfs-rio", "rustfs-rsc", "rustfs-utils", + "rustls 0.23.27", "s3s", "serde", + "serde-xml-rs 0.8.1", "serde_json", "serde_urlencoded", + "sha1 0.10.6", "sha2 0.10.9", "shadow-rs", "siphasher 1.0.1", "smallvec", - "temp-env", + "std-next", "tempfile", "thiserror 2.0.12", "time", @@ -3661,6 +3683,7 @@ dependencies = [ "tokio-stream", "tokio-util", "tonic", + "tower", "tracing", "tracing-error", "transform-stream", @@ -3715,9 +3738,9 @@ checksum = "a3d8a32ae18130a3c84dd492d4215c3d913c3b07c6b63c2eb3eb7ff1101ab7bf" [[package]] name = "enumflags2" -version = "0.7.12" +version = "0.7.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1027f7680c853e056ebcec683615fb6fbbc07dbaa13b4d5d9442b146ded4ecef" +checksum = "ba2f4b465f5318854c6f8dd686ede6c0a9dc67d4b1ac241cf0eb51521a309147" dependencies = [ "enumflags2_derive", "serde", @@ -3725,13 +3748,13 @@ dependencies = [ [[package]] name = "enumflags2_derive" -version = "0.7.12" +version = "0.7.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67c78a4d8fdf9953a5c9d458f9efe940fd97a0cab0941c075a813ac594733827" +checksum = "fc4caf64a58d7a6d65ab00639b046ff54399a39f5f2554728895ace4b297cd79" dependencies = [ "proc-macro2", "quote", - "syn 2.0.103", + "syn 2.0.101", ] [[package]] @@ -3752,7 +3775,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.103", + "syn 2.0.101", ] [[package]] @@ -3963,7 +3986,7 @@ checksum = "1a5c6c585bc94aaf2c7b51dd4c2ba22680844aba4c687be581871a6f518c5742" dependencies = [ "proc-macro2", "quote", - "syn 2.0.103", + "syn 2.0.101", ] [[package]] @@ -4085,7 +4108,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.103", + "syn 2.0.101", ] [[package]] @@ -4232,7 +4255,7 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a673cf4fb0ea6a91aa86c08695756dfe875277a912cdbf33db9a9f62d47ed82b" dependencies = [ - "parking_lot", + "parking_lot 0.12.4", "tracing", ] @@ -4266,7 +4289,7 @@ dependencies = [ "cfg-if", "js-sys", "libc", - "wasi 0.11.1+wasi-snapshot-preview1", + "wasi 0.11.0+wasi-snapshot-preview1", "wasm-bindgen", ] @@ -4366,7 +4389,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.103", + "syn 2.0.101", ] [[package]] @@ -4402,30 +4425,6 @@ dependencies = [ "x11-dl", ] -[[package]] -name = "globset" -version = "0.4.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54a1028dfc5f5df5da8a56a73e6c153c9a9708ec57232470703592a3f18e49f5" -dependencies = [ - "aho-corasick", - "bstr", - "log", - "regex-automata 0.4.9", - "regex-syntax 0.8.5", -] - -[[package]] -name = "globwalk" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93e3af942408868f6934a7b85134a3230832b9977cf66125df2f9edcfce4ddcc" -dependencies = [ - "bitflags 1.3.2", - "ignore", - "walkdir", -] - [[package]] name = "gloo-net" version = "0.6.0" @@ -4543,7 +4542,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.103", + "syn 2.0.101", ] [[package]] @@ -4600,6 +4599,9 @@ name = "hashbrown" version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +dependencies = [ + "ahash 0.7.8", +] [[package]] name = "hashbrown" @@ -4607,7 +4609,7 @@ version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" dependencies = [ - "ahash", + "ahash 0.8.12", "allocator-api2", ] @@ -4622,6 +4624,15 @@ dependencies = [ "foldhash", ] +[[package]] +name = "hashlink" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1" +dependencies = [ + "hashbrown 0.15.4", +] + [[package]] name = "heck" version = "0.4.1" @@ -4636,9 +4647,9 @@ checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] name = "hermit-abi" -version = "0.5.2" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" +checksum = "f154ce46856750ed433c8649605bf7ed2de3bc35fd9d2a9f30cddd873c80cb08" [[package]] name = "hex" @@ -4856,7 +4867,8 @@ dependencies = [ "http 1.3.1", "hyper 1.6.0", "hyper-util", - "rustls 0.23.28", + "log", + "rustls 0.23.27", "rustls-native-certs 0.8.1", "rustls-pki-types", "tokio", @@ -4911,7 +4923,6 @@ dependencies = [ "arc-swap", "async-trait", "base64-simd", - "common", "crypto", "ecstore", "futures", @@ -5071,22 +5082,6 @@ dependencies = [ "icu_properties", ] -[[package]] -name = "ignore" -version = "0.4.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d89fd380afde86567dfba715db065673989d6253f42b88179abd3eae47bda4b" -dependencies = [ - "crossbeam-deque", - "globset", - "log", - "memchr", - "regex-automata 0.4.9", - "same-file", - "walkdir", - "winapi-util", -] - [[package]] name = "include_dir" version = "0.7.4" @@ -5114,6 +5109,7 @@ checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" dependencies = [ "autocfg", "hashbrown 0.12.3", + "serde", ] [[package]] @@ -5124,6 +5120,7 @@ checksum = "cea70ddb795996207ad57735b50c5982d8844f38ba9ee5f1aedcfb708a2aa11e" dependencies = [ "equivalent", "hashbrown 0.15.4", + "serde", ] [[package]] @@ -5236,15 +5233,6 @@ dependencies = [ "either", ] -[[package]] -name = "itertools" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" -dependencies = [ - "either", -] - [[package]] name = "itertools" version = "0.12.1" @@ -5349,6 +5337,17 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "json5" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96b0db21af676c1ce64250b5f40f3ce2cf27e4e47cb91ed91eb6fe9350b430c1" +dependencies = [ + "pest", + "pest_derive", + "serde", +] + [[package]] name = "jsonwebtoken" version = "9.3.1" @@ -5533,9 +5532,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.174" +version = "0.2.172" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1171693293099992e19cddea4e8b849964e9846f4acee11b3948bcc337be8776" +checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa" [[package]] name = "libdbus-sys" @@ -5563,7 +5562,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07033963ba89ebaf1584d767badaa2e8fcec21aedea6b8c0346d487d49c28667" dependencies = [ "cfg-if", - "windows-targets 0.53.2", + "windows-targets 0.53.0", ] [[package]] @@ -5580,7 +5579,7 @@ checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" dependencies = [ "bitflags 2.9.1", "libc", - "redox_syscall 0.5.13", + "redox_syscall 0.5.12", ] [[package]] @@ -5708,6 +5707,15 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b3bd0dd2cd90571056fdb71f6275fada10131182f84899f4b2a916e565d81d86" +[[package]] +name = "lru" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999beba7b6e8345721bd280141ed958096a2e4abdf74f67ff4ce49b4b54e47a" +dependencies = [ + "hashbrown 0.12.3", +] + [[package]] name = "lru" version = "0.12.5" @@ -5744,11 +5752,11 @@ dependencies = [ [[package]] name = "lz4_flex" -version = "0.11.4" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c592ad9fbc1b7838633b3ae55ce69b17d01150c72fcef229fbb819d39ee51ee" +checksum = "75761162ae2b0e580d7e7c390558127e5f01b4194debd6221fd8c207fc80e3f5" dependencies = [ - "twox-hash 2.1.1", + "twox-hash", ] [[package]] @@ -5835,7 +5843,7 @@ dependencies = [ "manganis-core", "proc-macro2", "quote", - "syn 2.0.103", + "syn 2.0.101", ] [[package]] @@ -5891,9 +5899,9 @@ checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" [[package]] name = "memchr" -version = "2.7.5" +version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" [[package]] name = "memoffset" @@ -5928,9 +5936,9 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.8.9" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +checksum = "3be647b768db090acb35d5ec5db2b0e1f1de11133ca123b9eacf5137868f892a" dependencies = [ "adler2", "simd-adler32", @@ -5944,7 +5952,7 @@ checksum = "78bed444cc8a2160f01cbcf811ef18cac863ad68ae8ca62092e8db51d51c761c" dependencies = [ "libc", "log", - "wasi 0.11.1+wasi-snapshot-preview1", + "wasi 0.11.0+wasi-snapshot-preview1", "windows-sys 0.59.0", ] @@ -6114,15 +6122,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "normpath" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8911957c4b1549ac0dc74e30db9c8b0e66ddcd6d7acc33098f4c63a64a6d7ed" -dependencies = [ - "windows-sys 0.59.0", -] - [[package]] name = "notify" version = "8.0.0" @@ -6322,7 +6321,7 @@ dependencies = [ "proc-macro-crate 3.3.0", "proc-macro2", "quote", - "syn 2.0.103", + "syn 2.0.101", ] [[package]] @@ -6587,7 +6586,7 @@ dependencies = [ "futures", "humantime", "itertools 0.13.0", - "parking_lot", + "parking_lot 0.12.4", "percent-encoding", "snafu", "tokio", @@ -6733,6 +6732,16 @@ dependencies = [ "num-traits", ] +[[package]] +name = "ordered-multimap" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49203cdcae0030493bad186b28da2fa25645fa276a51b6fec8010d281e02ef79" +dependencies = [ + "dlv-list", + "hashbrown 0.14.5", +] + [[package]] name = "ordered-stream" version = "0.2.0" @@ -6797,6 +6806,17 @@ version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" +[[package]] +name = "parking_lot" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" +dependencies = [ + "instant", + "lock_api", + "parking_lot_core 0.8.6", +] + [[package]] name = "parking_lot" version = "0.12.4" @@ -6804,7 +6824,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "70d58bf43669b5795d1576d0641cfb6fbb2057bf629506267a92807158584a13" dependencies = [ "lock_api", - "parking_lot_core", + "parking_lot_core 0.9.11", +] + +[[package]] +name = "parking_lot_core" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc" +dependencies = [ + "cfg-if", + "instant", + "libc", + "redox_syscall 0.2.16", + "smallvec", + "winapi", ] [[package]] @@ -6815,7 +6849,7 @@ checksum = "bc838d2a56b5b1a6c25f55575dfc605fabb63bb2365f6c2353ef9159aa69e4a5" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.5.13", + "redox_syscall 0.5.12", "smallvec", "windows-targets 0.52.6", ] @@ -6826,7 +6860,7 @@ version = "54.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bfb15796ac6f56b429fd99e33ba133783ad75b27c36b4b5ce06f1f82cc97754e" dependencies = [ - "ahash", + "ahash 0.8.12", "arrow-array", "arrow-buffer", "arrow-cast", @@ -6852,7 +6886,7 @@ dependencies = [ "snap", "thrift", "tokio", - "twox-hash 1.6.3", + "twox-hash", "zstd", ] @@ -6906,6 +6940,12 @@ dependencies = [ "once_cell", ] +[[package]] +name = "pathdiff" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3" + [[package]] name = "pbkdf2" version = "0.12.2" @@ -6941,6 +6981,51 @@ version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" +[[package]] +name = "pest" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "198db74531d58c70a361c42201efde7e2591e976d518caf7662a47dc5720e7b6" +dependencies = [ + "memchr", + "thiserror 2.0.12", + "ucd-trie", +] + +[[package]] +name = "pest_derive" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d725d9cfd79e87dccc9341a2ef39d1b6f6353d68c4b33c177febbe1a402c97c5" +dependencies = [ + "pest", + "pest_generator", +] + +[[package]] +name = "pest_generator" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db7d01726be8ab66ab32f9df467ae8b1148906685bbe75c82d1e65d7f5b3f841" +dependencies = [ + "pest", + "pest_meta", + "proc-macro2", + "quote", + "syn 2.0.101", +] + +[[package]] +name = "pest_meta" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f9f832470494906d1fca5329f8ab5791cc60beb230c74815dff541cbd2b5ca0" +dependencies = [ + "once_cell", + "pest", + "sha2 0.10.9", +] + [[package]] name = "petgraph" version = "0.7.1" @@ -7098,7 +7183,7 @@ checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" dependencies = [ "proc-macro2", "quote", - "syn 2.0.103", + "syn 2.0.101", ] [[package]] @@ -7209,7 +7294,6 @@ dependencies = [ "arc-swap", "async-trait", "base64-simd", - "common", "crypto", "futures", "ipnetwork", @@ -7311,12 +7395,12 @@ checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" [[package]] name = "prettyplease" -version = "0.2.34" +version = "0.2.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6837b9e10d61f45f987d50808f83d1ee3d206c66acf650c3e4ae2e1f6ddedf55" +checksum = "9dee91521343f4c5c6a63edd65e54f31f5c92fe8978c40a4282f8372194c6a7d" dependencies = [ "proc-macro2", - "syn 2.0.103", + "syn 2.0.101", ] [[package]] @@ -7394,7 +7478,7 @@ checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.103", + "syn 2.0.101", "version_check", ] @@ -7424,7 +7508,7 @@ dependencies = [ "prost", "prost-types", "regex", - "syn 2.0.103", + "syn 2.0.101", "tempfile", ] @@ -7438,7 +7522,7 @@ dependencies = [ "itertools 0.14.0", "proc-macro2", "quote", - "syn 2.0.103", + "syn 2.0.101", ] [[package]] @@ -7505,7 +7589,7 @@ dependencies = [ "derive_builder", "futures", "lazy_static", - "parking_lot", + "parking_lot 0.12.4", "s3s", "snafu", "tokio", @@ -7520,7 +7604,6 @@ checksum = "331e97a1af0bf59823e6eadffe373d7b27f485be8748f71471c662c1f269b7fb" dependencies = [ "memchr", "serde", - "tokio", ] [[package]] @@ -7535,7 +7618,7 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash 2.1.1", - "rustls 0.23.28", + "rustls 0.23.27", "socket2", "thiserror 2.0.12", "tokio", @@ -7555,7 +7638,7 @@ dependencies = [ "rand 0.9.1", "ring", "rustc-hash 2.1.1", - "rustls 0.23.28", + "rustls 0.23.27", "rustls-pki-types", "slab", "thiserror 2.0.12", @@ -7589,9 +7672,9 @@ dependencies = [ [[package]] name = "r-efi" -version = "5.3.0" +version = "5.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" +checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5" [[package]] name = "rand" @@ -7765,6 +7848,25 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "reader" +version = "0.0.1" +dependencies = [ + "async-trait", + "base64-simd", + "bytes", + "common", + "futures", + "hex-simd", + "md-5", + "pin-project-lite", + "s3s", + "sha2 0.11.0-pre.5", + "thiserror 2.0.12", + "tokio", + "tracing", +] + [[package]] name = "readme-rustdocifier" version = "0.1.1" @@ -7788,7 +7890,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "76009fbe0614077fc1a2ce255e3a1881a2e3a3527097d5dc6d8212c585e7e38b" dependencies = [ "quote", - "syn 2.0.103", + "syn 2.0.101", +] + +[[package]] +name = "redox_syscall" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +dependencies = [ + "bitflags 1.3.2", ] [[package]] @@ -7802,9 +7913,9 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.5.13" +version = "0.5.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d04b7d0ee6b4a0207a0a7adb104d23ecb0b47d6beae7152d0fa34b692b29fd6" +checksum = "928fca9cf2aa042393a8325b9ead81d2f0df4cb12e1e24cef072922ccd99c5af" dependencies = [ "bitflags 2.9.1", ] @@ -7820,6 +7931,21 @@ dependencies = [ "thiserror 2.0.12", ] +[[package]] +name = "reed-solomon-erasure" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7263373d500d4d4f505d43a2a662d475a894aa94503a1ee28e9188b5f3960d4f" +dependencies = [ + "cc", + "libc", + "libm", + "lru 0.7.8", + "parking_lot 0.11.2", + "smallvec", + "spin", +] + [[package]] name = "reed-solomon-simd" version = "3.0.1" @@ -7882,9 +8008,9 @@ checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] name = "reqwest" -version = "0.12.20" +version = "0.12.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eabf4c97d9130e2bf606614eb937e86edac8292eaa6f422f995d7e8de1eb1813" +checksum = "a2f8e5513d63f2e5b386eb5106dc67eaf3f84e95258e210489136b8b92ad6119" dependencies = [ "base64 0.22.1", "bytes", @@ -7899,14 +8025,16 @@ dependencies = [ "hyper 1.6.0", "hyper-rustls 0.27.7", "hyper-util", + "ipnet", "js-sys", "log", "mime", "mime_guess", + "once_cell", "percent-encoding", "pin-project-lite", "quinn", - "rustls 0.23.28", + "rustls 0.23.27", "rustls-pki-types", "serde", "serde_json", @@ -8020,6 +8148,18 @@ dependencies = [ "serde", ] +[[package]] +name = "ron" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b91f7eff05f748767f183df4320a63d6936e9c6107d97c9e6bdd9784f4289c94" +dependencies = [ + "base64 0.21.7", + "bitflags 2.9.1", + "serde", + "serde_derive", +] + [[package]] name = "rsa" version = "0.9.8" @@ -8079,7 +8219,7 @@ dependencies = [ "quote", "rust-embed-utils", "shellexpand", - "syn 2.0.103", + "syn 2.0.101", "walkdir", ] @@ -8094,64 +8234,21 @@ dependencies = [ ] [[package]] -name = "rust-i18n" -version = "3.1.5" +name = "rust-ini" +version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fda2551fdfaf6cc5ee283adc15e157047b92ae6535cf80f6d4962d05717dc332" +checksum = "4e310ef0e1b6eeb79169a1171daf9abcb87a2e17c03bee2c4bb100b55c75409f" dependencies = [ - "globwalk", - "once_cell", - "regex", - "rust-i18n-macro", - "rust-i18n-support", - "smallvec", -] - -[[package]] -name = "rust-i18n-macro" -version = "3.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22baf7d7f56656d23ebe24f6bb57a5d40d2bce2a5f1c503e692b5b2fa450f965" -dependencies = [ - "glob", - "once_cell", - "proc-macro2", - "quote", - "rust-i18n-support", - "serde", - "serde_json", - "serde_yaml", - "syn 2.0.103", -] - -[[package]] -name = "rust-i18n-support" -version = "3.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "940ed4f52bba4c0152056d771e563b7133ad9607d4384af016a134b58d758f19" -dependencies = [ - "arc-swap", - "base62", - "globwalk", - "itertools 0.11.0", - "lazy_static", - "normpath", - "once_cell", - "proc-macro2", - "regex", - "serde", - "serde_json", - "serde_yaml", - "siphasher 1.0.1", - "toml", - "triomphe", + "cfg-if", + "ordered-multimap", + "trim-in-place", ] [[package]] name = "rustc-demangle" -version = "0.1.25" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "989e6739f80c4ad5b13e0fd7fe89531180375b18520cc8c82080e4dc4035b84f" +checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" [[package]] name = "rustc-hash" @@ -8187,7 +8284,6 @@ dependencies = [ "axum", "axum-extra", "axum-server", - "base64 0.22.1", "bytes", "chrono", "clap", @@ -8199,7 +8295,6 @@ dependencies = [ "flatbuffers 25.2.10", "futures", "futures-util", - "hmac 0.12.1", "http 1.3.1", "http-body 1.0.1", "hyper 1.6.0", @@ -8226,18 +8321,17 @@ dependencies = [ "rmp-serde", "rust-embed", "rustfs-config", + "rustfs-event-notifier", "rustfs-filemeta", - "rustfs-notify", "rustfs-obs", "rustfs-rio", "rustfs-utils", "rustfs-zip", - "rustls 0.23.28", + "rustls 0.23.27", "s3s", "serde", "serde_json", "serde_urlencoded", - "sha2 0.10.9", "shadow-rs", "socket2", "thiserror 2.0.12", @@ -8262,22 +8356,48 @@ dependencies = [ name = "rustfs-config" version = "0.0.1" dependencies = [ + "config", "const-str", "serde", "serde_json", ] +[[package]] +name = "rustfs-event-notifier" +version = "0.0.1" +dependencies = [ + "async-trait", + "axum", + "config", + "dotenvy", + "http 1.3.1", + "rdkafka", + "reqwest", + "rumqttc", + "serde", + "serde_json", + "serde_with", + "smallvec", + "strum", + "thiserror 2.0.12", + "tokio", + "tokio-util", + "tracing", + "tracing-subscriber", + "uuid", +] + [[package]] name = "rustfs-filemeta" version = "0.0.1" dependencies = [ "byteorder", - "bytes", "crc32fast", "criterion", "rmp", "rmp-serde", "rustfs-utils", + "s3s", "serde", "thiserror 2.0.12", "time", @@ -8299,7 +8419,6 @@ dependencies = [ "lazy_static", "rfd 0.15.3", "rust-embed", - "rust-i18n", "serde", "serde_json", "sha2 0.10.9", @@ -8308,35 +8427,6 @@ dependencies = [ "tracing-subscriber", ] -[[package]] -name = "rustfs-notify" -version = "0.0.1" -dependencies = [ - "async-trait", - "axum", - "chrono", - "const-str", - "dashmap 6.1.0", - "ecstore", - "form_urlencoded", - "once_cell", - "quick-xml", - "reqwest", - "rumqttc", - "rustfs-utils", - "serde", - "serde_json", - "snap", - "thiserror 2.0.12", - "tokio", - "tracing", - "tracing-subscriber", - "url", - "urlencoding", - "uuid", - "wildmatch", -] - [[package]] name = "rustfs-obs" version = "0.0.1" @@ -8344,7 +8434,6 @@ dependencies = [ "async-trait", "chrono", "flexi_logger", - "lazy_static", "nu-ansi-term 0.50.1", "nvml-wrapper", "opentelemetry", @@ -8377,23 +8466,25 @@ dependencies = [ "aes-gcm", "async-trait", "base64-simd", - "byteorder", + "brotli 8.0.1", "bytes", "crc32fast", "criterion", + "flate2", "futures", "hex-simd", "http 1.3.1", + "lz4", "md-5", "pin-project-lite", "rand 0.9.1", "reqwest", "rustfs-utils", - "serde", - "serde_json", + "snap", "tokio", "tokio-test", "tokio-util", + "zstd", ] [[package]] @@ -8418,7 +8509,7 @@ dependencies = [ "regex", "reqwest", "serde", - "serde-xml-rs", + "serde-xml-rs 0.6.0", "sha2 0.10.9", "urlencoding", ] @@ -8429,37 +8520,33 @@ version = "0.0.1" dependencies = [ "base64-simd", "blake3", - "brotli 8.0.1", - "bytes", + "common", "crc32fast", - "flate2", - "futures", "hex-simd", "highway", + "hmac 0.12.1", + "hyper 1.6.0", + "hyper-util", "lazy_static", "local-ip-address", - "lz4", "md-5", "netif", "nix 0.30.1", - "rand 0.9.1", "regex", "rustfs-config", - "rustls 0.23.28", + "rustls 0.23.27", "rustls-pemfile 2.2.0", "rustls-pki-types", + "s3s", "serde", + "sha1 0.10.6", "sha2 0.10.9", "siphasher 1.0.1", - "snap", - "sysinfo", "tempfile", "tokio", "tracing", - "transform-stream", "url", "winapi", - "zstd", ] [[package]] @@ -8529,9 +8616,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.28" +version = "0.23.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7160e3e10bf4535308537f3c4e1641468cd0e485175d6163087c0393c7d46643" +checksum = "730944ca083c1c233a75c09f199e973ca499344a2b7ba9e755c457e86fb4a321" dependencies = [ "aws-lc-rs", "log", @@ -8840,27 +8927,6 @@ dependencies = [ "serde_derive", ] -[[package]] -name = "serde-untagged" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "299d9c19d7d466db4ab10addd5703e4c615dec2a5a16dbbafe191045e87ee66e" -dependencies = [ - "erased-serde", - "serde", - "typeid", -] - -[[package]] -name = "serde-value" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3a1a3341211875ef120e117ea7fd5228530ae7e7036a779fdc9117be6b3282c" -dependencies = [ - "ordered-float", - "serde", -] - [[package]] name = "serde-wasm-bindgen" version = "0.5.0" @@ -8884,6 +8950,18 @@ dependencies = [ "xml-rs", ] +[[package]] +name = "serde-xml-rs" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53630160a98edebde0123eb4dfd0fce6adff091b2305db3154a9e920206eb510" +dependencies = [ + "log", + "serde", + "thiserror 1.0.69", + "xml-rs", +] + [[package]] name = "serde_derive" version = "1.0.219" @@ -8892,7 +8970,7 @@ checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" dependencies = [ "proc-macro2", "quote", - "syn 2.0.103", + "syn 2.0.101", ] [[package]] @@ -8945,7 +9023,7 @@ checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.103", + "syn 2.0.101", ] [[package]] @@ -8970,16 +9048,33 @@ dependencies = [ ] [[package]] -name = "serde_yaml" -version = "0.9.34+deprecated" +name = "serde_with" +version = "3.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" +checksum = "d6b6f7f2fcb69f747921f79f3926bd1e203fce4fef62c268dd3abfb6d86029aa" dependencies = [ + "base64 0.22.1", + "chrono", + "hex", + "indexmap 1.9.3", "indexmap 2.9.0", - "itoa 1.0.15", - "ryu", "serde", - "unsafe-libyaml", + "serde_derive", + "serde_json", + "serde_with_macros", + "time", +] + +[[package]] +name = "serde_with_macros" +version = "3.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d00caa5193a3c8362ac2b73be6b9e768aa5a4b2f721d8f4b339600c3cb51f8e" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn 2.0.101", ] [[package]] @@ -9021,7 +9116,7 @@ dependencies = [ "convert_case 0.6.0", "proc-macro2", "quote", - "syn 2.0.103", + "syn 2.0.101", "xxhash-rust", ] @@ -9032,7 +9127,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f2aa8119b558a17992e0ac1fd07f080099564f24532858811ce04f742542440" dependencies = [ "server_fn_macro", - "syn 2.0.103", + "syn 2.0.101", ] [[package]] @@ -9091,9 +9186,9 @@ dependencies = [ [[package]] name = "shadow-rs" -version = "1.2.0" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f6fd27df794ced2ef39872879c93a9f87c012607318af8621cd56d2c3a8b3a2" +checksum = "6d5625ed609cf66d7e505e7d487aca815626dc4ebb6c0dd07637ca61a44651a6" dependencies = [ "cargo_metadata", "const_format", @@ -9204,9 +9299,12 @@ checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d" [[package]] name = "slab" -version = "0.4.10" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04dc19736151f35336d325007ac991178d504a119863a2fcb3758cdb5e52c50d" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" +dependencies = [ + "autocfg", +] [[package]] name = "sledgehammer_bindgen" @@ -9225,7 +9323,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f62f06db0370222f7f498ef478fce9f8df5828848d1d3517e3331936d7074f55" dependencies = [ "quote", - "syn 2.0.103", + "syn 2.0.101", ] [[package]] @@ -9275,7 +9373,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.103", + "syn 2.0.101", ] [[package]] @@ -9368,7 +9466,7 @@ checksum = "da5fc6819faabb412da764b99d3b713bb55083c11e7e0c00144d386cd6a1939c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.103", + "syn 2.0.101", ] [[package]] @@ -9413,7 +9511,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bf776ba3fa74f83bf4b63c3dcbbf82173db2632ed8452cb2d891d33f459de70f" dependencies = [ "new_debug_unreachable", - "parking_lot", + "parking_lot 0.12.4", "phf_shared 0.11.3", "precomputed-hash", "serde", @@ -9456,7 +9554,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.103", + "syn 2.0.101", ] [[package]] @@ -9556,9 +9654,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.103" +version = "2.0.101" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4307e30089d6fd6aff212f2da3a1f9e32f3223b1f010fb09b7c95f90f3ca1e8" +checksum = "8ce2b7fc941b3a24138a0a7cf8e858bfc6a992e7978a068a5c760deb0ed43caf" dependencies = [ "proc-macro2", "quote", @@ -9582,7 +9680,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.103", + "syn 2.0.101", ] [[package]] @@ -9596,7 +9694,7 @@ dependencies = [ "ntapi", "objc2-core-foundation", "objc2-io-kit", - "windows 0.61.3", + "windows 0.61.1", ] [[package]] @@ -9660,7 +9758,7 @@ dependencies = [ "ndk-sys", "objc", "once_cell", - "parking_lot", + "parking_lot 0.12.4", "raw-window-handle 0.5.2", "raw-window-handle 0.6.2", "scopeguard", @@ -9681,7 +9779,7 @@ checksum = "f4e16beb8b2ac17db28eab8bca40e62dbfbb34c0fcdc6d9826b11b7b5d047dfd" dependencies = [ "proc-macro2", "quote", - "syn 2.0.103", + "syn 2.0.101", ] [[package]] @@ -9690,15 +9788,6 @@ version = "0.12.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1" -[[package]] -name = "temp-env" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45107136c2ddf8c4b87453c02294fd0adf41751796e81e8ba3f7fd951977ab57" -dependencies = [ - "once_cell", -] - [[package]] name = "tempfile" version = "3.20.0" @@ -9741,7 +9830,7 @@ dependencies = [ "cfg-if", "proc-macro2", "quote", - "syn 2.0.103", + "syn 2.0.101", ] [[package]] @@ -9752,7 +9841,7 @@ checksum = "5c89e72a01ed4c579669add59014b9a524d609c0c88c6a585ce37485879f6ffb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.103", + "syn 2.0.101", "test-case-core", ] @@ -9788,7 +9877,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.103", + "syn 2.0.101", ] [[package]] @@ -9799,16 +9888,17 @@ checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.103", + "syn 2.0.101", ] [[package]] name = "thread_local" -version = "1.1.9" +version = "1.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c" dependencies = [ "cfg-if", + "once_cell", ] [[package]] @@ -9929,7 +10019,7 @@ dependencies = [ "bytes", "libc", "mio", - "parking_lot", + "parking_lot 0.12.4", "pin-project-lite", "signal-hook-registry", "socket2", @@ -9946,7 +10036,7 @@ checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.103", + "syn 2.0.101", ] [[package]] @@ -9976,7 +10066,7 @@ version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e727b36a1a0e8b74c376ac2211e40c2c8af09fb4013c60d910495810f008e9b" dependencies = [ - "rustls 0.23.28", + "rustls 0.23.27", "tokio", ] @@ -10087,7 +10177,7 @@ dependencies = [ "serde_spanned", "toml_datetime", "toml_write", - "winnow 0.7.11", + "winnow 0.7.10", ] [[package]] @@ -10137,7 +10227,7 @@ dependencies = [ "prost-build", "prost-types", "quote", - "syn 2.0.103", + "syn 2.0.101", ] [[package]] @@ -10220,13 +10310,13 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.30" +version = "0.1.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81383ab64e72a7a8b8e13130c49e3dab29def6d0c7d76a03087b3cf71c5c6903" +checksum = "1b1ffbcf9c6f6b99d386e7444eb608ba646ae452a36b39737deb9663b610f662" dependencies = [ "proc-macro2", "quote", - "syn 2.0.103", + "syn 2.0.101", ] [[package]] @@ -10352,15 +10442,10 @@ dependencies = [ ] [[package]] -name = "triomphe" -version = "0.1.14" +name = "trim-in-place" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef8f7726da4807b58ea5c96fdc122f80702030edc33b35aff9190a51148ccc85" -dependencies = [ - "arc-swap", - "serde", - "stable_deref_trait", -] +checksum = "343e926fc669bc8cde4fa3129ab681c63671bae288b1f1081ceee6d9d37904fc" [[package]] name = "try-lock" @@ -10396,12 +10481,6 @@ dependencies = [ "static_assertions", ] -[[package]] -name = "twox-hash" -version = "2.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b907da542cbced5261bd3256de1b3a1bf340a3d37f93425a07362a1d687de56" - [[package]] name = "typeid" version = "1.0.3" @@ -10440,6 +10519,12 @@ dependencies = [ "tz-rs", ] +[[package]] +name = "ucd-trie" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" + [[package]] name = "uds_windows" version = "1.1.0" @@ -10471,9 +10556,9 @@ checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" [[package]] name = "unicode-width" -version = "0.2.1" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a1a07cc7db3810833284e8d372ccdc6da29741639ecc70c9ec107df0fa6154c" +checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd" [[package]] name = "unicode-xid" @@ -10491,12 +10576,6 @@ dependencies = [ "subtle", ] -[[package]] -name = "unsafe-libyaml" -version = "0.2.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" - [[package]] name = "untrusted" version = "0.9.0" @@ -10561,7 +10640,7 @@ checksum = "26b682e8c381995ea03130e381928e0e005b7c9eb483c6c8682f50e07b33c2b7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.103", + "syn 2.0.101", ] [[package]] @@ -10668,7 +10747,7 @@ checksum = "59195a1db0e95b920366d949ba5e0d3fc0e70b67c09be15ce5abb790106b0571" dependencies = [ "proc-macro2", "quote", - "syn 2.0.103", + "syn 2.0.101", ] [[package]] @@ -10679,9 +10758,9 @@ checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" [[package]] name = "wasi" -version = "0.11.1+wasi-snapshot-preview1" +version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasi" @@ -10714,7 +10793,7 @@ dependencies = [ "log", "proc-macro2", "quote", - "syn 2.0.103", + "syn 2.0.101", "wasm-bindgen-shared", ] @@ -10749,7 +10828,7 @@ checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" dependencies = [ "proc-macro2", "quote", - "syn 2.0.103", + "syn 2.0.101", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -10888,7 +10967,7 @@ checksum = "1d228f15bba3b9d56dde8bddbee66fa24545bd17b48d5128ccf4a8742b18e431" dependencies = [ "proc-macro2", "quote", - "syn 2.0.103", + "syn 2.0.101", ] [[package]] @@ -10914,15 +10993,6 @@ dependencies = [ "rustix 0.38.44", ] -[[package]] -name = "wildmatch" -version = "2.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ce1ab1f8c62655ebe1350f589c61e505cf94d385bc6a12899442d9081e71fd" -dependencies = [ - "serde", -] - [[package]] name = "winapi" version = "0.3.9" @@ -10966,9 +11036,9 @@ dependencies = [ [[package]] name = "windows" -version = "0.61.3" +version = "0.61.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9babd3a767a4c1aef6900409f85f5d53ce2544ccdfaa86dad48c91782c6d6893" +checksum = "c5ee8f3d025738cb02bad7868bbb5f8a6327501e870bf51f1b455b0a2454a419" dependencies = [ "windows-collections", "windows-core 0.61.2", @@ -11031,7 +11101,7 @@ checksum = "2bbd5b46c938e506ecbce286b6628a02171d56153ba733b6c741fc627ec9579b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.103", + "syn 2.0.101", ] [[package]] @@ -11042,7 +11112,7 @@ checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836" dependencies = [ "proc-macro2", "quote", - "syn 2.0.103", + "syn 2.0.101", ] [[package]] @@ -11053,7 +11123,7 @@ checksum = "053c4c462dc91d3b1504c6fe5a726dd15e216ba718e84a0e46a88fbe5ded3515" dependencies = [ "proc-macro2", "quote", - "syn 2.0.103", + "syn 2.0.101", ] [[package]] @@ -11064,14 +11134,14 @@ checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.103", + "syn 2.0.101", ] [[package]] name = "windows-link" -version = "0.1.3" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a" +checksum = "76840935b766e1b0a05c0066835fb9ec80071d4c09a16f6bd5f7e655e3c14c38" [[package]] name = "windows-numerics" @@ -11167,15 +11237,6 @@ dependencies = [ "windows-targets 0.52.6", ] -[[package]] -name = "windows-sys" -version = "0.60.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" -dependencies = [ - "windows-targets 0.53.2", -] - [[package]] name = "windows-targets" version = "0.42.2" @@ -11224,9 +11285,9 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.53.2" +version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c66f69fcc9ce11da9966ddb31a40968cad001c5bedeb5c2b82ede4253ab48aef" +checksum = "b1e4c7e8ceaaf9cb7d7507c974735728ab453b67ef8f18febdd7c11fe59dca8b" dependencies = [ "windows_aarch64_gnullvm 0.53.0", "windows_aarch64_msvc 0.53.0", @@ -11447,9 +11508,9 @@ dependencies = [ [[package]] name = "winnow" -version = "0.7.11" +version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74c7b26e3480b707944fc872477815d29a8e429d2f93a1ce000f5fa84a15cbcd" +checksum = "c06928c8748d81b05c9be96aad92e1b6ff01833332f281e8cfca3be4b35fc9ec" dependencies = [ "memchr", ] @@ -11481,7 +11542,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.103", + "syn 2.0.101", ] [[package]] @@ -11598,6 +11659,17 @@ dependencies = [ "lzma-sys", ] +[[package]] +name = "yaml-rust2" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18b783b2c2789414f8bb84ca3318fc9c2d7e7be1c22907d37839a58dedb369d3" +dependencies = [ + "arraydeque", + "encoding_rs", + "hashlink", +] + [[package]] name = "yoke" version = "0.8.0" @@ -11618,7 +11690,7 @@ checksum = "38da3c9736e16c5d3c8c597a9aaa5d1fa565d0532ae05e27c24aa62fb32c0ab6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.103", + "syn 2.0.101", "synstructure", ] @@ -11677,7 +11749,7 @@ dependencies = [ "tracing", "uds_windows", "windows-sys 0.59.0", - "winnow 0.7.11", + "winnow 0.7.10", "zbus_macros 5.7.1", "zbus_names 4.2.0", "zvariant 5.5.3", @@ -11692,7 +11764,7 @@ dependencies = [ "proc-macro-crate 3.3.0", "proc-macro2", "quote", - "syn 2.0.103", + "syn 2.0.101", "zvariant_utils 2.1.0", ] @@ -11705,7 +11777,7 @@ dependencies = [ "proc-macro-crate 3.3.0", "proc-macro2", "quote", - "syn 2.0.103", + "syn 2.0.101", "zbus_names 4.2.0", "zvariant 5.5.3", "zvariant_utils 3.2.0", @@ -11730,28 +11802,28 @@ checksum = "7be68e64bf6ce8db94f63e72f0c7eb9a60d733f7e0499e628dfab0f84d6bcb97" dependencies = [ "serde", "static_assertions", - "winnow 0.7.11", + "winnow 0.7.10", "zvariant 5.5.3", ] [[package]] name = "zerocopy" -version = "0.8.26" +version = "0.8.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1039dd0d3c310cf05de012d8a39ff557cb0d23087fd44cad61df08fc31907a2f" +checksum = "a1702d9583232ddb9174e01bb7c15a2ab8fb1bc6f227aa1233858c351a3ba0cb" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.26" +version = "0.8.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ecf5b4cc5364572d7f4c329661bcc82724222973f2cab6f050a4e5c22f75181" +checksum = "28a6e20d751156648aa063f3800b706ee209a32c0b4d9f24be3d980b01be55ef" dependencies = [ "proc-macro2", "quote", - "syn 2.0.103", + "syn 2.0.101", ] [[package]] @@ -11771,7 +11843,7 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn 2.0.103", + "syn 2.0.101", "synstructure", ] @@ -11792,7 +11864,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.103", + "syn 2.0.101", ] [[package]] @@ -11825,7 +11897,7 @@ checksum = "5b96237efa0c878c64bd89c436f661be4e46b2f3eff1ebb976f7ef2321d2f58f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.103", + "syn 2.0.101", ] [[package]] @@ -11922,7 +11994,7 @@ dependencies = [ "enumflags2", "serde", "url", - "winnow 0.7.11", + "winnow 0.7.10", "zvariant_derive 5.5.3", "zvariant_utils 3.2.0", ] @@ -11936,7 +12008,7 @@ dependencies = [ "proc-macro-crate 3.3.0", "proc-macro2", "quote", - "syn 2.0.103", + "syn 2.0.101", "zvariant_utils 2.1.0", ] @@ -11949,7 +12021,7 @@ dependencies = [ "proc-macro-crate 3.3.0", "proc-macro2", "quote", - "syn 2.0.103", + "syn 2.0.101", "zvariant_utils 3.2.0", ] @@ -11961,7 +12033,7 @@ checksum = "c51bcff7cc3dbb5055396bcf774748c3dab426b4b8659046963523cee4808340" dependencies = [ "proc-macro2", "quote", - "syn 2.0.103", + "syn 2.0.101", ] [[package]] @@ -11974,6 +12046,6 @@ dependencies = [ "quote", "serde", "static_assertions", - "syn 2.0.103", - "winnow 0.7.11", + "syn 2.0.101", + "winnow 0.7.10", ] diff --git a/Cargo.toml b/Cargo.toml index c688354ce..2c41ad628 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,7 +21,7 @@ members = [ "rustfs", # Core file system implementation "s3select/api", # S3 Select API interface "s3select/query", # S3 Select query engine - + "reader", ] resolver = "2" @@ -60,6 +60,7 @@ rustfs-utils = { path = "crates/utils", version = "0.0.1" } rustfs-rio = { path = "crates/rio", version = "0.0.1" } rustfs-filemeta = { path = "crates/filemeta", version = "0.0.1" } workers = { path = "./common/workers", version = "0.0.1" } +reader = { path = "./reader", version = "0.0.1" } aes-gcm = { version = "0.10.3", features = ["std"] } arc-swap = "1.7.1" argon2 = { version = "0.5.3", features = ["std"] } @@ -90,7 +91,7 @@ derive_builder = "0.20.2" dioxus = { version = "0.6.3", features = ["router"] } dirs = "6.0.0" flatbuffers = "25.2.10" -flexi_logger = { version = "0.30.2", features = ["trc", "dont_minimize_extra_stacks"] } +flexi_logger = { version = "0.30.2", features = ["trc","dont_minimize_extra_stacks"] } form_urlencoded = "1.2.1" futures = "0.3.31" futures-core = "0.3.31" @@ -106,6 +107,7 @@ hyper-util = { version = "0.1.14", features = [ "server-auto", "server-graceful", ] } +hyper-rustls = "0.27.5" http = "1.3.1" http-body = "1.0.1" humantime = "2.2.0" @@ -196,8 +198,12 @@ scopeguard = "1.2.0" shadow-rs = { version = "1.1.1", default-features = false } serde = { version = "1.0.219", features = ["derive"] } serde_json = "1.0.140" +serde-xml-rs = "0.8.1" serde_urlencoded = "0.7.1" +sha1 = "0.10.6" sha2 = "0.10.9" +hmac = "0.12.1" +std-next = "0.1.8" siphasher = "1.0.1" smallvec = { version = "1.15.1", features = ["serde"] } snafu = "0.8.6" @@ -223,6 +229,7 @@ tokio-tar = "0.3.1" tokio-util = { version = "0.7.15", features = ["io", "compat"] } tonic = { version = "0.13.1", features = ["gzip"] } tonic-build = { version = "0.13.1" } +async-channel = "2.3.1" tower = { version = "0.5.2", features = ["timeout"] } tower-http = { version = "0.6.6", features = ["cors"] } tracing = "0.1.41" diff --git a/common/lock/src/local_locker.rs b/common/lock/src/local_locker.rs index 3e50d0a2b..3ab4c7373 100644 --- a/common/lock/src/local_locker.rs +++ b/common/lock/src/local_locker.rs @@ -7,7 +7,7 @@ use std::{ use crate::{Locker, lock_args::LockArgs}; -const MAX_DELETE_LIST: usize = 1000; +pub const MAX_DELETE_LIST: usize = 1000; #[derive(Clone, Debug)] struct LockRequesterInfo { diff --git a/crates/filemeta/Cargo.toml b/crates/filemeta/Cargo.toml index 6f5e581aa..03e2310e0 100644 --- a/crates/filemeta/Cargo.toml +++ b/crates/filemeta/Cargo.toml @@ -21,6 +21,8 @@ byteorder = "1.5.0" tracing.workspace = true thiserror.workspace = true +s3s.workspace = true + [dev-dependencies] criterion = { version = "0.5", features = ["html_reports"] } diff --git a/crates/filemeta/src/fileinfo.rs b/crates/filemeta/src/fileinfo.rs index d0207b6b9..108a1d4db 100644 --- a/crates/filemeta/src/fileinfo.rs +++ b/crates/filemeta/src/fileinfo.rs @@ -17,6 +17,10 @@ pub const BLOCK_SIZE_V2: usize = 1024 * 1024; // 1M pub const NULL_VERSION_ID: &str = "null"; // pub const RUSTFS_ERASURE_UPGRADED: &str = "x-rustfs-internal-erasure-upgraded"; +pub const TIER_FV_ID: &str = "tier-free-versionID"; +pub const TIER_FV_MARKER: &str = "tier-free-marker"; +pub const TIER_SKIP_FV_ID: &str = "tier-skip-fvid"; + #[derive(Serialize, Deserialize, Debug, PartialEq, Clone, Default)] pub struct ObjectPartInfo { pub etag: String, @@ -152,11 +156,10 @@ pub struct FileInfo { pub version_id: Option, pub is_latest: bool, pub deleted: bool, - // Transition related fields - pub transition_status: Option, - pub transitioned_obj_name: Option, - pub transition_tier: Option, - pub transition_version_id: Option, + pub transition_status: String, + pub transitioned_objname: String, + pub transition_tier: String, + pub transition_version_id: Option, pub expire_restored: bool, pub data_dir: Option, pub mod_time: Option, @@ -225,7 +228,11 @@ impl FileInfo { } pub fn get_etag(&self) -> Option { - self.metadata.get("etag").cloned() + if let Some(meta) = self.metadata.get("etag") { + Some(meta.clone()) + } else { + None + } } pub fn write_quorum(&self, quorum: usize) -> usize { @@ -307,6 +314,35 @@ impl FileInfo { self.metadata.insert(RUSTFS_HEALING.to_string(), "true".to_string()); } + pub fn set_tier_free_version_id(&mut self, version_id: &str) { + self.metadata + .insert(format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TIER_FV_ID), version_id.to_string()); + } + + pub fn tier_free_version_id(&self) -> String { + self.metadata[&format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TIER_FV_ID)].clone() + } + + pub fn set_tier_free_version(&mut self) { + self.metadata + .insert(format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TIER_FV_MARKER), "".to_string()); + } + + pub fn set_skip_tier_free_version(&mut self) { + self.metadata + .insert(format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TIER_SKIP_FV_ID), "".to_string()); + } + + pub fn skip_tier_free_version(&self) -> bool { + self.metadata + .contains_key(&format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TIER_SKIP_FV_ID)) + } + + pub fn tier_free_version(&self) -> bool { + self.metadata + .contains_key(&format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TIER_FV_MARKER)) + } + pub fn set_inline_data(&mut self) { self.metadata .insert(format!("{}inline-data", RESERVED_METADATA_PREFIX_LOWER).to_owned(), "true".to_owned()); @@ -331,7 +367,7 @@ impl FileInfo { /// Check if the object is remote (transitioned to another tier) pub fn is_remote(&self) -> bool { - !self.transition_tier.as_ref().is_none_or(|s| s.is_empty()) + !self.transition_tier.is_empty() } /// Get the data directory for this object @@ -387,7 +423,7 @@ impl FileInfo { pub fn transition_info_equals(&self, other: &FileInfo) -> bool { self.transition_status == other.transition_status && self.transition_tier == other.transition_tier - && self.transitioned_obj_name == other.transitioned_obj_name + && self.transitioned_objname == other.transitioned_objname && self.transition_version_id == other.transition_version_id } diff --git a/crates/filemeta/src/filemeta.rs b/crates/filemeta/src/filemeta.rs index 308991ade..4abaf0e96 100644 --- a/crates/filemeta/src/filemeta.rs +++ b/crates/filemeta/src/filemeta.rs @@ -8,6 +8,7 @@ use crate::headers::{ use byteorder::ByteOrder; use bytes::Bytes; use rmp::Marker; +use s3s::header::X_AMZ_RESTORE; use serde::{Deserialize, Serialize}; use std::cmp::Ordering; use std::convert::TryFrom; @@ -38,6 +39,19 @@ const _XL_FLAG_INLINE_DATA: u8 = 1 << 2; const META_DATA_READ_DEFAULT: usize = 4 << 10; const MSGP_UINT32_SIZE: usize = 5; +pub const TRANSITION_COMPLETE: &str = "complete"; +pub const TRANSITION_PENDING: &str = "pending"; + +pub const FREE_VERSION: &str = "free-version"; + +pub const TRANSITION_STATUS: &str = "transition-status"; +pub const TRANSITIONED_OBJECTNAME: &str = "transitioned-object"; +pub const TRANSITIONED_VERSION_ID: &str = "transitioned-versionID"; +pub const TRANSITION_TIER: &str = "transition-tier"; + +const X_AMZ_RESTORE_EXPIRY_DAYS: &str = "X-Amz-Restore-Expiry-Days"; +const X_AMZ_RESTORE_REQUEST_DATE: &str = "X-Amz-Restore-Request-Date"; + // type ScanHeaderVersionFn = Box Result<()>>; #[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] @@ -468,6 +482,42 @@ impl FileMeta { Err(Error::other("add_version failed")) } + pub fn add_version_filemata(&mut self, ver: FileMetaVersion) -> Result<()> { + let mod_time = ver.get_mod_time().unwrap().nanosecond(); + if !ver.valid() { + return Err(Error::other("attempted to add invalid version")); + } + let encoded = ver.marshal_msg()?; + + if self.versions.len() + 1 > 100 { + return Err(Error::other( + "You've exceeded the limit on the number of versions you can create on this object", + )); + } + + self.versions.push(FileMetaShallowVersion { + header: FileMetaVersionHeader { + mod_time: Some(OffsetDateTime::from_unix_timestamp(-1)?), + ..Default::default() + }, + ..Default::default() + }); + + let len = self.versions.len(); + for (i, existing) in self.versions.iter().enumerate() { + if existing.header.mod_time.unwrap().nanosecond() <= mod_time { + let vers = self.versions[i..len - 1].to_vec(); + self.versions[i + 1..].clone_from_slice(vers.as_slice()); + self.versions[i] = FileMetaShallowVersion { + header: ver.header(), + meta: encoded, + }; + return Ok(()); + } + } + Err(Error::other("addVersion: Internal error, unable to add version")) + } + // delete_version deletes version, returns data_dir pub fn delete_version(&mut self, fi: &FileInfo) -> Result> { let mut ventry = FileMetaVersion::default(); @@ -503,6 +553,42 @@ impl FileMeta { } } + for (i, version) in self.versions.iter().enumerate() { + if version.header.version_type != VersionType::Object || version.header.version_id != fi.version_id { + continue; + } + + let mut ver = self.get_idx(i)?; + + if fi.expire_restored { + ver.object.as_mut().unwrap().remove_restore_hdrs(); + let _ = self.set_idx(i, ver.clone()); + } else if fi.transition_status == TRANSITION_COMPLETE { + ver.object.as_mut().unwrap().set_transition(fi); + ver.object.as_mut().unwrap().reset_inline_data(); + self.set_idx(i, ver.clone())?; + } else { + let vers = self.versions[i + 1..].to_vec(); + self.versions.extend(vers.iter().cloned()); + let (free_version, to_free) = ver.object.as_ref().unwrap().init_free_version(fi); + if to_free { + self.add_version_filemata(free_version)?; + } + } + + if fi.deleted { + self.add_version_filemata(ventry)?; + } + if self.shared_data_dir_count(ver.object.as_ref().unwrap().version_id, ver.object.as_ref().unwrap().data_dir) > 0 { + return Ok(None); + } + return Ok(ver.object.as_ref().unwrap().data_dir); + } + + if fi.deleted { + self.add_version_filemata(ventry)?; + } + Err(Error::FileVersionNotFound) } @@ -1850,10 +1936,29 @@ impl MetaObject { } } - /// Set transition metadata - pub fn set_transition(&mut self, _fi: &FileInfo) { - // Implementation for object lifecycle transitions - // This would handle storage class transitions + pub fn set_transition(&mut self, fi: &FileInfo) { + self.meta_sys.insert( + format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TRANSITION_STATUS), + fi.transition_status.as_bytes().to_vec(), + ); + self.meta_sys.insert( + format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TRANSITIONED_OBJECTNAME), + fi.transitioned_objname.as_bytes().to_vec(), + ); + self.meta_sys.insert( + format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TRANSITIONED_VERSION_ID), + fi.transition_version_id.unwrap().as_bytes().to_vec(), + ); + self.meta_sys.insert( + format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TRANSITION_TIER), + fi.transition_tier.as_bytes().to_vec(), + ); + } + + pub fn remove_restore_hdrs(&mut self) { + self.meta_user.remove(X_AMZ_RESTORE.as_str()); + self.meta_user.remove(X_AMZ_RESTORE_EXPIRY_DAYS); + self.meta_user.remove(X_AMZ_RESTORE_REQUEST_DATE); } pub fn uses_data_dir(&self) -> bool { @@ -1889,6 +1994,66 @@ impl MetaObject { let bytes = hash.to_le_bytes(); [bytes[0], bytes[1], bytes[2], bytes[3]] } + + pub fn init_free_version(&self, fi: &FileInfo) -> (FileMetaVersion, bool) { + if fi.skip_tier_free_version() { + return (FileMetaVersion::default(), false); + } + if let Some(status) = self + .meta_sys + .get(&format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TRANSITION_STATUS)) + { + if *status == TRANSITION_COMPLETE.as_bytes().to_vec() { + let vid = Uuid::parse_str(&fi.tier_free_version_id()); + if let Err(err) = vid { + panic!( + "Invalid Tier Object delete marker versionId {} {}", + fi.tier_free_version_id(), + err.to_string() + ); + } + let vid = vid.unwrap(); + let mut free_entry = FileMetaVersion { + version_type: VersionType::Delete, + write_version: 0, + ..Default::default() + }; + free_entry.delete_marker = Some(MetaDeleteMarker { + version_id: Some(vid), + mod_time: self.mod_time, + meta_sys: Some(HashMap::>::new()), + }); + + free_entry + .delete_marker + .as_mut() + .unwrap() + .meta_sys + .as_mut() + .unwrap() + .insert(format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, FREE_VERSION), vec![]); + let tier_key = format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TRANSITION_TIER); + let tier_obj_key = format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TRANSITIONED_OBJECTNAME); + let tier_obj_vid_key = format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TRANSITIONED_VERSION_ID); + + let aa = [tier_key, tier_obj_key, tier_obj_vid_key]; + for (k, v) in &self.meta_sys { + if aa.contains(&k) { + free_entry + .delete_marker + .as_mut() + .unwrap() + .meta_sys + .as_mut() + .unwrap() + .insert(k.clone(), v.clone()); + } + } + return (free_entry, true); + } + } + (FileMetaVersion::default(), false) + } } impl From for MetaObject { diff --git a/crates/filemeta/src/lib.rs b/crates/filemeta/src/lib.rs index 7f003680c..552370012 100644 --- a/crates/filemeta/src/lib.rs +++ b/crates/filemeta/src/lib.rs @@ -1,5 +1,5 @@ mod error; -mod fileinfo; +pub mod fileinfo; mod filemeta; mod filemeta_inline; pub mod headers; diff --git a/crates/utils/Cargo.toml b/crates/utils/Cargo.toml index 0ea415308..08339cbea 100644 --- a/crates/utils/Cargo.toml +++ b/crates/utils/Cargo.toml @@ -7,28 +7,27 @@ rust-version.workspace = true version.workspace = true [dependencies] -base64-simd = { workspace = true, optional = true } +base64-simd= { workspace = true , optional = true} blake3 = { workspace = true, optional = true } crc32fast.workspace = true -hex-simd = { workspace = true, optional = true } +hex-simd= { workspace = true , optional = true} highway = { workspace = true, optional = true } -lazy_static = { workspace = true, optional = true } +lazy_static= { workspace = true , optional = true} local-ip-address = { workspace = true, optional = true } md-5 = { workspace = true, optional = true } -netif = { workspace = true, optional = true } +netif= { workspace = true , optional = true} nix = { workspace = true, optional = true } -regex = { workspace = true, optional = true } +regex= { workspace = true, optional = true } rustfs-config = { workspace = true, features = ["constants"] } rustls = { workspace = true, optional = true } rustls-pemfile = { workspace = true, optional = true } rustls-pki-types = { workspace = true, optional = true } serde = { workspace = true, optional = true } -sha2 = { workspace = true, optional = true } siphasher = { workspace = true, optional = true } tempfile = { workspace = true, optional = true } tokio = { workspace = true, optional = true, features = ["io-util", "macros"] } tracing = { workspace = true } -url = { workspace = true, optional = true } +url = { workspace = true , optional = true} flate2 = { workspace = true, optional = true } brotli = { workspace = true, optional = true } zstd = { workspace = true, optional = true } @@ -39,6 +38,13 @@ futures = { workspace = true, optional = true } transform-stream = { workspace = true, optional = true } bytes = { workspace = true, optional = true } sysinfo = { workspace = true, optional = true } +hyper.workspace = true +hyper-util.workspace = true +common.workspace = true +sha1 = { workspace = true } +sha2 = { workspace = true, optional = true } +hmac.workspace = true +s3s.workspace = true [dev-dependencies] tempfile = { workspace = true } @@ -59,7 +65,7 @@ io = ["dep:tokio"] path = [] compress = ["dep:flate2", "dep:brotli", "dep:snap", "dep:lz4", "dep:zstd"] string = ["dep:regex", "dep:lazy_static", "dep:rand"] -crypto = ["dep:base64-simd", "dep:hex-simd"] +crypto = ["dep:base64-simd","dep:hex-simd"] hash = ["dep:highway", "dep:md-5", "dep:sha2", "dep:blake3", "dep:serde", "dep:siphasher"] os = ["dep:nix", "dep:tempfile", "winapi"] # operating system utilities integration = [] # integration test features diff --git a/crates/utils/src/crypto.rs b/crates/utils/src/crypto.rs index a075cbe36..0c271adf5 100644 --- a/crates/utils/src/crypto.rs +++ b/crates/utils/src/crypto.rs @@ -1,3 +1,8 @@ +use std::mem::MaybeUninit; + +use hex_simd::{AsOut, AsciiCase}; +use hyper::body::Bytes; + pub fn base64_encode(input: &[u8]) -> String { base64_simd::URL_SAFE_NO_PAD.encode_to_string(input) } @@ -24,6 +29,80 @@ pub fn hex(data: impl AsRef<[u8]>) -> String { // h.finish().unwrap() // } +/// verify sha256 checksum string +pub fn is_sha256_checksum(s: &str) -> bool { + // TODO: optimize + let is_lowercase_hex = |c: u8| matches!(c, b'0'..=b'9' | b'a'..=b'f'); + s.len() == 64 && s.as_bytes().iter().copied().all(is_lowercase_hex) +} + +/// `hmac_sha1(key, data)` +pub fn hmac_sha1(key: impl AsRef<[u8]>, data: impl AsRef<[u8]>) -> [u8; 20] { + use hmac::{Hmac, Mac}; + use sha1::Sha1; + + let mut m = >::new_from_slice(key.as_ref()).unwrap(); + m.update(data.as_ref()); + m.finalize().into_bytes().into() +} + +/// `hmac_sha256(key, data)` +pub fn hmac_sha256(key: impl AsRef<[u8]>, data: impl AsRef<[u8]>) -> [u8; 32] { + use hmac::{Hmac, Mac}; + use sha2::Sha256; + + let mut m = Hmac::::new_from_slice(key.as_ref()).unwrap(); + m.update(data.as_ref()); + m.finalize().into_bytes().into() +} + +/// `f(hex(src))` +fn hex_bytes32(src: impl AsRef<[u8]>, f: impl FnOnce(&str) -> R) -> R { + let buf: &mut [_] = &mut [MaybeUninit::uninit(); 64]; + let ans = hex_simd::encode_as_str(src.as_ref(), buf.as_out(), AsciiCase::Lower); + f(ans) +} + +#[cfg(not(all(feature = "openssl", not(windows))))] +fn sha256(data: &[u8]) -> impl AsRef<[u8; 32]> + use<> { + use sha2::{Digest, Sha256}; + ::digest(data) +} + +#[cfg(all(feature = "openssl", not(windows)))] +fn sha256(data: &[u8]) -> impl AsRef<[u8]> { + use openssl::hash::{Hasher, MessageDigest}; + let mut h = Hasher::new(MessageDigest::sha256()).unwrap(); + h.update(data).unwrap(); + h.finish().unwrap() +} + +#[cfg(not(all(feature = "openssl", not(windows))))] +fn sha256_chunk(chunk: &[Bytes]) -> impl AsRef<[u8; 32]> + use<> { + use sha2::{Digest, Sha256}; + let mut h = ::new(); + chunk.iter().for_each(|data| h.update(data)); + h.finalize() +} + +#[cfg(all(feature = "openssl", not(windows)))] +fn sha256_chunk(chunk: &[Bytes]) -> impl AsRef<[u8]> { + use openssl::hash::{Hasher, MessageDigest}; + let mut h = Hasher::new(MessageDigest::sha256()).unwrap(); + chunk.iter().for_each(|data| h.update(data).unwrap()); + h.finish().unwrap() +} + +/// `f(hex(sha256(data)))` +pub fn hex_sha256(data: &[u8], f: impl FnOnce(&str) -> R) -> R { + hex_bytes32(sha256(data).as_ref(), f) +} + +/// `f(hex(sha256(chunk)))` +pub fn hex_sha256_chunk(chunk: &[Bytes], f: impl FnOnce(&str) -> R) -> R { + hex_bytes32(sha256_chunk(chunk).as_ref(), f) +} + #[test] fn test_base64_encoding_decoding() { let original_uuid_timestamp = "c0194290-d911-45cb-8e12-79ec563f46a8x1735460504394878000"; diff --git a/crates/utils/src/hash.rs b/crates/utils/src/hash.rs index 182af0fd7..a2b503a02 100644 --- a/crates/utils/src/hash.rs +++ b/crates/utils/src/hash.rs @@ -93,6 +93,8 @@ impl HashAlgorithm { use crc32fast::Hasher; use siphasher::sip::SipHasher; +pub const EMPTY_STRING_SHA256_HASH: &str = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + pub fn sip_hash(key: &str, cardinality: usize, id: &[u8; 16]) -> usize { // 你的密钥,必须是 16 字节 diff --git a/crates/utils/src/lib.rs b/crates/utils/src/lib.rs index 0b05f712f..19d1a9b9d 100644 --- a/crates/utils/src/lib.rs +++ b/crates/utils/src/lib.rs @@ -7,6 +7,8 @@ pub mod net; #[cfg(feature = "net")] pub use net::*; +pub mod retry; + #[cfg(feature = "io")] pub mod io; diff --git a/crates/utils/src/net.rs b/crates/utils/src/net.rs index 81bb9f9de..1d8ac5fde 100644 --- a/crates/utils/src/net.rs +++ b/crates/utils/src/net.rs @@ -1,14 +1,21 @@ use bytes::Bytes; use futures::pin_mut; use futures::{Stream, StreamExt}; +use hyper::client::conn::http2::Builder; +use hyper_util::rt::TokioExecutor; use lazy_static::lazy_static; use std::{ - collections::HashSet, + collections::{HashMap, HashSet}, fmt::Display, net::{IpAddr, Ipv6Addr, SocketAddr, TcpListener, ToSocketAddrs}, }; use transform_stream::AsyncTryStream; -use url::Host; +use url::{Host, Url}; +//use hyper::{client::conn::http2::Builder, rt::Executor}; +//use tonic::{SharedExec, UserAgent}; +//use hyper_util::rt::TokioTimer; + +use s3s::header::X_AMZ_STORAGE_CLASS; lazy_static! { static ref LOCAL_IPS: Vec = must_get_local_ips().unwrap(); @@ -108,6 +115,111 @@ pub fn must_get_local_ips() -> std::io::Result> { } } +pub fn get_default_location(u: Url, region_override: &str) -> String { + todo!(); +} + +pub fn get_endpoint_url(endpoint: &str, secure: bool) -> Result { + let mut scheme = "https"; + if !secure { + scheme = "http"; + } + + let endpoint_url_str = format!("{scheme}://{endpoint}"); + let Ok(endpoint_url) = Url::parse(&endpoint_url_str) else { + return Err(std::io::Error::other("url parse error.")); + }; + + //is_valid_endpoint_url(endpoint_url)?; + Ok(endpoint_url) +} + +pub const DEFAULT_DIAL_TIMEOUT: i64 = 5; + +pub fn new_remotetarget_http_transport(insecure: bool) -> Builder { + todo!(); +} + +lazy_static! { + static ref SUPPORTED_QUERY_VALUES: HashMap = { + let mut m = HashMap::new(); + m.insert("attributes".to_string(), true); + m.insert("partNumber".to_string(), true); + m.insert("versionId".to_string(), true); + m.insert("response-cache-control".to_string(), true); + m.insert("response-content-disposition".to_string(), true); + m.insert("response-content-encoding".to_string(), true); + m.insert("response-content-language".to_string(), true); + m.insert("response-content-type".to_string(), true); + m.insert("response-expires".to_string(), true); + m + }; + static ref SUPPORTED_HEADERS: HashMap = { + let mut m = HashMap::new(); + m.insert("content-type".to_string(), true); + m.insert("cache-control".to_string(), true); + m.insert("content-encoding".to_string(), true); + m.insert("content-disposition".to_string(), true); + m.insert("content-language".to_string(), true); + m.insert("x-amz-website-redirect-location".to_string(), true); + m.insert("x-amz-object-lock-mode".to_string(), true); + m.insert("x-amz-metadata-directive".to_string(), true); + m.insert("x-amz-object-lock-retain-until-date".to_string(), true); + m.insert("expires".to_string(), true); + m.insert("x-amz-replication-status".to_string(), true); + m + }; + static ref SSE_HEADERS: HashMap = { + let mut m = HashMap::new(); + m.insert("x-amz-server-side-encryption".to_string(), true); + m.insert("x-amz-server-side-encryption-aws-kms-key-id".to_string(), true); + m.insert("x-amz-server-side-encryption-context".to_string(), true); + m.insert("x-amz-server-side-encryption-customer-algorithm".to_string(), true); + m.insert("x-amz-server-side-encryption-customer-key".to_string(), true); + m.insert("x-amz-server-side-encryption-customer-key-md5".to_string(), true); + m + }; +} + +pub fn is_standard_query_value(qs_key: &str) -> bool { + SUPPORTED_QUERY_VALUES[qs_key] +} + +const ALLOWED_CUSTOM_QUERY_PREFIX: &str = "x-"; + +pub fn is_custom_query_value(qs_key: &str) -> bool { + qs_key.starts_with(ALLOWED_CUSTOM_QUERY_PREFIX) +} + +pub fn is_storageclass_header(header_key: &str) -> bool { + header_key.to_lowercase() == X_AMZ_STORAGE_CLASS.as_str().to_lowercase() +} + +pub fn is_standard_header(header_key: &str) -> bool { + *SUPPORTED_HEADERS.get(&header_key.to_lowercase()).unwrap_or(&false) +} + +pub fn is_sse_header(header_key: &str) -> bool { + *SSE_HEADERS.get(&header_key.to_lowercase()).unwrap_or(&false) +} + +pub fn is_amz_header(header_key: &str) -> bool { + let key = header_key.to_lowercase(); + key.starts_with("x-amz-meta-") + || key.starts_with("x-amz-grant-") + || key == "x-amz-acl" + || is_sse_header(header_key) + || key.starts_with("x-amz-checksum-") +} + +pub fn is_rustfs_header(header_key: &str) -> bool { + header_key.to_lowercase().starts_with("x-rustfs-") +} + +pub fn is_rustfs_header(header_key: &str) -> bool { + header_key.to_lowercase().starts_with("x-rustfs-") +} + #[derive(Debug, Clone)] pub struct XHost { pub name: String, diff --git a/crates/utils/src/path.rs b/crates/utils/src/path.rs index 0c63b960a..f706d8cb2 100644 --- a/crates/utils/src/path.rs +++ b/crates/utils/src/path.rs @@ -253,6 +253,11 @@ pub fn dir(path: &str) -> String { let (a, _) = split(path); clean(a) } + +pub fn trim_etag(etag: &str) -> String { + etag.trim_matches('"').to_string() +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/utils/src/retry.rs b/crates/utils/src/retry.rs new file mode 100644 index 000000000..98411f4d4 --- /dev/null +++ b/crates/utils/src/retry.rs @@ -0,0 +1,175 @@ +use std::time::Duration; + +// MaxRetry is the maximum number of retries before stopping. +pub const MAX_RETRY: i64 = 10; + +// MaxJitter will randomize over the full exponential backoff time +pub const MAX_JITTER: f64 = 1.0; + +// NoJitter disables the use of jitter for randomizing the exponential backoff time +pub const NO_JITTER: f64 = 0.0; + +// DefaultRetryUnit - default unit multiplicative per retry. +// defaults to 200 * time.Millisecond +//const DefaultRetryUnit = 200 * time.Millisecond; + +// DefaultRetryCap - Each retry attempt never waits no longer than +// this maximum time duration. +//const DefaultRetryCap = time.Second; + +/* +struct Delay { + when: Instant, +} + +impl Future for Delay { + type Output = &'static str; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) + -> Poll<&'static str> + { + if Instant::now() >= self.when { + println!("Hello world"); + Poll::Ready("done") + } else { + // Ignore this line for now. + cx.waker().wake_by_ref(); + Poll::Pending + } + } +} + +struct RetryTimer { + rem: usize, + delay: Delay, +} + +impl RetryTimer { + fn new() -> Self { + Self { + rem: 3, + delay: Delay { when: Instant::now() } + } + } +} + +impl Stream for RetryTimer { + type Item = (); + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) + -> Poll> + { + if self.rem == 0 { + // No more delays + return Poll::Ready(None); + } + + match Pin::new(&mut self.delay).poll(cx) { + Poll::Ready(_) => { + let when = self.delay.when + Duration::from_millis(10); + self.delay = Delay { when }; + self.rem -= 1; + Poll::Ready(Some(())) + } + Poll::Pending => Poll::Pending, + } + } +}*/ + +pub fn new_retry_timer(max_retry: i32, base_sleep: Duration, max_sleep: Duration, jitter: f64) -> Vec { + /*attemptCh := make(chan int) + + exponentialBackoffWait := func(attempt int) time.Duration { + // normalize jitter to the range [0, 1.0] + if jitter < NoJitter { + jitter = NoJitter + } + if jitter > MaxJitter { + jitter = MaxJitter + } + + // sleep = random_between(0, min(maxSleep, base * 2 ** attempt)) + sleep := baseSleep * time.Duration(1< maxSleep { + sleep = maxSleep + } + if jitter != NoJitter { + sleep -= time.Duration(c.random.Float64() * float64(sleep) * jitter) + } + return sleep + } + + go func() { + defer close(attemptCh) + for i := 0; i < maxRetry; i++ { + select { + case attemptCh <- i + 1: + case <-ctx.Done(): + return + } + + select { + case <-time.After(exponentialBackoffWait(i)): + case <-ctx.Done(): + return + } + } + }() + return attemptCh*/ + todo!(); +} + +/*var retryableS3Codes = map[string]struct{}{ + "RequestError": {}, + "RequestTimeout": {}, + "Throttling": {}, + "ThrottlingException": {}, + "RequestLimitExceeded": {}, + "RequestThrottled": {}, + "InternalError": {}, + "ExpiredToken": {}, + "ExpiredTokenException": {}, + "SlowDown": {}, +} + +fn isS3CodeRetryable(s3Code string) (ok bool) { + _, ok = retryableS3Codes[s3Code] + return ok +} + +var retryableHTTPStatusCodes = map[int]struct{}{ + http.StatusRequestTimeout: {}, + 429: {}, // http.StatusTooManyRequests is not part of the Go 1.5 library, yet + 499: {}, // client closed request, retry. A non-standard status code introduced by nginx. + http.StatusInternalServerError: {}, + http.StatusBadGateway: {}, + http.StatusServiceUnavailable: {}, + http.StatusGatewayTimeout: {}, + 520: {}, // It is used by Cloudflare as a catch-all response for when the origin server sends something unexpected. + // Add more HTTP status codes here. +} + +fn isHTTPStatusRetryable(httpStatusCode int) (ok bool) { + _, ok = retryableHTTPStatusCodes[httpStatusCode] + return ok +} + +fn isRequestErrorRetryable(ctx context.Context, err error) bool { + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + // Retry if internal timeout in the HTTP call. + return ctx.Err() == nil + } + if ue, ok := err.(*url.Error); ok { + e := ue.Unwrap() + switch e.(type) { + // x509: certificate signed by unknown authority + case x509.UnknownAuthorityError: + return false + } + switch e.Error() { + case "http: server gave HTTP response to HTTPS client": + return false + } + } + return true +}*/ diff --git a/ecstore/Cargo.toml b/ecstore/Cargo.toml index 48e863589..33af4ffdc 100644 --- a/ecstore/Cargo.toml +++ b/ecstore/Cargo.toml @@ -32,9 +32,12 @@ serde.workspace = true time.workspace = true bytesize.workspace = true serde_json.workspace = true +serde-xml-rs.workspace = true +serde_urlencoded.workspace = true tracing-error.workspace = true s3s.workspace = true http.workspace = true +http-body-util = "0.1.1" highway = { workspace = true } url.workspace = true uuid = { workspace = true, features = ["v4", "fast-rng", "serde"] } @@ -53,16 +56,29 @@ tokio-util = { workspace = true, features = ["io", "compat"] } crc32fast = { workspace = true } siphasher = { workspace = true } base64-simd = { workspace = true } + base64 = { workspace = true } hmac = { workspace = true } sha2 = { workspace = true } +sha1 = { workspace = true } + hex-simd = { workspace = true } path-clean = { workspace = true } tempfile.workspace = true +hyper.workspace = true +hyper-util.workspace = true +hyper-rustls.workspace = true +rustls.workspace = true tokio = { workspace = true, features = ["io-util", "sync", "signal"] } tokio-stream = { workspace = true } tonic.workspace = true xxhash-rust = { workspace = true, features = ["xxh64", "xxh3"] } +tower.workspace = true +async-channel.workspace = true +num = "0.4.3" +enumset = "1.1.5" +hmac.workspace = true +std-next.workspace = true num_cpus = { workspace = true } rand.workspace = true pin-project-lite.workspace = true @@ -77,10 +93,11 @@ urlencoding = { workspace = true } smallvec = { workspace = true } shadow-rs.workspace = true rustfs-filemeta.workspace = true -rustfs-utils = { workspace = true, features = ["full"] } +rustfs-utils ={workspace = true, features=["full"]} rustfs-rio.workspace = true futures-util.workspace = true serde_urlencoded.workspace = true +reader = { workspace = true } [target.'cfg(not(windows))'.dependencies] nix = { workspace = true } diff --git a/ecstore/src/admin_server_info.rs b/ecstore/src/admin_server_info.rs index 577eac1f9..231926d16 100644 --- a/ecstore/src/admin_server_info.rs +++ b/ecstore/src/admin_server_info.rs @@ -23,7 +23,7 @@ use protos::{ }; use std::{ collections::{HashMap, HashSet}, - time::SystemTime, + time::{SystemTime, UNIX_EPOCH}, }; use time::OffsetDateTime; use tonic::Request; diff --git a/ecstore/src/bucket/lifecycle/bucket_lifecycle_audit.rs b/ecstore/src/bucket/lifecycle/bucket_lifecycle_audit.rs new file mode 100644 index 000000000..565d99758 --- /dev/null +++ b/ecstore/src/bucket/lifecycle/bucket_lifecycle_audit.rs @@ -0,0 +1,29 @@ +use super::lifecycle; + +#[derive(Debug, Clone, Default)] +pub enum LcEventSrc { + #[default] + None, + Heal, + Scanner, + Decom, + Rebal, + S3HeadObject, + S3GetObject, + S3ListObjects, + S3PutObject, + S3CopyObject, + S3CompleteMultipartUpload, +} + +#[derive(Clone, Debug, Default)] +pub struct LcAuditEvent { + pub event: lifecycle::Event, + pub source: LcEventSrc, +} + +impl LcAuditEvent { + pub fn new(event: lifecycle::Event, source: LcEventSrc) -> Self { + Self { event, source } + } +} diff --git a/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs b/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs new file mode 100644 index 000000000..32942e1aa --- /dev/null +++ b/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs @@ -0,0 +1,843 @@ +use async_channel::{Receiver as A_Receiver, Sender as A_Sender, bounded}; +use futures::Future; +use http::HeaderMap; +use lazy_static::lazy_static; +use s3s::Body; +use sha2::{Digest, Sha256}; +use std::any::{Any, TypeId}; +use std::collections::HashMap; +use std::env; +use std::io::{Cursor, Write}; +use std::pin::Pin; +use std::sync::atomic::{AtomicI32, AtomicI64, Ordering}; +use std::sync::{Arc, Mutex}; +use tokio::select; +use tokio::sync::mpsc::{Receiver, Sender}; +use tokio::sync::{RwLock, mpsc}; +use tracing::{error, info, warn}; +use uuid::Uuid; +use xxhash_rust::xxh64; + +use super::bucket_lifecycle_audit::{LcAuditEvent, LcEventSrc}; +use super::lifecycle::{self, ExpirationOptions, IlmAction, Lifecycle, TransitionOptions}; +use super::tier_last_day_stats::{DailyAllTierStats, LastDayTierStats}; +use super::tier_sweeper::{Jentry, delete_object_from_remote_tier}; +use crate::bucket::{metadata_sys::get_lifecycle_config, versioning_sys::BucketVersioningSys}; +use crate::client::object_api_utils::new_getobjectreader; +use crate::error::Error; +use crate::error::{error_resp_to_object_err, is_err_object_not_found, is_err_version_not_found, is_network_or_host_down}; +use crate::event::name::EventName; +use crate::event_notification::{EventArgs, send_event}; +use crate::global::GLOBAL_LocalNodeName; +use crate::global::{GLOBAL_LifecycleSys, GLOBAL_TierConfigMgr, get_global_deployment_id}; +use crate::heal::{ + data_scanner::{apply_expiry_on_non_transitioned_objects, apply_expiry_on_transitioned_object}, + data_scanner_metric::ScannerMetrics, + data_usage_cache::TierStats, +}; +use crate::store::ECStore; +use crate::store_api::StorageAPI; +use crate::store_api::{GetObjectReader, HTTPRangeSpec, ObjectInfo, ObjectOptions, ObjectToDelete}; +use crate::tier::warm_backend::WarmBackendGetOpts; +use s3s::dto::BucketLifecycleConfiguration; + +pub type TimeFn = Arc Pin + Send>> + Send + Sync + 'static>; +pub type TraceFn = + Arc) -> Pin + Send>> + Send + Sync + 'static>; +pub type ExpiryOpType = Box; + +static XXHASH_SEED: u64 = 0; + +const DISABLED: &str = "Disabled"; + +//pub const ERR_INVALID_STORAGECLASS: &str = "invalid storage class."; +pub const ERR_INVALID_STORAGECLASS: &str = "invalid tier."; + +lazy_static! { + pub static ref GLOBAL_ExpiryState: Arc> = ExpiryState::new(); + pub static ref GLOBAL_TransitionState: Arc = TransitionState::new(); +} + +pub struct LifecycleSys; + +impl LifecycleSys { + pub fn new() -> Arc { + Arc::new(Self) + } + + pub async fn get(&self, bucket: &str) -> Option { + let lc = get_lifecycle_config(bucket).await.expect("get_lifecycle_config err!").0; + Some(lc) + } + + pub fn trace(oi: &ObjectInfo) -> TraceFn { + todo!(); + } +} + +struct ExpiryTask { + obj_info: ObjectInfo, + event: lifecycle::Event, + src: LcEventSrc, +} + +impl ExpiryOp for ExpiryTask { + fn op_hash(&self) -> u64 { + let mut hasher = Sha256::new(); + let _ = hasher.write(format!("{}", self.obj_info.bucket).as_bytes()); + let _ = hasher.write(format!("{}", self.obj_info.name).as_bytes()); + hasher.flush(); + xxh64::xxh64(hasher.clone().finalize().as_slice(), XXHASH_SEED) + } + + fn as_any(&self) -> &dyn Any { + self + } +} + +struct ExpiryStats { + missed_expiry_tasks: AtomicI64, + missed_freevers_tasks: AtomicI64, + missed_tier_journal_tasks: AtomicI64, + workers: AtomicI64, +} + +impl ExpiryStats { + pub fn missed_tasks(&self) -> i64 { + self.missed_expiry_tasks.load(Ordering::SeqCst) + } + + fn missed_free_vers_tasks(&self) -> i64 { + self.missed_freevers_tasks.load(Ordering::SeqCst) + } + + fn missed_tier_journal_tasks(&self) -> i64 { + self.missed_tier_journal_tasks.load(Ordering::SeqCst) + } + + fn num_workers(&self) -> i64 { + self.workers.load(Ordering::SeqCst) + } +} + +pub trait ExpiryOp: 'static { + fn op_hash(&self) -> u64; + fn as_any(&self) -> &dyn Any; +} + +#[derive(Debug, Default, Clone)] +pub struct TransitionedObject { + pub name: String, + pub version_id: String, + pub tier: String, + pub free_version: bool, + pub status: String, +} + +struct FreeVersionTask(ObjectInfo); + +impl ExpiryOp for FreeVersionTask { + fn op_hash(&self) -> u64 { + let mut hasher = Sha256::new(); + let _ = hasher.write(format!("{}", self.0.transitioned_object.tier).as_bytes()); + let _ = hasher.write(format!("{}", self.0.transitioned_object.name).as_bytes()); + hasher.flush(); + xxh64::xxh64(hasher.clone().finalize().as_slice(), XXHASH_SEED) + } + + fn as_any(&self) -> &dyn Any { + self + } +} + +struct NewerNoncurrentTask { + bucket: String, + versions: Vec, + event: lifecycle::Event, +} + +impl ExpiryOp for NewerNoncurrentTask { + fn op_hash(&self) -> u64 { + let mut hasher = Sha256::new(); + let _ = hasher.write(format!("{}", self.bucket).as_bytes()); + let _ = hasher.write(format!("{}", self.versions[0].object_name).as_bytes()); + hasher.flush(); + xxh64::xxh64(hasher.clone().finalize().as_slice(), XXHASH_SEED) + } + + fn as_any(&self) -> &dyn Any { + self + } +} + +pub struct ExpiryState { + tasks_tx: Vec>>, + tasks_rx: Vec>>>>, + stats: Option, +} + +impl ExpiryState { + #[allow(clippy::new_ret_no_self)] + pub fn new() -> Arc> { + Arc::new(RwLock::new(Self { + tasks_tx: vec![], + tasks_rx: vec![], + stats: Some(ExpiryStats { + missed_expiry_tasks: AtomicI64::new(0), + missed_freevers_tasks: AtomicI64::new(0), + missed_tier_journal_tasks: AtomicI64::new(0), + workers: AtomicI64::new(0), + }), + })) + } + + pub async fn pending_tasks(&self) -> usize { + let rxs = &self.tasks_rx; + if rxs.len() == 0 { + return 0; + } + let mut tasks = 0; + for rx in rxs.iter() { + tasks += rx.lock().await.len(); + } + tasks + } + + pub async fn enqueue_tier_journal_entry(&mut self, je: &Jentry) -> Result<(), std::io::Error> { + let wrkr = self.get_worker_ch(je.op_hash()); + if wrkr.is_none() { + *self.stats.as_mut().expect("err").missed_tier_journal_tasks.get_mut() += 1; + } + let wrkr = wrkr.expect("err"); + select! { + //_ -> GlobalContext.Done() => () + _ = wrkr.send(Some(Box::new(je.clone()))) => (), + else => { + *self.stats.as_mut().expect("err").missed_tier_journal_tasks.get_mut() += 1; + } + } + return Ok(()); + } + + pub async fn enqueue_free_version(&mut self, oi: ObjectInfo) { + let task = FreeVersionTask(oi); + let wrkr = self.get_worker_ch(task.op_hash()); + if wrkr.is_none() { + *self.stats.as_mut().expect("err").missed_freevers_tasks.get_mut() += 1; + return; + } + let wrkr = wrkr.expect("err!"); + select! { + //_ -> GlobalContext.Done() => {} + _ = wrkr.send(Some(Box::new(task))) => (), + else => { + *self.stats.as_mut().expect("err").missed_freevers_tasks.get_mut() += 1; + } + } + } + + pub async fn enqueue_by_days(&mut self, oi: &ObjectInfo, event: &lifecycle::Event, src: &LcEventSrc) { + let task = ExpiryTask { + obj_info: oi.clone(), + event: event.clone(), + src: src.clone(), + }; + let wrkr = self.get_worker_ch(task.op_hash()); + if wrkr.is_none() { + *self.stats.as_mut().expect("err").missed_expiry_tasks.get_mut() += 1; + return; + } + let wrkr = wrkr.expect("err!"); + select! { + //_ -> GlobalContext.Done() => {} + _ = wrkr.send(Some(Box::new(task))) => (), + else => { + *self.stats.as_mut().expect("err").missed_expiry_tasks.get_mut() += 1; + } + } + } + + pub async fn enqueue_by_newer_noncurrent(&mut self, bucket: &str, versions: Vec, lc_event: lifecycle::Event) { + if versions.len() == 0 { + return; + } + + let task = NewerNoncurrentTask { + bucket: String::from(bucket), + versions: versions, + event: lc_event, + }; + let wrkr = self.get_worker_ch(task.op_hash()); + if wrkr.is_none() { + *self.stats.as_mut().expect("err").missed_expiry_tasks.get_mut() += 1; + return; + } + let wrkr = wrkr.expect("err!"); + select! { + //_ -> GlobalContext.Done() => {} + _ = wrkr.send(Some(Box::new(task))) => (), + else => { + *self.stats.as_mut().expect("err").missed_expiry_tasks.get_mut() += 1; + } + } + } + + pub fn get_worker_ch(&self, h: u64) -> Option>> { + if self.tasks_tx.len() == 0 { + return None; + } + Some(self.tasks_tx[h as usize % self.tasks_tx.len()].clone()) + } + + pub async fn resize_workers(n: usize, api: Arc) { + if n == GLOBAL_ExpiryState.read().await.tasks_tx.len() || n < 1 { + return; + } + + let mut state = GLOBAL_ExpiryState.write().await; + + while state.tasks_tx.len() < n { + let (tx, mut rx) = mpsc::channel(10000); + let api = api.clone(); + let rx = Arc::new(tokio::sync::Mutex::new(rx)); + state.tasks_tx.push(tx); + state.tasks_rx.push(rx.clone()); + *state.stats.as_mut().expect("err").workers.get_mut() += 1; + tokio::spawn(async move { + let mut rx = rx.lock().await; + //let mut expiry_state = GLOBAL_ExpiryState.read().await; + ExpiryState::worker(&mut *rx, api).await; + }); + } + + let mut l = state.tasks_tx.len(); + while l > n { + let worker = state.tasks_tx[l - 1].clone(); + worker.send(None).await.unwrap_or(()); + state.tasks_tx.remove(l - 1); + state.tasks_rx.remove(l - 1); + *state.stats.as_mut().expect("err").workers.get_mut() -= 1; + l -= 1; + } + } + + pub async fn worker(rx: &mut Receiver>, api: Arc) { + loop { + select! { + _ = tokio::signal::ctrl_c() => { + info!("got ctrl+c, exits"); + break; + } + v = rx.recv() => { + if v.is_none() { + break; + } + let v = v.expect("err!"); + if v.is_none() { + //rx.close(); + //drop(rx); + let _ = rx; + return; + } + let v = v.expect("err!"); + if v.as_any().is::() { + let v = v.as_any().downcast_ref::().expect("err!"); + if v.obj_info.transitioned_object.status != "" { + apply_expiry_on_transitioned_object(api.clone(), &v.obj_info, &v.event, &v.src).await; + } else { + apply_expiry_on_non_transitioned_objects(api.clone(), &v.obj_info, &v.event, &v.src).await; + } + } + else if v.as_any().is::() { + let v = v.as_any().downcast_ref::().expect("err!"); + //delete_object_versions(api, &v.bucket, &v.versions, v.event).await; + } + else if v.as_any().is::() { + //transitionLogIf(es.ctx, deleteObjectFromRemoteTier(es.ctx, v.ObjName, v.VersionID, v.TierName)) + } + else if v.as_any().is::() { + let v = v.as_any().downcast_ref::().expect("err!"); + let oi = v.0.clone(); + + } + else { + //info!("Invalid work type - {:?}", v); + todo!(); + } + } + } + } + } +} + +struct TransitionTask { + obj_info: ObjectInfo, + src: LcEventSrc, + event: lifecycle::Event, +} + +impl ExpiryOp for TransitionTask { + fn op_hash(&self) -> u64 { + let mut hasher = Sha256::new(); + let _ = hasher.write(format!("{}", self.obj_info.bucket).as_bytes()); + //let _ = hasher.write(format!("{}", self.obj_info.versions[0].object_name).as_bytes()); + hasher.flush(); + xxh64::xxh64(hasher.clone().finalize().as_slice(), XXHASH_SEED) + } + + fn as_any(&self) -> &dyn Any { + self + } +} + +pub struct TransitionState { + transition_tx: A_Sender>, + transition_rx: A_Receiver>, + pub num_workers: AtomicI64, + kill_tx: A_Sender<()>, + kill_rx: A_Receiver<()>, + active_tasks: AtomicI64, + missed_immediate_tasks: AtomicI64, + last_day_stats: Arc>>, +} + +type BoxFuture<'a, T> = Pin + Send + 'a>>; +//type RetKill = impl Future> + Send + 'static; +//type RetTransitionTask = impl Future>> + Send + 'static; + +impl TransitionState { + #[allow(clippy::new_ret_no_self)] + pub fn new() -> Arc { + let (tx1, rx1) = bounded(100000); + let (tx2, rx2) = bounded(1); + Arc::new(Self { + transition_tx: tx1, + transition_rx: rx1, + num_workers: AtomicI64::new(0), + kill_tx: tx2, + kill_rx: rx2, + active_tasks: AtomicI64::new(0), + missed_immediate_tasks: AtomicI64::new(0), + last_day_stats: Arc::new(Mutex::new(HashMap::new())), + }) + } + + pub async fn queue_transition_task(&self, oi: &ObjectInfo, event: &lifecycle::Event, src: &LcEventSrc) { + let task = TransitionTask { + obj_info: oi.clone(), + src: src.clone(), + event: event.clone(), + }; + select! { + //_ -> t.ctx.Done() => (), + _ = self.transition_tx.send(Some(task)) => (), + else => { + match src { + LcEventSrc::S3PutObject | LcEventSrc::S3CopyObject | LcEventSrc::S3CompleteMultipartUpload => { + self.missed_immediate_tasks.fetch_add(1, Ordering::SeqCst); + } + _ => () + } + }, + } + } + + pub async fn init(api: Arc) { + let mut n = 10; //globalAPIConfig.getTransitionWorkers(); + let tw = 10; //globalILMConfig.getTransitionWorkers(); + if tw > 0 { + n = tw; + } + + //let mut transition_state = GLOBAL_TransitionState.write().await; + //self.objAPI = objAPI + Self::update_workers(api, n).await; + } + + pub fn pending_tasks(&self) -> usize { + //let transition_rx = GLOBAL_TransitionState.transition_rx.lock().unwrap(); + let transition_rx = &GLOBAL_TransitionState.transition_rx; + transition_rx.len() + } + + pub fn active_tasks(&self) -> i64 { + self.active_tasks.load(Ordering::SeqCst) + } + + pub fn missed_immediate_tasks(&self) -> i64 { + self.missed_immediate_tasks.load(Ordering::SeqCst) + } + + pub async fn worker(api: Arc) { + loop { + select! { + _ = GLOBAL_TransitionState.kill_rx.recv() => { + return; + } + task = GLOBAL_TransitionState.transition_rx.recv() => { + if task.is_err() { + break; + } + let task = task.expect("err!"); + if task.is_none() { + //self.transition_rx.close(); + //drop(self.transition_rx); + return; + } + let task = task.expect("err!"); + if task.as_any().is::() { + let task = task.as_any().downcast_ref::().expect("err!"); + + GLOBAL_TransitionState.active_tasks.fetch_add(1, Ordering::SeqCst); + if let Err(err) = transition_object(api.clone(), &task.obj_info, LcAuditEvent::new(task.event.clone(), task.src.clone())).await { + if !is_err_version_not_found(&err) && !is_err_object_not_found(&err) && !is_network_or_host_down(&err.to_string(), false) { + if !err.to_string().contains("use of closed network connection") { + error!("Transition to {} failed for {}/{} version:{} with {}", + task.event.storage_class, task.obj_info.bucket, task.obj_info.name, task.obj_info.version_id.expect("err"), err.to_string()); + } + } + } else { + let mut ts = TierStats { + total_size: task.obj_info.size as u64, + num_versions: 1, + ..Default::default() + }; + if task.obj_info.is_latest { + ts.num_objects = 1; + } + GLOBAL_TransitionState.add_lastday_stats(&task.event.storage_class, ts); + } + GLOBAL_TransitionState.active_tasks.fetch_add(-1, Ordering::SeqCst); + } + } + else => () + } + } + } + + pub fn add_lastday_stats(&self, tier: &str, ts: TierStats) { + let mut tier_stats = self.last_day_stats.lock().unwrap(); + tier_stats + .entry(tier.to_string()) + .and_modify(|e| e.add_stats(ts)) + .or_insert(LastDayTierStats::default()); + } + + pub fn get_daily_all_tier_stats(&self) -> DailyAllTierStats { + let tier_stats = self.last_day_stats.lock().unwrap(); + let mut res = DailyAllTierStats::with_capacity(tier_stats.len()); + for (tier, st) in tier_stats.iter() { + res.insert(tier.clone(), st.clone()); + } + res + } + + pub async fn update_workers(api: Arc, n: i64) { + Self::update_workers_inner(api, n).await; + } + + pub async fn update_workers_inner(api: Arc, n: i64) { + let mut n = n; + if n == 0 { + n = 100; + } + + let mut num_workers = GLOBAL_TransitionState.num_workers.load(Ordering::SeqCst); + while num_workers < n { + let clone_api = api.clone(); + tokio::spawn(async move { + TransitionState::worker(clone_api).await; + }); + num_workers = num_workers + 1; + GLOBAL_TransitionState.num_workers.fetch_add(1, Ordering::SeqCst); + } + + let mut num_workers = GLOBAL_TransitionState.num_workers.load(Ordering::SeqCst); + while num_workers > n { + let worker = GLOBAL_TransitionState.kill_tx.clone(); + worker.send(()).await; + num_workers = num_workers - 1; + GLOBAL_TransitionState.num_workers.fetch_add(-1, Ordering::SeqCst); + } + } +} + +struct AuditTierOp { + tier: String, + time_to_responsens: i64, + output_bytes: i64, + error: String, +} + +impl AuditTierOp { + #[allow(clippy::new_ret_no_self)] + pub async fn new() -> Result { + Ok(Self { + tier: String::from("tier"), + time_to_responsens: 0, + output_bytes: 0, + error: String::from(""), + }) + } + + pub fn string(&self) -> String { + format!( + "tier:{},respNS:{},tx:{},err:{}", + self.tier, self.time_to_responsens, self.output_bytes, self.error + ) + } +} + +pub async fn init_background_expiry(api: Arc) { + let mut workers = num_cpus::get() / 2; + //globalILMConfig.getExpirationWorkers() + if let Ok(env_expiration_workers) = env::var("_RUSTFS_EXPIRATION_WORKERS") { + if let Ok(num_expirations) = env_expiration_workers.parse::() { + workers = num_expirations; + } + } + + if workers == 0 { + workers = 100; + } + + //let expiry_state = GLOBAL_ExpiryStSate.write().await; + ExpiryState::resize_workers(workers, api).await; +} + +pub async fn validate_transition_tier(lc: &BucketLifecycleConfiguration) -> Result<(), std::io::Error> { + for rule in &lc.rules { + if let Some(transitions) = &rule.transitions { + for transition in transitions { + if let Some(storage_class) = &transition.storage_class { + if storage_class.as_str() != "" { + let valid = GLOBAL_TierConfigMgr.read().await.is_tier_valid(storage_class.as_str()); + if !valid { + return Err(std::io::Error::other(ERR_INVALID_STORAGECLASS)); + } + } + } + } + } + if let Some(noncurrent_version_transitions) = &rule.noncurrent_version_transitions { + for noncurrent_version_transition in noncurrent_version_transitions { + if let Some(storage_class) = &noncurrent_version_transition.storage_class { + if storage_class.as_str() != "" { + let valid = GLOBAL_TierConfigMgr.read().await.is_tier_valid(storage_class.as_str()); + if !valid { + return Err(std::io::Error::other(ERR_INVALID_STORAGECLASS)); + } + } + } + } + } + } + Ok(()) +} + +pub async fn enqueue_transition_immediate(oi: &ObjectInfo, src: LcEventSrc) { + let lc = GLOBAL_LifecycleSys.get(&oi.bucket).await; + if !lc.is_none() { + let event = lc.expect("err").eval(&oi.to_lifecycle_opts()).await; + match event.action { + lifecycle::IlmAction::TransitionAction | lifecycle::IlmAction::TransitionVersionAction => { + if oi.delete_marker || oi.is_dir { + return; + } + GLOBAL_TransitionState.queue_transition_task(oi, &event, &src).await; + } + _ => (), + } + } +} + +pub async fn expire_transitioned_object( + api: Arc, + oi: &ObjectInfo, + lc_event: &lifecycle::Event, + src: &LcEventSrc, +) -> Result { + //let traceFn = GLOBAL_LifecycleSys.trace(oi); + let mut opts = ObjectOptions { + versioned: BucketVersioningSys::prefix_enabled(&oi.bucket, &oi.name).await, + expiration: ExpirationOptions { expire: true }, + ..Default::default() + }; + if lc_event.action == IlmAction::DeleteVersionAction { + opts.version_id = oi.version_id.map(|id| id.to_string()); + } + //let tags = LcAuditEvent::new(src, lcEvent).Tags(); + if lc_event.action == IlmAction::DeleteRestoredAction { + opts.transition.expire_restored = true; + match api.delete_object(&oi.bucket, &oi.name, opts).await { + Ok(dobj) => { + //audit_log_lifecycle(*oi, ILMExpiry, tags, traceFn); + return Ok(dobj); + } + Err(err) => return Err(std::io::Error::other(err)), + } + } + + let ret = delete_object_from_remote_tier( + &oi.transitioned_object.name, + &oi.transitioned_object.version_id, + &oi.transitioned_object.tier, + ) + .await; + if ret.is_ok() { + opts.skip_decommissioned = true; + } else { + //transitionLogIf(ctx, err); + } + + let dobj = api.delete_object(&oi.bucket, &oi.name, opts).await?; + + //defer auditLogLifecycle(ctx, *oi, ILMExpiry, tags, traceFn) + + let mut event_name = EventName::ObjectRemovedDelete; + if oi.delete_marker { + event_name = EventName::ObjectRemovedDeleteMarkerCreated; + } + let obj_info = ObjectInfo { + name: oi.name.clone(), + version_id: oi.version_id, + delete_marker: oi.delete_marker, + ..Default::default() + }; + send_event(EventArgs { + event_name: event_name.as_ref().to_string(), + bucket_name: obj_info.bucket.clone(), + object: obj_info, + user_agent: "Internal: [ILM-Expiry]".to_string(), + host: GLOBAL_LocalNodeName.to_string(), + ..Default::default() + }); + + Ok(dobj) +} + +pub fn gen_transition_objname(bucket: &str) -> Result { + let us = Uuid::new_v4().to_string(); + let mut hasher = Sha256::new(); + let _ = hasher.write(format!("{}/{}", get_global_deployment_id().unwrap_or_default(), bucket).as_bytes()); + hasher.flush(); + let hash = rustfs_utils::crypto::hex(hasher.clone().finalize().as_slice()); + let obj = format!("{}/{}/{}/{}", &hash[0..16], &us[0..2], &us[2..4], &us); + Ok(obj) +} + +pub async fn transition_object(api: Arc, oi: &ObjectInfo, lae: LcAuditEvent) -> Result<(), Error> { + let time_ilm = ScannerMetrics::time_ilm(lae.event.action); + + let opts = ObjectOptions { + transition: TransitionOptions { + status: lifecycle::TRANSITION_PENDING.to_string(), + tier: lae.event.storage_class, + etag: oi.etag.clone().expect("err").to_string(), + ..Default::default() + }, + //lifecycle_audit_event: lae, + version_id: Some(oi.version_id.expect("err").to_string()), + versioned: BucketVersioningSys::prefix_enabled(&oi.bucket, &oi.name).await, + version_suspended: BucketVersioningSys::prefix_suspended(&oi.bucket, &oi.name).await, + mod_time: oi.mod_time, + ..Default::default() + }; + time_ilm(1); + api.transition_object(&oi.bucket, &oi.name, &opts).await +} + +pub fn audit_tier_actions(api: ECStore, tier: &str, bytes: i64) -> TimeFn { + todo!(); +} + +pub async fn get_transitioned_object_reader( + bucket: &str, + object: &str, + rs: HTTPRangeSpec, + h: HeaderMap, + oi: ObjectInfo, + opts: &ObjectOptions, +) -> Result { + let mut tier_config_mgr = GLOBAL_TierConfigMgr.write().await; + let tgt_client = match tier_config_mgr.get_driver(&oi.transitioned_object.tier).await { + Ok(d) => d, + Err(err) => return Err(std::io::Error::other(err)), + }; + + let ret = new_getobjectreader(rs, &oi, opts, &h); + if let Err(err) = ret { + return Err(error_resp_to_object_err(err, vec![bucket, object])); + } + let (get_fn, off, length) = ret.expect("err"); + let mut gopts = WarmBackendGetOpts::default(); + + if off >= 0 && length >= 0 { + gopts.start_offset = off; + gopts.length = length; + } + + //return Ok(HttpFileReader::new(rs, &oi, opts, &h)); + //timeTierAction := auditTierActions(oi.transitioned_object.Tier, length) + let reader = tgt_client + .get(&oi.transitioned_object.name, &oi.transitioned_object.version_id, gopts) + .await?; + Ok(get_fn(reader, h)) +} + +pub fn post_restore_opts(r: http::Request, bucket: &str, object: &str) -> Result { + todo!(); +} + +pub fn put_restore_opts(bucket: &str, object: &str, rreq: &RestoreObjectRequest, oi: &ObjectInfo) -> ObjectOptions { + todo!(); +} + +pub trait LifecycleOps { + fn to_lifecycle_opts(&self) -> lifecycle::ObjectOpts; +} + +impl LifecycleOps for ObjectInfo { + fn to_lifecycle_opts(&self) -> lifecycle::ObjectOpts { + lifecycle::ObjectOpts { + name: self.name.clone(), + user_tags: self.user_tags.clone(), + version_id: self.version_id.expect("err").to_string(), + mod_time: self.mod_time, + size: self.size, + is_latest: self.is_latest, + num_versions: self.num_versions, + delete_marker: self.delete_marker, + successor_mod_time: self.successor_mod_time, + //restore_ongoing: self.restore_ongoing, + //restore_expires: self.restore_expires, + transition_status: self.transitioned_object.status.clone(), + ..Default::default() + } + } +} + +#[derive(Debug, Default, Clone)] +pub struct S3Location { + pub bucketname: String, + //pub encryption: Encryption, + pub prefix: String, + pub storage_class: String, + //pub tagging: Tags, + pub user_metadata: HashMap, +} + +#[derive(Debug, Default, Clone)] +pub struct OutputLocation(pub S3Location); + +#[derive(Debug, Default, Clone)] +pub struct RestoreObjectRequest { + pub days: i64, + pub ror_type: String, + pub tier: String, + pub description: String, + //pub select_parameters: SelectParameters, + pub output_location: OutputLocation, +} + +const MAX_RESTORE_OBJECT_REQUEST_SIZE: i64 = 2 << 20; diff --git a/ecstore/src/bucket/lifecycle/lifecycle.rs b/ecstore/src/bucket/lifecycle/lifecycle.rs new file mode 100644 index 000000000..94d3ce5e0 --- /dev/null +++ b/ecstore/src/bucket/lifecycle/lifecycle.rs @@ -0,0 +1,730 @@ +use s3s::dto::{ + BucketLifecycleConfiguration, ExpirationStatus, LifecycleExpiration, LifecycleRule, NoncurrentVersionTransition, + ObjectLockConfiguration, ObjectLockEnabled, Transition, +}; +use std::cmp::Ordering; +use std::env; +use std::fmt::Display; +use time::macros::{datetime, offset}; +use time::{self, Duration, OffsetDateTime}; + +use crate::bucket::lifecycle::rule::TransitionOps; + +use super::bucket_lifecycle_ops::RestoreObjectRequest; + +pub const TRANSITION_COMPLETE: &str = "complete"; +pub const TRANSITION_PENDING: &str = "pending"; + +const ERR_LIFECYCLE_TOO_MANY_RULES: &str = "Lifecycle configuration allows a maximum of 1000 rules"; +const ERR_LIFECYCLE_NO_RULE: &str = "Lifecycle configuration should have at least one rule"; +const ERR_LIFECYCLE_DUPLICATE_ID: &str = "Rule ID must be unique. Found same ID for more than one rule"; +const ERR_XML_NOT_WELL_FORMED: &str = "The XML you provided was not well-formed or did not validate against our published schema"; +const ERR_LIFECYCLE_BUCKET_LOCKED: &str = + "ExpiredObjectAllVersions element and DelMarkerExpiration action cannot be used on an object locked bucket"; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum IlmAction { + NoneAction = 0, + DeleteAction, + DeleteVersionAction, + TransitionAction, + TransitionVersionAction, + DeleteRestoredAction, + DeleteRestoredVersionAction, + DeleteAllVersionsAction, + DelMarkerDeleteAllVersionsAction, + ActionCount, +} + +impl IlmAction { + pub fn delete_restored(&self) -> bool { + *self == Self::DeleteRestoredAction || *self == Self::DeleteRestoredVersionAction + } + + pub fn delete_versioned(&self) -> bool { + *self == Self::DeleteVersionAction || *self == Self::DeleteRestoredVersionAction + } + + pub fn delete_all(&self) -> bool { + *self == Self::DeleteAllVersionsAction || *self == Self::DelMarkerDeleteAllVersionsAction + } + + pub fn delete(&self) -> bool { + if self.delete_restored() { + return true; + } + *self == Self::DeleteVersionAction + || *self == Self::DeleteAction + || *self == Self::DeleteAllVersionsAction + || *self == Self::DelMarkerDeleteAllVersionsAction + } +} + +impl Display for IlmAction { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{:?}", self) + } +} + +#[async_trait::async_trait] +pub trait RuleValidate { + fn validate(&self) -> Result<(), std::io::Error>; +} + +#[async_trait::async_trait] +impl RuleValidate for LifecycleRule { + /*fn validate_id(&self) -> Result<()> { + if self.id.len() > 255 { + return errInvalidRuleID; + } + Ok(()) + } + + fn validate_status(&self) -> Result<()> { + if self.Status.len() == 0 { + return errEmptyRuleStatus; + } + + if self.Status != Enabled && self.Status != Disabled { + return errInvalidRuleStatus; + } + Ok(()) + } + + fn validate_expiration(&self) -> Result<()> { + self.Expiration.Validate(); + } + + fn validate_noncurrent_expiration(&self) -> Result<()> { + self.NoncurrentVersionExpiration.Validate() + } + + fn validate_prefix_and_filter(&self) -> Result<()> { + if !self.Prefix.set && self.Filter.IsEmpty() || self.Prefix.set && !self.Filter.IsEmpty() { + return errXMLNotWellFormed; + } + if !self.Prefix.set { + return self.Filter.Validate(); + } + Ok(()) + } + + fn validate_transition(&self) -> Result<()> { + self.Transition.Validate() + } + + fn validate_noncurrent_transition(&self) -> Result<()> { + self.NoncurrentVersionTransition.Validate() + } + + fn get_prefix(&self) -> String { + if p := self.Prefix.String(); p != "" { + return p + } + if p := self.Filter.Prefix.String(); p != "" { + return p + } + if p := self.Filter.And.Prefix.String(); p != "" { + return p + } + "".to_string() + }*/ + + fn validate(&self) -> Result<(), std::io::Error> { + /*self.validate_id()?; + self.validate_status()?; + self.validate_expiration()?; + self.validate_noncurrent_expiration()?; + self.validate_prefix_and_filter()?; + self.validate_transition()?; + self.validate_noncurrent_transition()?; + if (!self.Filter.Tag.IsEmpty() || len(self.Filter.And.Tags) != 0) && !self.delmarker_expiration.Empty() { + return errInvalidRuleDelMarkerExpiration + } + if !self.expiration.set && !self.transition.set && !self.noncurrent_version_expiration.set && !self.noncurrent_version_transitions.unwrap()[0].set && self.delmarker_expiration.Empty() { + return errXMLNotWellFormed + }*/ + Ok(()) + } +} + +#[async_trait::async_trait] +pub trait Lifecycle { + async fn has_transition(&self) -> bool; + fn has_expiry(&self) -> bool; + async fn has_active_rules(&self, prefix: &str) -> bool; + async fn validate(&self, lr: &ObjectLockConfiguration) -> Result<(), std::io::Error>; + async fn filter_rules(&self, obj: &ObjectOpts) -> Option>; + async fn eval(&self, obj: &ObjectOpts) -> Event; + async fn eval_inner(&self, obj: &ObjectOpts, now: OffsetDateTime) -> Event; + //fn set_prediction_headers(&self, w: http.ResponseWriter, obj: ObjectOpts); + async fn noncurrent_versions_expiration_limit(&self, obj: &ObjectOpts) -> Event; +} + +#[async_trait::async_trait] +impl Lifecycle for BucketLifecycleConfiguration { + async fn has_transition(&self) -> bool { + for rule in self.rules.iter() { + if !rule.transitions.is_none() { + return true; + } + } + false + } + + fn has_expiry(&self) -> bool { + for rule in self.rules.iter() { + if !rule.expiration.is_none() || !rule.noncurrent_version_expiration.is_none() { + return true; + } + } + false + } + + async fn has_active_rules(&self, prefix: &str) -> bool { + if self.rules.len() == 0 { + return false; + } + for rule in self.rules.iter() { + if rule.status.as_str() == ExpirationStatus::DISABLED { + continue; + } + + let rule_prefix = rule.prefix.as_ref().expect("err!"); + if prefix.len() > 0 && rule_prefix.len() > 0 { + if !prefix.starts_with(rule_prefix) && !rule_prefix.starts_with(&prefix) { + continue; + } + } + + let rule_noncurrent_version_expiration = rule.noncurrent_version_expiration.as_ref().expect("err!"); + if rule_noncurrent_version_expiration.noncurrent_days.expect("err!") > 0 { + return true; + } + if rule_noncurrent_version_expiration.newer_noncurrent_versions.expect("err!") > 0 { + return true; + } + if !rule.noncurrent_version_transitions.is_none() { + return true; + } + let rule_expiration = rule.expiration.as_ref().expect("err!"); + if !rule_expiration.date.is_none() + && OffsetDateTime::from(rule_expiration.date.clone().expect("err!")).unix_timestamp() + < OffsetDateTime::now_utc().unix_timestamp() + { + return true; + } + if !rule_expiration.date.is_none() { + return true; + } + if rule_expiration.expired_object_delete_marker.expect("err!") { + return true; + } + let rule_transitions: &[Transition] = &rule.transitions.as_ref().expect("err!"); + let rule_transitions_0 = rule_transitions[0].clone(); + if !rule_transitions_0.date.is_none() + && OffsetDateTime::from(rule_transitions_0.date.expect("err!")).unix_timestamp() + < OffsetDateTime::now_utc().unix_timestamp() + { + return true; + } + if !rule.transitions.is_none() { + return true; + } + } + false + } + + async fn validate(&self, lr: &ObjectLockConfiguration) -> Result<(), std::io::Error> { + if self.rules.len() > 1000 { + return Err(std::io::Error::other(ERR_LIFECYCLE_TOO_MANY_RULES)); + } + if self.rules.len() == 0 { + return Err(std::io::Error::other(ERR_LIFECYCLE_NO_RULE)); + } + + for r in &self.rules { + r.validate()?; + if let Some(object_lock_enabled) = lr.object_lock_enabled.as_ref() { + if let Some(expiration) = r.expiration.as_ref() { + if let Some(expired_object_delete_marker) = expiration.expired_object_delete_marker { + if object_lock_enabled.as_str() == ObjectLockEnabled::ENABLED && (!expired_object_delete_marker) { + return Err(std::io::Error::other(ERR_LIFECYCLE_BUCKET_LOCKED)); + } + } /*else { + if object_lock_enabled.as_str() == ObjectLockEnabled::ENABLED { + return Err(Error::msg(ERR_LIFECYCLE_BUCKET_LOCKED)); + } + }*/ + } + } + } + for (i, _) in self.rules.iter().enumerate() { + if i == self.rules.len() - 1 { + break; + } + let other_rules = &self.rules[i + 1..]; + for other_rule in other_rules { + if self.rules[i].id == other_rule.id { + return Err(std::io::Error::other(ERR_LIFECYCLE_DUPLICATE_ID)); + } + } + } + Ok(()) + } + + async fn filter_rules(&self, obj: &ObjectOpts) -> Option> { + if obj.name == "" { + return None; + } + let mut rules = Vec::::new(); + for rule in self.rules.iter() { + if rule.status.as_str() == ExpirationStatus::DISABLED { + continue; + } + if let Some(prefix) = rule.prefix.clone() { + if !obj.name.starts_with(prefix.as_str()) { + continue; + } + } + /*if !rule.filter.test_tags(obj.user_tags) { + continue; + }*/ + //if !obj.delete_marker && !rule.filter.BySize(obj.size) { + if !obj.delete_marker && false { + continue; + } + rules.push(rule.clone()); + } + Some(rules) + } + + async fn eval(&self, obj: &ObjectOpts) -> Event { + self.eval_inner(obj, OffsetDateTime::now_utc()).await + } + + async fn eval_inner(&self, obj: &ObjectOpts, now: OffsetDateTime) -> Event { + let mut events = Vec::::new(); + if obj.mod_time.expect("err").unix_timestamp() == 0 { + return Event::default(); + } + + if let Some(restore_expires) = obj.restore_expires { + if !restore_expires.unix_timestamp() == 0 && now.unix_timestamp() > restore_expires.unix_timestamp() { + let mut action = IlmAction::DeleteRestoredAction; + if !obj.is_latest { + action = IlmAction::DeleteRestoredVersionAction; + } + + events.push(Event { + action: action, + due: Some(now), + rule_id: "".into(), + noncurrent_days: 0, + newer_noncurrent_versions: 0, + storage_class: "".into(), + }); + } + } + + if let Some(ref lc_rules) = self.filter_rules(obj).await { + for rule in lc_rules.iter() { + if obj.expired_object_deletemarker() { + if let Some(expiration) = rule.expiration.as_ref() { + if let Some(expired_object_delete_marker) = expiration.expired_object_delete_marker { + events.push(Event { + action: IlmAction::DeleteVersionAction, + rule_id: rule.id.clone().expect("err!"), + due: Some(now), + noncurrent_days: 0, + newer_noncurrent_versions: 0, + storage_class: "".into(), + }); + break; + } + } + + if let Some(expiration) = rule.expiration.as_ref() { + if let Some(days) = expiration.days { + let expected_expiry = expected_expiry_time(obj.mod_time.expect("err!"), days /*, date*/); + if now.unix_timestamp() == 0 || now.unix_timestamp() > expected_expiry.unix_timestamp() { + events.push(Event { + action: IlmAction::DeleteVersionAction, + rule_id: rule.id.clone().expect("err!"), + due: Some(expected_expiry), + noncurrent_days: 0, + newer_noncurrent_versions: 0, + storage_class: "".into(), + }); + break; + } + } + } + } + + if obj.is_latest { + if let Some(ref expiration) = rule.expiration { + if let Some(expired_object_delete_marker) = expiration.expired_object_delete_marker { + if obj.delete_marker && expired_object_delete_marker { + let due = expiration.next_due(obj); + if let Some(due) = due { + if now.unix_timestamp() == 0 || now.unix_timestamp() > due.unix_timestamp() { + events.push(Event { + action: IlmAction::DelMarkerDeleteAllVersionsAction, + rule_id: rule.id.clone().expect("err!"), + due: Some(due), + noncurrent_days: 0, + newer_noncurrent_versions: 0, + storage_class: "".into(), + }); + } + } + continue; + } + } + } + } + + if !obj.is_latest { + if let Some(ref noncurrent_version_expiration) = rule.noncurrent_version_expiration { + if let Some(newer_noncurrent_versions) = noncurrent_version_expiration.newer_noncurrent_versions { + if newer_noncurrent_versions > 0 { + continue; + } + } + } + } + + if !obj.is_latest { + if let Some(ref noncurrent_version_expiration) = rule.noncurrent_version_expiration { + if let Some(noncurrent_days) = noncurrent_version_expiration.noncurrent_days { + if noncurrent_days != 0 { + if let Some(successor_mod_time) = obj.successor_mod_time { + let expected_expiry = expected_expiry_time(successor_mod_time, noncurrent_days); + if now.unix_timestamp() == 0 || now.unix_timestamp() > expected_expiry.unix_timestamp() { + events.push(Event { + action: IlmAction::DeleteVersionAction, + rule_id: rule.id.clone().expect("err!"), + due: Some(expected_expiry), + noncurrent_days: 0, + newer_noncurrent_versions: 0, + storage_class: "".into(), + }); + } + } + } + } + } + } + + if !obj.is_latest { + if let Some(ref noncurrent_version_transitions) = rule.noncurrent_version_transitions { + if let Some(ref storage_class) = noncurrent_version_transitions[0].storage_class { + if storage_class.as_str() != "" { + if !obj.delete_marker && obj.transition_status != TRANSITION_COMPLETE { + let due = rule.noncurrent_version_transitions.as_ref().unwrap()[0].next_due(obj); + if due.is_some() + && (now.unix_timestamp() == 0 || now.unix_timestamp() > due.unwrap().unix_timestamp()) + { + events.push(Event { + action: IlmAction::TransitionVersionAction, + rule_id: rule.id.clone().expect("err!"), + due, + storage_class: rule.noncurrent_version_transitions.as_ref().unwrap()[0] + .storage_class + .clone() + .unwrap() + .as_str() + .to_string(), + ..Default::default() + }); + } + } + } + } + } + } + + if obj.is_latest && !obj.delete_marker { + if let Some(ref expiration) = rule.expiration { + if let Some(ref date) = expiration.date { + let date0 = OffsetDateTime::from(date.clone()); + if date0.unix_timestamp() != 0 { + if now.unix_timestamp() == 0 || now.unix_timestamp() > date0.unix_timestamp() { + events.push(Event { + action: IlmAction::DeleteAction, + rule_id: rule.id.clone().expect("err!"), + due: Some(date0), + noncurrent_days: 0, + newer_noncurrent_versions: 0, + storage_class: "".into(), + }); + } + } + } else if let Some(days) = expiration.days { + if days != 0 { + let expected_expiry: OffsetDateTime = expected_expiry_time(obj.mod_time.expect("err!"), days); + if now.unix_timestamp() == 0 || now.unix_timestamp() > expected_expiry.unix_timestamp() { + let mut event = Event { + action: IlmAction::DeleteAction, + rule_id: rule.id.clone().expect("err!"), + due: Some(expected_expiry), + noncurrent_days: 0, + newer_noncurrent_versions: 0, + storage_class: "".into(), + }; + /*if rule.expiration.expect("err!").delete_all.val { + event.action = IlmAction::DeleteAllVersionsAction + }*/ + events.push(event); + } + } + } + } + + if obj.transition_status != TRANSITION_COMPLETE { + if let Some(ref transitions) = rule.transitions { + let due = transitions[0].next_due(obj); + if let Some(due) = due { + if due.unix_timestamp() > 0 + && (now.unix_timestamp() == 0 || now.unix_timestamp() > due.unix_timestamp()) + { + events.push(Event { + action: IlmAction::TransitionAction, + rule_id: rule.id.clone().expect("err!"), + due: Some(due), + storage_class: transitions[0].storage_class.clone().expect("err!").as_str().to_string(), + noncurrent_days: 0, + newer_noncurrent_versions: 0, + }); + } + } + } + } + } + } + } + + if events.len() > 0 { + events.sort_by(|a, b| { + if now.unix_timestamp() > a.due.expect("err!").unix_timestamp() + && now.unix_timestamp() > b.due.expect("err").unix_timestamp() + || a.due.expect("err").unix_timestamp() == b.due.expect("err").unix_timestamp() + { + match a.action { + IlmAction::DeleteAllVersionsAction + | IlmAction::DelMarkerDeleteAllVersionsAction + | IlmAction::DeleteAction + | IlmAction::DeleteVersionAction => { + return Ordering::Less; + } + _ => (), + } + match b.action { + IlmAction::DeleteAllVersionsAction + | IlmAction::DelMarkerDeleteAllVersionsAction + | IlmAction::DeleteAction + | IlmAction::DeleteVersionAction => { + return Ordering::Greater; + } + _ => (), + } + return Ordering::Less; + } + + if a.due.expect("err").unix_timestamp() < b.due.expect("err").unix_timestamp() { + return Ordering::Less; + } + return Ordering::Greater; + }); + return events[0].clone(); + } + + Event::default() + } + + async fn noncurrent_versions_expiration_limit(&self, obj: &ObjectOpts) -> Event { + if let Some(filter_rules) = self.filter_rules(obj).await { + for rule in filter_rules.iter() { + if let Some(ref noncurrent_version_expiration) = rule.noncurrent_version_expiration { + if let Some(newer_noncurrent_versions) = noncurrent_version_expiration.newer_noncurrent_versions { + if newer_noncurrent_versions == 0 { + continue; + } + return Event { + action: IlmAction::DeleteVersionAction, + rule_id: rule.id.clone().expect("err"), + noncurrent_days: noncurrent_version_expiration.noncurrent_days.expect("noncurrent_days err.") as u32, + newer_noncurrent_versions: newer_noncurrent_versions as usize, + due: Some(OffsetDateTime::UNIX_EPOCH), + storage_class: "".into(), + }; + } else { + return Event { + action: IlmAction::DeleteVersionAction, + rule_id: rule.id.clone().expect("err"), + noncurrent_days: noncurrent_version_expiration.noncurrent_days.expect("noncurrent_days err.") as u32, + newer_noncurrent_versions: 0, + due: Some(OffsetDateTime::UNIX_EPOCH), + storage_class: "".into(), + }; + } + } + } + } + Event::default() + } +} + +#[async_trait::async_trait] +pub trait LifecycleCalculate { + fn next_due(&self, obj: &ObjectOpts) -> Option; +} + +#[async_trait::async_trait] +impl LifecycleCalculate for LifecycleExpiration { + fn next_due(&self, obj: &ObjectOpts) -> Option { + if !obj.is_latest || !obj.delete_marker { + return None; + } + + Some(expected_expiry_time(obj.mod_time.unwrap(), self.days.unwrap())) + } +} + +#[async_trait::async_trait] +impl LifecycleCalculate for NoncurrentVersionTransition { + fn next_due(&self, obj: &ObjectOpts) -> Option { + if obj.is_latest || self.storage_class.is_none() { + return None; + } + if self.noncurrent_days.is_none() { + return obj.successor_mod_time; + } + Some(expected_expiry_time(obj.successor_mod_time.unwrap(), self.noncurrent_days.unwrap())) + } +} + +#[async_trait::async_trait] +impl LifecycleCalculate for Transition { + fn next_due(&self, obj: &ObjectOpts) -> Option { + if !obj.is_latest || self.days.is_none() { + return None; + } + + if let Some(date) = self.date.clone() { + return Some(date.into()); + } + + if self.days.is_none() { + return obj.mod_time; + } + Some(expected_expiry_time(obj.mod_time.unwrap(), self.days.unwrap())) + } +} + +pub fn expected_expiry_time(mod_time: OffsetDateTime, days: i32) -> OffsetDateTime { + if days == 0 { + return mod_time; + } + let t = mod_time + .to_offset(offset!(-0:00:00)) + .saturating_add(Duration::days(0 /*days as i64*/)); //debug + let mut hour = 3600; + if let Ok(env_ilm_hour) = env::var("_RUSTFS_ILM_HOUR") { + if let Ok(num_hour) = env_ilm_hour.parse::() { + hour = num_hour; + } + } + //t.Truncate(24 * hour) + t +} + +#[derive(Default)] +pub struct ObjectOpts { + pub name: String, + pub user_tags: String, + pub mod_time: Option, + pub size: usize, + pub version_id: String, + pub is_latest: bool, + pub delete_marker: bool, + pub num_versions: usize, + pub successor_mod_time: Option, + pub transition_status: String, + pub restore_ongoing: bool, + pub restore_expires: Option, + pub versioned: bool, + pub version_suspended: bool, +} + +impl ObjectOpts { + pub fn expired_object_deletemarker(&self) -> bool { + self.delete_marker && self.num_versions == 1 + } +} + +#[derive(Debug, Clone)] +pub struct Event { + pub action: IlmAction, + pub rule_id: String, + pub due: Option, + pub noncurrent_days: u32, + pub newer_noncurrent_versions: usize, + pub storage_class: String, +} + +impl Default for Event { + fn default() -> Self { + Self { + action: IlmAction::NoneAction, + rule_id: "".into(), + due: Some(OffsetDateTime::UNIX_EPOCH), + noncurrent_days: 0, + newer_noncurrent_versions: 0, + storage_class: "".into(), + } + } +} + +#[derive(Debug, Clone, Default)] +pub struct ExpirationOptions { + pub expire: bool, +} + +impl ExpirationOptions { + fn marshal_msg(&self, b: &[u8]) -> Result, std::io::Error> { + todo!(); + } + + fn unmarshal_msg(&self, bts: &[u8]) -> Result, std::io::Error> { + todo!(); + } + + fn msg_size(&self) -> i64 { + 1 + 7 + 10 + } +} + +#[derive(Debug, Clone)] +pub struct TransitionOptions { + pub status: String, + pub tier: String, + pub etag: String, + pub restore_request: RestoreObjectRequest, + pub restore_expiry: OffsetDateTime, + pub expire_restored: bool, +} + +impl Default for TransitionOptions { + fn default() -> Self { + Self { + status: Default::default(), + tier: Default::default(), + etag: Default::default(), + restore_request: Default::default(), + restore_expiry: OffsetDateTime::now_utc(), + expire_restored: Default::default(), + } + } +} diff --git a/ecstore/src/bucket/lifecycle/mod.rs b/ecstore/src/bucket/lifecycle/mod.rs new file mode 100644 index 000000000..689f49ed8 --- /dev/null +++ b/ecstore/src/bucket/lifecycle/mod.rs @@ -0,0 +1,6 @@ +pub mod bucket_lifecycle_audit; +pub mod bucket_lifecycle_ops; +pub mod lifecycle; +pub mod rule; +pub mod tier_last_day_stats; +pub mod tier_sweeper; diff --git a/ecstore/src/bucket/lifecycle/rule.rs b/ecstore/src/bucket/lifecycle/rule.rs new file mode 100644 index 000000000..f0c660c88 --- /dev/null +++ b/ecstore/src/bucket/lifecycle/rule.rs @@ -0,0 +1,49 @@ +use s3s::dto::{LifecycleRuleFilter, Transition}; + +const ERR_TRANSITION_INVALID_DAYS: &str = "Days must be 0 or greater when used with Transition"; +const ERR_TRANSITION_INVALID_DATE: &str = "Date must be provided in ISO 8601 format"; +const ERR_TRANSITION_INVALID: &str = + "Exactly one of Days (0 or greater) or Date (positive ISO 8601 format) should be present in Transition."; +const ERR_TRANSITION_DATE_NOT_MIDNIGHT: &str = "'Date' must be at midnight GMT"; + +pub trait Filter { + fn test_tags(&self, user_tags: &str) -> bool; + fn by_size(&self, sz: i64) -> bool; +} + +impl Filter for LifecycleRuleFilter { + fn test_tags(&self, user_tags: &str) -> bool { + true + } + + fn by_size(&self, sz: i64) -> bool { + true + } +} + +pub trait TransitionOps { + fn validate(&self) -> Result<(), std::io::Error>; +} + +impl TransitionOps for Transition { + fn validate(&self) -> Result<(), std::io::Error> { + if !self.date.is_none() && self.days.expect("err!") > 0 { + return Err(std::io::Error::other(ERR_TRANSITION_INVALID)); + } + + if self.storage_class.is_none() { + return Err(std::io::Error::other("ERR_XML_NOT_WELL_FORMED")); + } + Ok(()) + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[tokio::test] + async fn test_rule() { + //assert!(skip_access_checks(p.to_str().unwrap())); + } +} diff --git a/ecstore/src/bucket/lifecycle/tier_last_day_stats.rs b/ecstore/src/bucket/lifecycle/tier_last_day_stats.rs new file mode 100644 index 000000000..85755be7f --- /dev/null +++ b/ecstore/src/bucket/lifecycle/tier_last_day_stats.rs @@ -0,0 +1,83 @@ +use sha2::Sha256; + +use std::collections::HashMap; +use std::ops::Sub; +use time::OffsetDateTime; +use tracing::{error, warn}; + +use crate::heal::data_usage_cache::TierStats; + +pub type DailyAllTierStats = HashMap; + +#[derive(Clone)] +pub struct LastDayTierStats { + bins: [TierStats; 24], + updated_at: OffsetDateTime, +} + +impl Default for LastDayTierStats { + fn default() -> Self { + Self { + bins: Default::default(), + updated_at: OffsetDateTime::now_utc(), + } + } +} + +impl LastDayTierStats { + pub fn add_stats(&mut self, ts: TierStats) { + let mut now = OffsetDateTime::now_utc(); + self.forward_to(&mut now); + + let now_idx = now.hour() as usize; + self.bins[now_idx] = self.bins[now_idx].add(&ts); + } + + fn forward_to(&mut self, t: &mut OffsetDateTime) { + if t.unix_timestamp() == 0 { + *t = OffsetDateTime::now_utc(); + } + + let since = t.sub(self.updated_at).whole_hours(); + if since < 1 { + return; + } + + let (idx, mut last_idx) = (t.hour(), self.updated_at.hour()); + + self.updated_at = *t; + + if since >= 24 { + self.bins = [TierStats::default(); 24]; + return; + } + + while last_idx != idx { + last_idx = (last_idx + 1) % 24; + self.bins[last_idx as usize] = TierStats::default(); + } + } + + fn merge(&self, m: LastDayTierStats) -> LastDayTierStats { + let mut cl = self.clone(); + let mut cm = m.clone(); + let mut merged = LastDayTierStats::default(); + + if cl.updated_at.unix_timestamp() > cm.updated_at.unix_timestamp() { + cm.forward_to(&mut cl.updated_at); + merged.updated_at = cl.updated_at; + } else { + cl.forward_to(&mut cm.updated_at); + merged.updated_at = cm.updated_at; + } + + for (i, _) in cl.bins.iter().enumerate() { + merged.bins[i] = cl.bins[i].add(&cm.bins[i]); + } + + merged + } +} + +#[cfg(test)] +mod test {} diff --git a/ecstore/src/bucket/lifecycle/tier_sweeper.rs b/ecstore/src/bucket/lifecycle/tier_sweeper.rs new file mode 100644 index 000000000..541ec30ec --- /dev/null +++ b/ecstore/src/bucket/lifecycle/tier_sweeper.rs @@ -0,0 +1,129 @@ +use sha2::{Digest, Sha256}; +use std::any::Any; +use std::io::{Cursor, Write}; +use xxhash_rust::xxh64; + +use super::bucket_lifecycle_ops::{ExpiryOp, GLOBAL_ExpiryState, TransitionedObject}; +use super::lifecycle::{self, ObjectOpts}; +use crate::global::GLOBAL_TierConfigMgr; + +static XXHASH_SEED: u64 = 0; + +#[derive(Default)] +struct ObjSweeper { + object: String, + bucket: String, + version_id: String, + versioned: bool, + suspended: bool, + transition_status: String, + transition_tier: String, + transition_version_id: String, + remote_object: String, +} + +impl ObjSweeper { + #[allow(clippy::new_ret_no_self)] + pub async fn new(bucket: &str, object: &str) -> Result { + Ok(Self { + object: object.into(), + bucket: bucket.into(), + ..Default::default() + }) + } + + pub fn with_version(&mut self, vid: String) -> &Self { + self.version_id = vid; + self + } + + pub fn with_versioning(&mut self, versioned: bool, suspended: bool) -> &Self { + self.versioned = versioned; + self.suspended = suspended; + self + } + + pub fn get_opts(&self) -> lifecycle::ObjectOpts { + let mut opts = ObjectOpts { + version_id: self.version_id.clone(), + versioned: self.versioned, + version_suspended: self.suspended, + ..Default::default() + }; + if self.suspended && self.version_id == "" { + opts.version_id = String::from(""); + } + opts + } + + pub fn set_transition_state(&mut self, info: TransitionedObject) { + self.transition_tier = info.tier; + self.transition_status = info.status; + self.remote_object = info.name; + self.transition_version_id = info.version_id; + } + + pub fn should_remove_remote_object(&self) -> Option { + if self.transition_status != lifecycle::TRANSITION_COMPLETE { + return None; + } + + let mut del_tier = false; + if !self.versioned || self.suspended { + // 1, 2.a, 2.b + del_tier = true; + } else if self.versioned && self.version_id != "" { + // 3.a + del_tier = true; + } + if del_tier { + return Some(Jentry { + obj_name: self.remote_object.clone(), + version_id: self.transition_version_id.clone(), + tier_name: self.transition_tier.clone(), + }); + } + None + } + + pub async fn sweep(&self) { + let je = self.should_remove_remote_object(); + if !je.is_none() { + let mut expiry_state = GLOBAL_ExpiryState.write().await; + expiry_state.enqueue_tier_journal_entry(&je.expect("err!")); + } + } +} + +#[derive(Debug, Clone)] +pub struct Jentry { + obj_name: String, + version_id: String, + tier_name: String, +} + +impl ExpiryOp for Jentry { + fn op_hash(&self) -> u64 { + let mut hasher = Sha256::new(); + let _ = hasher.write(format!("{}", self.tier_name).as_bytes()); + let _ = hasher.write(format!("{}", self.obj_name).as_bytes()); + hasher.flush(); + xxh64::xxh64(hasher.clone().finalize().as_slice(), XXHASH_SEED) + } + + fn as_any(&self) -> &dyn Any { + self + } +} + +pub async fn delete_object_from_remote_tier(obj_name: &str, rv_id: &str, tier_name: &str) -> Result<(), std::io::Error> { + let mut config_mgr = GLOBAL_TierConfigMgr.write().await; + let w = match config_mgr.get_driver(tier_name).await { + Ok(w) => w, + Err(e) => return Err(std::io::Error::other(e)), + }; + w.remove(obj_name, rv_id).await +} + +#[cfg(test)] +mod test {} diff --git a/ecstore/src/bucket/mod.rs b/ecstore/src/bucket/mod.rs index a4e79c93b..bbe4bdaac 100644 --- a/ecstore/src/bucket/mod.rs +++ b/ecstore/src/bucket/mod.rs @@ -1,4 +1,5 @@ pub mod error; +pub mod lifecycle; pub mod metadata; pub mod metadata_sys; pub mod object_lock; diff --git a/ecstore/src/bucket/object_lock/mod.rs b/ecstore/src/bucket/object_lock/mod.rs index 38eb48861..64b834c94 100644 --- a/ecstore/src/bucket/object_lock/mod.rs +++ b/ecstore/src/bucket/object_lock/mod.rs @@ -1,3 +1,6 @@ +pub mod objectlock; +pub mod objectlock_sys; + use s3s::dto::{ObjectLockConfiguration, ObjectLockEnabled}; pub trait ObjectLockApi { diff --git a/ecstore/src/bucket/object_lock/objectlock.rs b/ecstore/src/bucket/object_lock/objectlock.rs new file mode 100644 index 000000000..032ad1305 --- /dev/null +++ b/ecstore/src/bucket/object_lock/objectlock.rs @@ -0,0 +1,100 @@ +use std::any::{Any, TypeId}; +use std::collections::HashMap; +use time::{OffsetDateTime, format_description}; +use tracing::{error, warn}; + +use s3s::dto::{Date, ObjectLockLegalHold, ObjectLockLegalHoldStatus, ObjectLockRetention, ObjectLockRetentionMode}; +use s3s::header::{X_AMZ_OBJECT_LOCK_LEGAL_HOLD, X_AMZ_OBJECT_LOCK_MODE, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE}; + +//const AMZ_OBJECTLOCK_BYPASS_RET_GOVERNANCE: &str = "X-Amz-Bypass-Governance-Retention"; +//const AMZ_OBJECTLOCK_RETAIN_UNTIL_DATE: &str = "X-Amz-Object-Lock-Retain-Until-Date"; +//const AMZ_OBJECTLOCK_MODE: &str = "X-Amz-Object-Lock-Mode"; +//const AMZ_OBJECTLOCK_LEGALHOLD: &str = "X-Amz-Object-Lock-Legal-Hold"; + +const ERR_MALFORMED_BUCKET_OBJECT_CONFIG: &str = "invalid bucket object lock config"; +const ERR_INVALID_RETENTION_DATE: &str = "date must be provided in ISO 8601 format"; +const ERR_PAST_OBJECTLOCK_RETAIN_DATE: &str = "the retain until date must be in the future"; +const ERR_UNKNOWN_WORMMODE_DIRECTIVE: &str = "unknown WORM mode directive"; +const ERR_OBJECTLOCK_MISSING_CONTENT_MD5: &str = + "content-MD5 HTTP header is required for Put Object requests with Object Lock parameters"; +const ERR_OBJECTLOCK_INVALID_HEADERS: &str = + "x-amz-object-lock-retain-until-date and x-amz-object-lock-mode must both be supplied"; +const ERR_MALFORMED_XML: &str = "the XML you provided was not well-formed or did not validate against our published schema"; + +pub fn utc_now_ntp() -> OffsetDateTime { + return OffsetDateTime::now_utc(); +} + +pub fn get_object_retention_meta(meta: HashMap) -> ObjectLockRetention { + let mode: ObjectLockRetentionMode; + let mut retain_until_date: Date = Date::from(OffsetDateTime::UNIX_EPOCH); + + let mut mode_str = meta.get(X_AMZ_OBJECT_LOCK_MODE.as_str().to_lowercase().as_str()); + if mode_str.is_none() { + mode_str = Some(&meta[X_AMZ_OBJECT_LOCK_MODE.as_str()]); + } + if let Some(mode_str) = mode_str { + mode = parse_ret_mode(mode_str.as_str()); + } else { + return ObjectLockRetention { + mode: None, + retain_until_date: None, + }; + } + + let mut till_str = meta.get(X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str().to_lowercase().as_str()); + if till_str.is_none() { + till_str = Some(&meta[X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str()]); + } + if let Some(till_str) = till_str { + let t = OffsetDateTime::parse(till_str, &format_description::well_known::Iso8601::DEFAULT); + if t.is_err() { + retain_until_date = Date::from(t.expect("err")); //TODO: utc + } + } + ObjectLockRetention { + mode: Some(mode), + retain_until_date: Some(retain_until_date), + } +} + +pub fn get_object_legalhold_meta(meta: HashMap) -> ObjectLockLegalHold { + let mut hold_str = meta.get(X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str().to_lowercase().as_str()); + if hold_str.is_none() { + hold_str = Some(&meta[X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str()]); + } + if let Some(hold_str) = hold_str { + return ObjectLockLegalHold { + status: Some(parse_legalhold_status(hold_str)), + }; + } + ObjectLockLegalHold { status: None } +} + +pub fn parse_ret_mode(mode_str: &str) -> ObjectLockRetentionMode { + let mut mode; + match mode_str.to_uppercase().as_str() { + "GOVERNANCE" => { + mode = ObjectLockRetentionMode::from_static(ObjectLockRetentionMode::GOVERNANCE); + } + "COMPLIANCE" => { + mode = ObjectLockRetentionMode::from_static(ObjectLockRetentionMode::COMPLIANCE); + } + _ => unreachable!(), + } + mode +} + +pub fn parse_legalhold_status(hold_str: &str) -> ObjectLockLegalHoldStatus { + let mut st; + match hold_str { + "ON" => { + st = ObjectLockLegalHoldStatus::from_static(ObjectLockLegalHoldStatus::ON); + } + "OFF" => { + st = ObjectLockLegalHoldStatus::from_static(ObjectLockLegalHoldStatus::OFF); + } + _ => unreachable!(), + } + st +} diff --git a/ecstore/src/bucket/object_lock/objectlock_sys.rs b/ecstore/src/bucket/object_lock/objectlock_sys.rs new file mode 100644 index 000000000..5c71a093b --- /dev/null +++ b/ecstore/src/bucket/object_lock/objectlock_sys.rs @@ -0,0 +1,58 @@ +use std::any::{Any, TypeId}; +use std::sync::Arc; +use time::OffsetDateTime; +use tracing::{error, warn}; + +use s3s::dto::{DefaultRetention, ObjectLockLegalHoldStatus, ObjectLockRetentionMode}; + +use crate::bucket::metadata_sys::get_object_lock_config; +use crate::store_api::ObjectInfo; + +use super::objectlock; + +pub struct BucketObjectLockSys {} + +impl BucketObjectLockSys { + #[allow(clippy::new_ret_no_self)] + pub async fn new() -> Arc { + Arc::new(Self {}) + } + + pub async fn get(bucket: &str) -> Option { + if let Some(object_lock_rule) = get_object_lock_config(bucket) + .await + .expect("get_object_lock_config err!") + .0 + .rule + { + return object_lock_rule.default_retention; + } + None + } +} + +pub fn enforce_retention_for_deletion(obj_info: &ObjectInfo) -> bool { + if obj_info.delete_marker { + return false; + } + + let lhold = objectlock::get_object_legalhold_meta(obj_info.user_defined.clone().expect("err")); + match lhold.status { + Some(st) if st.as_str() == ObjectLockLegalHoldStatus::ON => { + return true; + } + _ => (), + } + + let ret = objectlock::get_object_retention_meta(obj_info.user_defined.clone().expect("err")); + match ret.mode { + Some(r) if (r.as_str() == ObjectLockRetentionMode::COMPLIANCE || r.as_str() == ObjectLockRetentionMode::GOVERNANCE) => { + let t = objectlock::utc_now_ntp(); + if OffsetDateTime::from(ret.retain_until_date.expect("err!")).unix_timestamp() > t.unix_timestamp() { + return true; + } + } + _ => (), + } + false +} diff --git a/ecstore/src/checksum.rs b/ecstore/src/checksum.rs new file mode 100644 index 000000000..80d533132 --- /dev/null +++ b/ecstore/src/checksum.rs @@ -0,0 +1,298 @@ +#![allow(clippy::map_entry)] +use lazy_static::lazy_static; +use std::ops::{BitAnd, BitOr}; +use std::{collections::HashMap, sync::Arc}; + +use crate::client::{api_put_object::PutObjectOptions, api_s3_datatypes::ObjectPart}; +use crate::{disk::DiskAPI, store_api::GetObjectReader}; +use reader::hasher::{Hasher, Sha256}; +use rustfs_utils::crypto::{base64_decode, base64_encode}; +use s3s::header::{ + X_AMZ_CHECKSUM_ALGORITHM, X_AMZ_CHECKSUM_CRC32, X_AMZ_CHECKSUM_CRC32C, X_AMZ_CHECKSUM_SHA1, X_AMZ_CHECKSUM_SHA256, +}; + +use enumset::{EnumSet, EnumSetType, enum_set}; + +#[derive(Debug, EnumSetType, Default)] +#[enumset(repr = "u8")] +pub enum ChecksumMode { + #[default] + ChecksumNone, + ChecksumSHA256, + ChecksumSHA1, + ChecksumCRC32, + ChecksumCRC32C, + ChecksumCRC64NVME, + ChecksumFullObject, +} + +lazy_static! { + static ref C_ChecksumMask: EnumSet = { + let mut s = EnumSet::all(); + s.remove(ChecksumMode::ChecksumFullObject); + s + }; + static ref C_ChecksumFullObjectCRC32: EnumSet = + enum_set!(ChecksumMode::ChecksumCRC32 | ChecksumMode::ChecksumFullObject); + static ref C_ChecksumFullObjectCRC32C: EnumSet = + enum_set!(ChecksumMode::ChecksumCRC32C | ChecksumMode::ChecksumFullObject); +} +const AMZ_CHECKSUM_CRC64NVME: &str = "x-amz-checksum-crc64nvme"; + +impl ChecksumMode { + //pub const CRC64_NVME_POLYNOMIAL: i64 = 0xad93d23594c93659; + + pub fn base(&self) -> ChecksumMode { + let s = EnumSet::from(*self).intersection(*C_ChecksumMask); + match s.as_u8() { + 1_u8 => ChecksumMode::ChecksumNone, + 2_u8 => ChecksumMode::ChecksumSHA256, + 4_u8 => ChecksumMode::ChecksumSHA1, + 8_u8 => ChecksumMode::ChecksumCRC32, + 16_u8 => ChecksumMode::ChecksumCRC32C, + 32_u8 => ChecksumMode::ChecksumCRC64NVME, + _ => panic!("enum err."), + } + } + + pub fn is(&self, t: ChecksumMode) -> bool { + *self & t == t + } + + pub fn key(&self) -> String { + //match c & checksumMask { + match self { + ChecksumMode::ChecksumCRC32 => { + return X_AMZ_CHECKSUM_CRC32.to_string(); + } + ChecksumMode::ChecksumCRC32C => { + return X_AMZ_CHECKSUM_CRC32C.to_string(); + } + ChecksumMode::ChecksumSHA1 => { + return X_AMZ_CHECKSUM_SHA1.to_string(); + } + ChecksumMode::ChecksumSHA256 => { + return X_AMZ_CHECKSUM_SHA256.to_string(); + } + ChecksumMode::ChecksumCRC64NVME => { + return AMZ_CHECKSUM_CRC64NVME.to_string(); + } + _ => { + return "".to_string(); + } + } + } + + pub fn can_composite(&self) -> bool { + todo!(); + } + + pub fn can_merge_crc(&self) -> bool { + todo!(); + } + + pub fn full_object_requested(&self) -> bool { + todo!(); + } + + pub fn key_capitalized(&self) -> String { + self.key() + } + + pub fn raw_byte_len(&self) -> usize { + let u = EnumSet::from(*self).intersection(*C_ChecksumMask).as_u8(); + if u == ChecksumMode::ChecksumCRC32 as u8 || u == ChecksumMode::ChecksumCRC32C as u8 { + 4 + } else if u == ChecksumMode::ChecksumSHA1 as u8 { + 4 //sha1.size + } else if u == ChecksumMode::ChecksumSHA256 as u8 { + 4 //sha256.size + } else if u == ChecksumMode::ChecksumCRC64NVME as u8 { + 4 //crc64.size + } else { + 0 + } + } + + pub fn hasher(&self) -> Result, std::io::Error> { + match /*C_ChecksumMask & **/self { + /*ChecksumMode::ChecksumCRC32 => { + return Ok(Box::new(crc32fast::Hasher::new())); + }*/ + /*ChecksumMode::ChecksumCRC32C => { + return Ok(Box::new(crc32::new(crc32.MakeTable(crc32.Castagnoli)))); + } + ChecksumMode::ChecksumSHA1 => { + return Ok(Box::new(sha1::new())); + }*/ + ChecksumMode::ChecksumSHA256 => { + return Ok(Box::new(Sha256::new())); + } + /*ChecksumMode::ChecksumCRC64NVME => { + return Ok(Box::new(crc64nvme.New()); + }*/ + _ => return Err(std::io::Error::other("unsupported checksum type")), + } + } + + pub fn is_set(&self) -> bool { + let s = EnumSet::from(*self).intersection(*C_ChecksumMask); + s.len() == 1 + } + + pub fn set_default(&mut self, t: ChecksumMode) { + if !self.is_set() { + *self = t; + } + } + + pub fn encode_to_string(&self, b: &[u8]) -> Result { + if !self.is_set() { + return Ok("".to_string()); + } + let mut h = self.hasher()?; + h.write(b); + Ok(base64_encode(h.sum().as_bytes())) + } + + pub fn to_string(&self) -> String { + //match c & checksumMask { + match self { + ChecksumMode::ChecksumCRC32 => { + return "CRC32".to_string(); + } + ChecksumMode::ChecksumCRC32C => { + return "CRC32C".to_string(); + } + ChecksumMode::ChecksumSHA1 => { + return "SHA1".to_string(); + } + ChecksumMode::ChecksumSHA256 => { + return "SHA256".to_string(); + } + ChecksumMode::ChecksumNone => { + return "".to_string(); + } + ChecksumMode::ChecksumCRC64NVME => { + return "CRC64NVME".to_string(); + } + _ => { + return "".to_string(); + } + } + } + + pub fn check_sum_reader(&self, r: GetObjectReader) -> Result { + let mut h = self.hasher()?; + Ok(Checksum::new(self.clone(), h.sum().as_bytes())) + } + + pub fn check_sum_bytes(&self, b: &[u8]) -> Result { + let mut h = self.hasher()?; + Ok(Checksum::new(self.clone(), h.sum().as_bytes())) + } + + pub fn composite_checksum(&self, p: &mut [ObjectPart]) -> Result { + if !self.can_composite() { + return Err(std::io::Error::other("cannot do composite checksum")); + } + p.sort_by(|i, j| { + if i.part_num < j.part_num { + std::cmp::Ordering::Less + } else if i.part_num > j.part_num { + std::cmp::Ordering::Greater + } else { + std::cmp::Ordering::Equal + } + }); + let c = self.base(); + let mut crc_bytes = Vec::::with_capacity(p.len() * self.raw_byte_len() as usize); + let mut h = self.hasher()?; + h.write(&crc_bytes); + Ok(Checksum { + checksum_type: self.clone(), + r: h.sum().as_bytes().to_vec(), + }) + } + + pub fn full_object_checksum(&self, p: &mut [ObjectPart]) -> Result { + todo!(); + } +} + +#[derive(Default)] +struct Checksum { + checksum_type: ChecksumMode, + r: Vec, +} + +impl Checksum { + fn new(t: ChecksumMode, b: &[u8]) -> Checksum { + if t.is_set() && b.len() == t.raw_byte_len() { + return Checksum { + checksum_type: t, + r: b.to_vec(), + }; + } + Checksum::default() + } + + fn new_checksum_string(t: ChecksumMode, s: &str) -> Result { + let b = match base64_decode(s.as_bytes()) { + Ok(b) => b, + Err(err) => return Err(std::io::Error::other(err.to_string())), + }; + if t.is_set() && b.len() == t.raw_byte_len() { + return Ok(Checksum { checksum_type: t, r: b }); + } + Ok(Checksum::default()) + } + + fn is_set(&self) -> bool { + self.checksum_type.is_set() && self.r.len() == self.checksum_type.raw_byte_len() + } + + fn encoded(&self) -> String { + if !self.is_set() { + return "".to_string(); + } + base64_encode(&self.r) + } + + fn raw(&self) -> Option> { + if !self.is_set() { + return None; + } + Some(self.r.clone()) + } +} + +pub fn add_auto_checksum_headers(opts: &mut PutObjectOptions) { + opts.user_metadata + .insert("X-Amz-Checksum-Algorithm".to_string(), opts.auto_checksum.to_string()); + if opts.auto_checksum.full_object_requested() { + opts.user_metadata + .insert("X-Amz-Checksum-Type".to_string(), "FULL_OBJECT".to_string()); + } +} + +pub fn apply_auto_checksum(opts: &mut PutObjectOptions, all_parts: &mut [ObjectPart]) -> Result<(), std::io::Error> { + if opts.auto_checksum.can_composite() && !opts.auto_checksum.is(ChecksumMode::ChecksumFullObject) { + let crc = opts.auto_checksum.composite_checksum(all_parts)?; + opts.user_metadata = { + let mut hm = HashMap::new(); + hm.insert(opts.auto_checksum.key(), crc.encoded()); + hm + } + } else if opts.auto_checksum.can_merge_crc() { + let crc = opts.auto_checksum.full_object_checksum(all_parts)?; + opts.user_metadata = { + let mut hm = HashMap::new(); + hm.insert(opts.auto_checksum.key_capitalized(), crc.encoded()); + hm.insert("X-Amz-Checksum-Type".to_string(), "FULL_OBJECT".to_string()); + hm + } + } + + Ok(()) +} diff --git a/ecstore/src/client/admin_handler_utils.rs b/ecstore/src/client/admin_handler_utils.rs new file mode 100644 index 000000000..91e047f6f --- /dev/null +++ b/ecstore/src/client/admin_handler_utils.rs @@ -0,0 +1,33 @@ +use http::status::StatusCode; +use std::fmt::{self, Display, Formatter}; + +#[derive(Default, thiserror::Error, Debug, PartialEq)] +pub struct AdminError { + pub code: &'static str, + pub message: &'static str, + pub status_code: StatusCode, +} + +impl Display for AdminError { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + write!(f, "{}", self.message) + } +} + +impl AdminError { + pub fn new(code: &'static str, message: &'static str, status_code: StatusCode) -> Self { + Self { + code, + message, + status_code, + } + } + + pub fn msg(message: &'static str) -> Self { + Self { + code: "InternalError", + message, + status_code: StatusCode::INTERNAL_SERVER_ERROR, + } + } +} diff --git a/ecstore/src/client/api_bucket_policy.rs b/ecstore/src/client/api_bucket_policy.rs new file mode 100644 index 000000000..0e5b33aad --- /dev/null +++ b/ecstore/src/client/api_bucket_policy.rs @@ -0,0 +1,123 @@ +#![allow(clippy::map_entry)] +use bytes::Bytes; +use http::{HeaderMap, StatusCode}; +use std::collections::HashMap; + +use crate::client::{ + api_error_response::http_resp_to_error_response, + transition_api::{ReaderImpl, RequestMetadata, TransitionClient}, +}; +use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH; + +impl TransitionClient { + pub async fn set_bucket_policy(&self, bucket_name: &str, policy: &str) -> Result<(), std::io::Error> { + if policy == "" { + return self.remove_bucket_policy(bucket_name).await; + } + + self.put_bucket_policy(bucket_name, policy).await + } + + pub async fn put_bucket_policy(&self, bucket_name: &str, policy: &str) -> Result<(), std::io::Error> { + let mut url_values = HashMap::new(); + url_values.insert("policy".to_string(), "".to_string()); + + let mut req_metadata = RequestMetadata { + bucket_name: bucket_name.to_string(), + query_values: url_values, + content_body: ReaderImpl::Body(Bytes::from(policy.as_bytes().to_vec())), + content_length: policy.len() as i64, + object_name: "".to_string(), + custom_header: HeaderMap::new(), + content_md5_base64: "".to_string(), + content_sha256_hex: "".to_string(), + stream_sha256: false, + trailer: HeaderMap::new(), + pre_sign_url: Default::default(), + add_crc: Default::default(), + extra_pre_sign_header: Default::default(), + bucket_location: Default::default(), + expires: Default::default(), + }; + + let resp = self.execute_method(http::Method::PUT, &mut req_metadata).await?; + //defer closeResponse(resp) + //if resp != nil { + if resp.status() != StatusCode::NO_CONTENT && resp.status() != StatusCode::OK { + return Err(std::io::Error::other(http_resp_to_error_response(resp, vec![], bucket_name, ""))); + } + //} + Ok(()) + } + + pub async fn remove_bucket_policy(&self, bucket_name: &str) -> Result<(), std::io::Error> { + let mut url_values = HashMap::new(); + url_values.insert("policy".to_string(), "".to_string()); + + let resp = self + .execute_method( + http::Method::DELETE, + &mut RequestMetadata { + bucket_name: bucket_name.to_string(), + query_values: url_values, + content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(), + object_name: "".to_string(), + custom_header: HeaderMap::new(), + content_body: ReaderImpl::Body(Bytes::new()), + content_length: 0, + content_md5_base64: "".to_string(), + stream_sha256: false, + trailer: HeaderMap::new(), + pre_sign_url: Default::default(), + add_crc: Default::default(), + extra_pre_sign_header: Default::default(), + bucket_location: Default::default(), + expires: Default::default(), + }, + ) + .await?; + //defer closeResponse(resp) + + if resp.status() != StatusCode::NO_CONTENT { + return Err(std::io::Error::other(http_resp_to_error_response(resp, vec![], bucket_name, ""))); + } + + Ok(()) + } + + pub async fn get_bucket_policy(&self, bucket_name: &str) -> Result { + let bucket_policy = self.get_bucket_policy_inner(bucket_name).await?; + Ok(bucket_policy) + } + + pub async fn get_bucket_policy_inner(&self, bucket_name: &str) -> Result { + let mut url_values = HashMap::new(); + url_values.insert("policy".to_string(), "".to_string()); + + let resp = self + .execute_method( + http::Method::GET, + &mut RequestMetadata { + bucket_name: bucket_name.to_string(), + query_values: url_values, + content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(), + object_name: "".to_string(), + custom_header: HeaderMap::new(), + content_body: ReaderImpl::Body(Bytes::new()), + content_length: 0, + content_md5_base64: "".to_string(), + stream_sha256: false, + trailer: HeaderMap::new(), + pre_sign_url: Default::default(), + add_crc: Default::default(), + extra_pre_sign_header: Default::default(), + bucket_location: Default::default(), + expires: Default::default(), + }, + ) + .await?; + + let policy = String::from_utf8_lossy(&resp.body().bytes().expect("err").to_vec()).to_string(); + Ok(policy) + } +} diff --git a/ecstore/src/client/api_error_response.rs b/ecstore/src/client/api_error_response.rs new file mode 100644 index 000000000..5cc9e3014 --- /dev/null +++ b/ecstore/src/client/api_error_response.rs @@ -0,0 +1,266 @@ +#![allow(clippy::map_entry)] +use http::StatusCode; +use serde::{Deserialize, Serialize}; +use serde::{de::Deserializer, ser::Serializer}; +use std::fmt::Display; + +use s3s::Body; +use s3s::S3ErrorCode; + +const REPORT_ISSUE: &str = "Please report this issue at https://github.com/rustfs/rustfs/issues."; + +#[derive(Serialize, Deserialize, Debug, Clone, thiserror::Error, PartialEq, Eq)] +#[serde(default, rename_all = "PascalCase")] +pub struct ErrorResponse { + #[serde(serialize_with = "serialize_code", deserialize_with = "deserialize_code")] + pub code: S3ErrorCode, + pub message: String, + pub bucket_name: String, + pub key: String, + pub resource: String, + pub request_id: String, + pub host_id: String, + pub region: String, + pub server: String, + #[serde(skip)] + pub status_code: StatusCode, +} + +fn serialize_code(data: &S3ErrorCode, s: S) -> Result +where + S: Serializer, +{ + s.serialize_str("") +} + +fn deserialize_code<'de, D>(d: D) -> Result +where + D: Deserializer<'de>, +{ + Ok(S3ErrorCode::from_bytes(String::deserialize(d)?.as_bytes()).unwrap_or(S3ErrorCode::Custom("".into()))) +} + +impl Default for ErrorResponse { + fn default() -> Self { + ErrorResponse { + code: S3ErrorCode::Custom("".into()), + message: Default::default(), + bucket_name: Default::default(), + key: Default::default(), + resource: Default::default(), + request_id: Default::default(), + host_id: Default::default(), + region: Default::default(), + server: Default::default(), + status_code: Default::default(), + } + } +} + +impl Display for ErrorResponse { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.message) + } +} + +pub fn to_error_response(err: &std::io::Error) -> ErrorResponse { + if let Some(err) = err.get_ref() { + if err.is::() { + err.downcast_ref::().expect("err!").clone() + } else { + ErrorResponse::default() + } + } else { + ErrorResponse::default() + } +} + +pub fn http_resp_to_error_response( + resp: http::Response, + b: Vec, + bucket_name: &str, + object_name: &str, +) -> ErrorResponse { + let err_body = String::from_utf8(b).unwrap(); + //let err_body = xml_decode_and_body(resp.body, &err_resp); + let err_resp_ = serde_xml_rs::from_str::(&err_body); + let mut err_resp = ErrorResponse::default(); + if err_resp_.is_err() { + match resp.status() { + StatusCode::NOT_FOUND => { + if object_name == "" { + err_resp = ErrorResponse { + status_code: resp.status(), + code: S3ErrorCode::NoSuchBucket, + message: "The specified bucket does not exist.".to_string(), + bucket_name: bucket_name.to_string(), + ..Default::default() + }; + } else { + err_resp = ErrorResponse { + status_code: resp.status(), + code: S3ErrorCode::NoSuchKey, + message: "The specified key does not exist.".to_string(), + bucket_name: bucket_name.to_string(), + key: object_name.to_string(), + ..Default::default() + }; + } + } + StatusCode::FORBIDDEN => { + err_resp = ErrorResponse { + status_code: resp.status(), + code: S3ErrorCode::AccessDenied, + message: "Access Denied.".to_string(), + bucket_name: bucket_name.to_string(), + key: object_name.to_string(), + ..Default::default() + }; + } + StatusCode::CONFLICT => { + err_resp = ErrorResponse { + status_code: resp.status(), + code: S3ErrorCode::BucketNotEmpty, + message: "Bucket not empty.".to_string(), + bucket_name: bucket_name.to_string(), + ..Default::default() + }; + } + StatusCode::PRECONDITION_FAILED => { + err_resp = ErrorResponse { + status_code: resp.status(), + code: S3ErrorCode::PreconditionFailed, + message: "Pre condition failed.".to_string(), + bucket_name: bucket_name.to_string(), + key: object_name.to_string(), + ..Default::default() + }; + } + _ => { + let mut msg = resp.status().to_string(); + if err_body.len() > 0 { + msg = err_body; + } + err_resp = ErrorResponse { + status_code: resp.status(), + code: S3ErrorCode::Custom(resp.status().to_string().into()), + message: msg, + bucket_name: bucket_name.to_string(), + ..Default::default() + }; + } + } + } else { + err_resp = err_resp_.unwrap(); + } + err_resp.status_code = resp.status(); + if let Some(server_name) = resp.headers().get("Server") { + err_resp.server = server_name.to_str().expect("err").to_string(); + } + + let code = resp.headers().get("x-minio-error-code"); + if code.is_some() { + err_resp.code = S3ErrorCode::Custom(code.expect("err").to_str().expect("err").into()); + } + let desc = resp.headers().get("x-minio-error-desc"); + if desc.is_some() { + err_resp.message = desc.expect("err").to_str().expect("err").trim_matches('"').to_string(); + } + + if err_resp.request_id == "" { + if let Some(x_amz_request_id) = resp.headers().get("x-amz-request-id") { + err_resp.request_id = x_amz_request_id.to_str().expect("err").to_string(); + } + } + if err_resp.host_id == "" { + if let Some(x_amz_id_2) = resp.headers().get("x-amz-id-2") { + err_resp.host_id = x_amz_id_2.to_str().expect("err").to_string(); + } + } + if err_resp.region == "" { + if let Some(x_amz_bucket_region) = resp.headers().get("x-amz-bucket-region") { + err_resp.region = x_amz_bucket_region.to_str().expect("err").to_string(); + } + } + if err_resp.code == S3ErrorCode::InvalidLocationConstraint/*InvalidRegion*/ && err_resp.region != "" { + err_resp.message = format!("Region does not match, expecting region ‘{}’.", err_resp.region); + } + + err_resp +} + +pub fn err_transfer_acceleration_bucket(bucket_name: &str) -> ErrorResponse { + ErrorResponse { + status_code: StatusCode::BAD_REQUEST, + code: S3ErrorCode::InvalidArgument, + message: "The name of the bucket used for Transfer Acceleration must be DNS-compliant and must not contain periods ‘.’." + .to_string(), + bucket_name: bucket_name.to_string(), + ..Default::default() + } +} + +pub fn err_entity_too_large(total_size: i64, max_object_size: i64, bucket_name: &str, object_name: &str) -> ErrorResponse { + let msg = format!( + "Your proposed upload size ‘{}’ exceeds the maximum allowed object size ‘{}’ for single PUT operation.", + total_size, max_object_size + ); + ErrorResponse { + status_code: StatusCode::BAD_REQUEST, + code: S3ErrorCode::EntityTooLarge, + message: msg, + bucket_name: bucket_name.to_string(), + key: object_name.to_string(), + ..Default::default() + } +} + +pub fn err_entity_too_small(total_size: i64, bucket_name: &str, object_name: &str) -> ErrorResponse { + let msg = format!( + "Your proposed upload size ‘{}’ is below the minimum allowed object size ‘0B’ for single PUT operation.", + total_size + ); + ErrorResponse { + status_code: StatusCode::BAD_REQUEST, + code: S3ErrorCode::EntityTooSmall, + message: msg, + bucket_name: bucket_name.to_string(), + key: object_name.to_string(), + ..Default::default() + } +} + +pub fn err_unexpected_eof(total_read: i64, total_size: i64, bucket_name: &str, object_name: &str) -> ErrorResponse { + let msg = format!( + "Data read ‘{}’ is not equal to the size ‘{}’ of the input Reader.", + total_read, total_size + ); + ErrorResponse { + status_code: StatusCode::BAD_REQUEST, + code: S3ErrorCode::Custom("UnexpectedEOF".into()), + message: msg, + bucket_name: bucket_name.to_string(), + key: object_name.to_string(), + ..Default::default() + } +} + +pub fn err_invalid_argument(message: &str) -> ErrorResponse { + ErrorResponse { + status_code: StatusCode::BAD_REQUEST, + code: S3ErrorCode::InvalidArgument, + message: message.to_string(), + request_id: "rustfs".to_string(), + ..Default::default() + } +} + +pub fn err_api_not_supported(message: &str) -> ErrorResponse { + ErrorResponse { + status_code: StatusCode::NOT_IMPLEMENTED, + code: S3ErrorCode::Custom("APINotSupported".into()), + message: message.to_string(), + request_id: "rustfs".to_string(), + ..Default::default() + } +} diff --git a/ecstore/src/client/api_get_object.rs b/ecstore/src/client/api_get_object.rs new file mode 100644 index 000000000..efd5d573b --- /dev/null +++ b/ecstore/src/client/api_get_object.rs @@ -0,0 +1,206 @@ +#![allow(clippy::map_entry)] +use bytes::Bytes; +use http::HeaderMap; +use std::io::Cursor; +use tokio::io::BufReader; + +use crate::client::{ + api_error_response::err_invalid_argument, + api_get_options::GetObjectOptions, + transition_api::{ObjectInfo, ReadCloser, ReaderImpl, RequestMetadata, TransitionClient, to_object_info}, +}; +use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH; + +impl TransitionClient { + pub fn get_object(&self, bucket_name: &str, object_name: &str, opts: &GetObjectOptions) -> Result { + todo!(); + } + + pub async fn get_object_inner( + &self, + bucket_name: &str, + object_name: &str, + opts: &GetObjectOptions, + ) -> Result<(ObjectInfo, HeaderMap, ReadCloser), std::io::Error> { + let resp = self + .execute_method( + http::Method::GET, + &mut RequestMetadata { + bucket_name: bucket_name.to_string(), + object_name: object_name.to_string(), + query_values: opts.to_query_values(), + custom_header: opts.header(), + content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(), + content_body: ReaderImpl::Body(Bytes::new()), + content_length: 0, + content_md5_base64: "".to_string(), + stream_sha256: false, + trailer: HeaderMap::new(), + pre_sign_url: Default::default(), + add_crc: Default::default(), + extra_pre_sign_header: Default::default(), + bucket_location: Default::default(), + expires: Default::default(), + }, + ) + .await?; + + let resp = &resp; + let object_stat = to_object_info(bucket_name, object_name, resp.headers())?; + + let b = resp.body().bytes().expect("err").to_vec(); + Ok((object_stat, resp.headers().clone(), BufReader::new(Cursor::new(b)))) + } +} + +#[derive(Default)] +struct GetRequest { + pub buffer: Vec, + pub offset: i64, + pub did_offset_change: bool, + pub been_read: bool, + pub is_read_at: bool, + pub is_read_op: bool, + pub is_first_req: bool, + pub setting_object_info: bool, +} + +struct GetResponse { + pub size: i64, + //pub error: error, + pub did_read: bool, + pub object_info: ObjectInfo, +} + +#[derive(Default)] +struct Object { + //pub reqch: chan<- getRequest, + //pub resch: <-chan getResponse, + //pub cancel: context.CancelFunc, + pub curr_offset: i64, + pub object_info: ObjectInfo, + pub seek_data: bool, + pub is_closed: bool, + pub is_started: bool, + //pub prev_err: error, + pub been_read: bool, + pub object_info_set: bool, +} + +impl Object { + pub fn new() -> Object { + Self { ..Default::default() } + } + + fn do_get_request(&self, request: &GetRequest) -> Result { + todo!() + } + + fn set_offset(&mut self, bytes_read: i64) -> Result<(), std::io::Error> { + self.curr_offset += bytes_read; + + Ok(()) + } + + fn read(&mut self, b: &[u8]) -> Result { + let mut read_req = GetRequest { + is_read_op: true, + been_read: self.been_read, + buffer: b.to_vec(), + ..Default::default() + }; + + if !self.is_started { + read_req.is_first_req = true; + } + + read_req.did_offset_change = self.seek_data; + read_req.offset = self.curr_offset; + + let response = self.do_get_request(&read_req)?; + + let bytes_read = response.size; + + let oerr = self.set_offset(bytes_read); + + Ok(response.size) + } + + fn stat(&self) -> Result { + if !self.is_started || !self.object_info_set { + let _ = self.do_get_request(&GetRequest { + is_first_req: !self.is_started, + setting_object_info: !self.object_info_set, + ..Default::default() + })?; + } + + Ok(self.object_info.clone()) + } + + fn read_at(&mut self, b: &[u8], offset: i64) -> Result { + self.curr_offset = offset; + + let mut read_at_req = GetRequest { + is_read_op: true, + is_read_at: true, + did_offset_change: true, + been_read: self.been_read, + offset, + buffer: b.to_vec(), + ..Default::default() + }; + + if !self.is_started { + read_at_req.is_first_req = true; + } + + let response = self.do_get_request(&read_at_req)?; + let bytes_read = response.size; + if !self.object_info_set { + self.curr_offset += bytes_read; + } else { + let oerr = self.set_offset(bytes_read); + } + Ok(response.size) + } + + fn seek(&mut self, offset: i64, whence: i64) -> Result { + if !self.is_started || !self.object_info_set { + let seek_req = GetRequest { + is_read_op: false, + offset: offset, + is_first_req: true, + ..Default::default() + }; + let _ = self.do_get_request(&seek_req); + } + + let mut new_offset = self.curr_offset; + + match whence { + 0 => { + new_offset = offset; + } + 1 => { + new_offset += offset; + } + 2 => { + new_offset = self.object_info.size as i64 + offset as i64; + } + _ => { + return Err(std::io::Error::other(err_invalid_argument(&format!("Invalid whence {}", whence)))); + } + } + + self.seek_data = (new_offset != self.curr_offset) || self.seek_data; + self.curr_offset = new_offset; + + Ok(self.curr_offset) + } + + fn close(&mut self) -> Result<(), std::io::Error> { + self.is_closed = true; + Ok(()) + } +} diff --git a/ecstore/src/client/api_get_options.rs b/ecstore/src/client/api_get_options.rs new file mode 100644 index 000000000..2faab80ad --- /dev/null +++ b/ecstore/src/client/api_get_options.rs @@ -0,0 +1,127 @@ +#![allow(clippy::map_entry)] +use http::{HeaderMap, HeaderName, HeaderValue}; +use std::collections::HashMap; +use time::OffsetDateTime; +use tracing::warn; + +use crate::client::api_error_response::err_invalid_argument; + +#[derive(Default)] +pub struct AdvancedGetOptions { + replication_deletemarker: bool, + is_replication_ready_for_deletemarker: bool, + replication_proxy_request: String, +} + +pub struct GetObjectOptions { + pub headers: HashMap, + pub req_params: HashMap, + //pub server_side_encryption: encrypt.ServerSide, + pub version_id: String, + pub part_number: i64, + pub checksum: bool, + pub internal: AdvancedGetOptions, +} + +type StatObjectOptions = GetObjectOptions; + +impl Default for GetObjectOptions { + fn default() -> Self { + Self { + headers: HashMap::new(), + req_params: HashMap::new(), + //server_side_encryption: encrypt.ServerSide::default(), + version_id: "".to_string(), + part_number: 0, + checksum: false, + internal: AdvancedGetOptions::default(), + } + } +} + +impl GetObjectOptions { + pub fn header(&self) -> HeaderMap { + let mut headers: HeaderMap = HeaderMap::with_capacity(self.headers.len()); + for (k, v) in &self.headers { + if let Ok(header_name) = HeaderName::from_bytes(k.as_bytes()) { + headers.insert(header_name, v.parse().expect("err")); + } else { + warn!("Invalid header name: {}", k); + } + } + if self.checksum { + headers.insert("x-amz-checksum-mode", "ENABLED".parse().expect("err")); + } + headers + } + + pub fn set(&self, key: &str, value: &str) { + //self.headers[http.CanonicalHeaderKey(key)] = value; + } + + pub fn set_req_param(&mut self, key: &str, value: &str) { + self.req_params.insert(key.to_string(), value.to_string()); + } + + pub fn add_req_param(&mut self, key: &str, value: &str) { + self.req_params.insert(key.to_string(), value.to_string()); + } + + pub fn set_match_etag(&mut self, etag: &str) -> Result<(), std::io::Error> { + self.set("If-Match", &format!("\"{etag}\"")); + Ok(()) + } + + pub fn set_match_etag_except(&mut self, etag: &str) -> Result<(), std::io::Error> { + self.set("If-None-Match", &format!("\"{etag}\"")); + Ok(()) + } + + pub fn set_unmodified(&mut self, mod_time: OffsetDateTime) -> Result<(), std::io::Error> { + if mod_time.unix_timestamp() == 0 { + return Err(std::io::Error::other(err_invalid_argument("Modified since cannot be empty."))); + } + self.set("If-Unmodified-Since", &mod_time.to_string()); + Ok(()) + } + + pub fn set_modified(&mut self, mod_time: OffsetDateTime) -> Result<(), std::io::Error> { + if mod_time.unix_timestamp() == 0 { + return Err(std::io::Error::other(err_invalid_argument("Modified since cannot be empty."))); + } + self.set("If-Modified-Since", &mod_time.to_string()); + Ok(()) + } + + pub fn set_range(&mut self, start: i64, end: i64) -> Result<(), std::io::Error> { + if start == 0 && end < 0 { + self.set("Range", &format!("bytes={}", end)); + } else if 0 < start && end == 0 { + self.set("Range", &format!("bytes={}-", start)); + } else if 0 <= start && start <= end { + self.set("Range", &format!("bytes={}-{}", start, end)); + } else { + return Err(std::io::Error::other(err_invalid_argument(&format!( + "Invalid range specified: start={} end={}", + start, end + )))); + } + Ok(()) + } + + pub fn to_query_values(&self) -> HashMap { + let mut url_values = HashMap::new(); + if self.version_id != "" { + url_values.insert("versionId".to_string(), self.version_id.clone()); + } + if self.part_number > 0 { + url_values.insert("partNumber".to_string(), self.part_number.to_string()); + } + + for (key, value) in self.req_params.iter() { + url_values.insert(key.to_string(), value.to_string()); + } + + url_values + } +} diff --git a/ecstore/src/client/api_list.rs b/ecstore/src/client/api_list.rs new file mode 100644 index 000000000..d4c011078 --- /dev/null +++ b/ecstore/src/client/api_list.rs @@ -0,0 +1,282 @@ +#![allow(clippy::map_entry)] +use bytes::Bytes; +use http::{HeaderMap, StatusCode}; +use std::collections::HashMap; + +use crate::client::{ + api_error_response::http_resp_to_error_response, + api_s3_datatypes::{ + ListBucketResult, ListBucketV2Result, ListMultipartUploadsResult, ListObjectPartsResult, ListVersionsResult, ObjectPart, + }, + credentials, + transition_api::{ReaderImpl, RequestMetadata, TransitionClient}, +}; +use crate::store_api::BucketInfo; +use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH; + +impl TransitionClient { + pub fn list_buckets(&self) -> Result, std::io::Error> { + todo!(); + } + + pub async fn list_objects_v2_query( + &self, + bucket_name: &str, + object_prefix: &str, + continuation_token: &str, + fetch_owner: bool, + metadata: bool, + delimiter: &str, + start_after: &str, + max_keys: i64, + headers: HeaderMap, + ) -> Result { + let mut url_values = HashMap::new(); + + url_values.insert("list-type".to_string(), "2".to_string()); + if metadata { + url_values.insert("metadata".to_string(), "true".to_string()); + } + if start_after != "" { + url_values.insert("start-after".to_string(), start_after.to_string()); + } + url_values.insert("encoding-type".to_string(), "url".to_string()); + url_values.insert("prefix".to_string(), object_prefix.to_string()); + url_values.insert("delimiter".to_string(), delimiter.to_string()); + + if continuation_token != "" { + url_values.insert("continuation-token".to_string(), continuation_token.to_string()); + } + + if fetch_owner { + url_values.insert("fetch-owner".to_string(), "true".to_string()); + } + + if max_keys > 0 { + url_values.insert("max-keys".to_string(), max_keys.to_string()); + } + + let mut resp = self + .execute_method( + http::Method::GET, + &mut RequestMetadata { + bucket_name: bucket_name.to_string(), + object_name: "".to_string(), + query_values: url_values, + content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(), + custom_header: headers, + content_body: ReaderImpl::Body(Bytes::new()), + content_length: 0, + content_md5_base64: "".to_string(), + stream_sha256: false, + trailer: HeaderMap::new(), + pre_sign_url: Default::default(), + add_crc: Default::default(), + extra_pre_sign_header: Default::default(), + bucket_location: Default::default(), + expires: Default::default(), + }, + ) + .await?; + if resp.status() != StatusCode::OK { + return Err(std::io::Error::other(http_resp_to_error_response(resp, vec![], bucket_name, ""))); + } + + //let mut list_bucket_result = ListBucketV2Result::default(); + let b = resp.body_mut().store_all_unlimited().await.unwrap().to_vec(); + let mut list_bucket_result = match serde_xml_rs::from_str::(&String::from_utf8(b).unwrap()) { + Ok(result) => result, + Err(err) => { + return Err(std::io::Error::other(err.to_string())); + } + }; + //println!("list_bucket_result: {:?}", list_bucket_result); + + if list_bucket_result.is_truncated && list_bucket_result.next_continuation_token == "" { + return Err(std::io::Error::other(credentials::ErrorResponse { + sts_error: credentials::STSError { + r#type: "".to_string(), + code: "NotImplemented".to_string(), + message: "Truncated response should have continuation token set".to_string(), + }, + request_id: "".to_string(), + })); + } + + for (i, obj) in list_bucket_result.contents.iter_mut().enumerate() { + obj.name = decode_s3_name(&obj.name, &list_bucket_result.encoding_type)?; + //list_bucket_result.contents[i].mod_time = list_bucket_result.contents[i].mod_time.Truncate(time.Millisecond); + } + + for (i, obj) in list_bucket_result.common_prefixes.iter_mut().enumerate() { + obj.prefix = decode_s3_name(&obj.prefix, &list_bucket_result.encoding_type)?; + } + + Ok(list_bucket_result) + } + + pub fn list_object_versions_query( + &self, + bucket_name: &str, + opts: &ListObjectsOptions, + key_marker: &str, + version_id_marker: &str, + delimiter: &str, + ) -> Result { + /*if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return ListVersionsResult{}, err + } + if err := s3utils.CheckValidObjectNamePrefix(opts.Prefix); err != nil { + return ListVersionsResult{}, err + } + urlValues := make(url.Values) + + urlValues.Set("versions", "") + + urlValues.Set("prefix", opts.Prefix) + + urlValues.Set("delimiter", delimiter) + + if keyMarker != "" { + urlValues.Set("key-marker", keyMarker) + } + + if opts.max_keys > 0 { + urlValues.Set("max-keys", fmt.Sprintf("%d", opts.max_keys)) + } + + if versionIDMarker != "" { + urlValues.Set("version-id-marker", versionIDMarker) + } + + if opts.WithMetadata { + urlValues.Set("metadata", "true") + } + + urlValues.Set("encoding-type", "url") + + let resp = self.executeMethod(http::Method::GET, &mut RequestMetadata{ + bucketName: bucketName, + queryValues: urlValues, + contentSHA256Hex: emptySHA256Hex, + customHeader: opts.headers, + }).await?; + defer closeResponse(resp) + if err != nil { + return ListVersionsResult{}, err + } + if resp != nil { + if resp.StatusCode != http.StatusOK { + return ListVersionsResult{}, httpRespToErrorResponse(resp, bucketName, "") + } + } + + listObjectVersionsOutput := ListVersionsResult{} + err = xml_decoder(resp.Body, &listObjectVersionsOutput) + if err != nil { + return ListVersionsResult{}, err + } + + for i, obj := range listObjectVersionsOutput.Versions { + listObjectVersionsOutput.Versions[i].Key, err = decode_s3_name(obj.Key, listObjectVersionsOutput.EncodingType) + if err != nil { + return listObjectVersionsOutput, err + } + } + + for i, obj := range listObjectVersionsOutput.CommonPrefixes { + listObjectVersionsOutput.CommonPrefixes[i].Prefix, err = decode_s3_name(obj.Prefix, listObjectVersionsOutput.EncodingType) + if err != nil { + return listObjectVersionsOutput, err + } + } + + if listObjectVersionsOutput.NextKeyMarker != "" { + listObjectVersionsOutput.NextKeyMarker, err = decode_s3_name(listObjectVersionsOutput.NextKeyMarker, listObjectVersionsOutput.EncodingType) + if err != nil { + return listObjectVersionsOutput, err + } + } + + Ok(listObjectVersionsOutput)*/ + todo!(); + } + + pub fn list_objects_query( + &self, + bucket_name: &str, + object_prefix: &str, + object_marker: &str, + delimiter: &str, + max_keys: i64, + headers: HeaderMap, + ) -> Result { + todo!(); + } + + pub fn list_multipart_uploads_query( + &self, + bucket_name: &str, + key_marker: &str, + upload_id_marker: &str, + prefix: &str, + delimiter: &str, + max_uploads: i64, + ) -> Result { + todo!(); + } + + pub fn list_object_parts( + &self, + bucket_name: &str, + object_name: &str, + upload_id: &str, + ) -> Result, std::io::Error> { + todo!(); + } + + pub fn find_upload_ids(&self, bucket_name: &str, object_name: &str) -> Result, std::io::Error> { + todo!(); + } + + pub async fn list_object_parts_query( + &self, + bucket_name: &str, + object_name: &str, + upload_id: &str, + part_number_marker: i64, + max_parts: i64, + ) -> Result { + todo!(); + } +} + +pub struct ListObjectsOptions { + reverse_versions: bool, + with_versions: bool, + with_metadata: bool, + prefix: String, + recursive: bool, + max_keys: i64, + start_after: String, + use_v1: bool, + headers: HeaderMap, +} + +impl ListObjectsOptions { + pub fn set(&mut self, key: &str, value: &str) { + todo!(); + } +} + +fn decode_s3_name(name: &str, encoding_type: &str) -> Result { + match encoding_type { + "url" => { + //return url::QueryUnescape(name); + return Ok(name.to_string()); + } + _ => { + return Ok(name.to_string()); + } + } +} diff --git a/ecstore/src/client/api_put_object.rs b/ecstore/src/client/api_put_object.rs new file mode 100644 index 000000000..254e4f81a --- /dev/null +++ b/ecstore/src/client/api_put_object.rs @@ -0,0 +1,423 @@ +#![allow(clippy::map_entry)] +use bytes::Bytes; +use http::{HeaderMap, HeaderName, HeaderValue}; +use std::{collections::HashMap, sync::Arc}; +use time::{Duration, OffsetDateTime, macros::format_description}; +use tracing::{error, info, warn}; + +use reader::hasher::Hasher; +use s3s::dto::{ObjectLockLegalHoldStatus, ObjectLockRetentionMode, ReplicationStatus}; +use s3s::header::{ + X_AMZ_OBJECT_LOCK_LEGAL_HOLD, X_AMZ_OBJECT_LOCK_MODE, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, X_AMZ_REPLICATION_STATUS, + X_AMZ_STORAGE_CLASS, X_AMZ_WEBSITE_REDIRECT_LOCATION, +}; +//use crate::disk::{BufferReader, Reader}; +use crate::checksum::ChecksumMode; +use crate::client::{ + api_error_response::{err_entity_too_large, err_invalid_argument}, + api_put_object_common::optimal_part_info, + api_put_object_multipart::UploadPartParams, + api_s3_datatypes::{CompleteMultipartUpload, CompletePart, ObjectPart}, + constants::{ISO8601_DATEFORMAT, MAX_MULTIPART_PUT_OBJECT_SIZE, MIN_PART_SIZE, TOTAL_WORKERS}, + credentials::SignatureType, + transition_api::{ReaderImpl, TransitionClient, UploadInfo}, +}; +use rustfs_utils::{ + crypto::base64_encode, + net::{is_amz_header, is_minio_header, is_rustfs_header, is_standard_header, is_storageclass_header}, +}; + +#[derive(Debug, Clone)] +pub struct AdvancedPutOptions { + pub source_version_id: String, + pub source_etag: String, + pub replication_status: ReplicationStatus, + pub source_mtime: OffsetDateTime, + pub replication_request: bool, + pub retention_timestamp: OffsetDateTime, + pub tagging_timestamp: OffsetDateTime, + pub legalhold_timestamp: OffsetDateTime, + pub replication_validity_check: bool, +} + +impl Default for AdvancedPutOptions { + fn default() -> Self { + Self { + source_version_id: "".to_string(), + source_etag: "".to_string(), + replication_status: ReplicationStatus::from_static(ReplicationStatus::PENDING), + source_mtime: OffsetDateTime::now_utc(), + replication_request: false, + retention_timestamp: OffsetDateTime::now_utc(), + tagging_timestamp: OffsetDateTime::now_utc(), + legalhold_timestamp: OffsetDateTime::now_utc(), + replication_validity_check: false, + } + } +} + +#[derive(Clone)] +pub struct PutObjectOptions { + pub user_metadata: HashMap, + pub user_tags: HashMap, + //pub progress: ReaderImpl, + pub content_type: String, + pub content_encoding: String, + pub content_disposition: String, + pub content_language: String, + pub cache_control: String, + pub expires: OffsetDateTime, + pub mode: ObjectLockRetentionMode, + pub retain_until_date: OffsetDateTime, + //pub server_side_encryption: encrypt.ServerSide, + pub num_threads: u64, + pub storage_class: String, + pub website_redirect_location: String, + pub part_size: u64, + pub legalhold: ObjectLockLegalHoldStatus, + pub send_content_md5: bool, + pub disable_content_sha256: bool, + pub disable_multipart: bool, + pub auto_checksum: ChecksumMode, + pub checksum: ChecksumMode, + pub concurrent_stream_parts: bool, + pub internal: AdvancedPutOptions, + pub custom_header: HeaderMap, +} + +impl Default for PutObjectOptions { + fn default() -> Self { + Self { + user_metadata: HashMap::new(), + user_tags: HashMap::new(), + //progress: ReaderImpl::Body(Bytes::new()), + content_type: "".to_string(), + content_encoding: "".to_string(), + content_disposition: "".to_string(), + content_language: "".to_string(), + cache_control: "".to_string(), + expires: OffsetDateTime::UNIX_EPOCH, + mode: ObjectLockRetentionMode::from_static(""), + retain_until_date: OffsetDateTime::UNIX_EPOCH, + //server_side_encryption: encrypt.ServerSide::default(), + num_threads: 0, + storage_class: "".to_string(), + website_redirect_location: "".to_string(), + part_size: 0, + legalhold: ObjectLockLegalHoldStatus::from_static(ObjectLockLegalHoldStatus::OFF), + send_content_md5: false, + disable_content_sha256: false, + disable_multipart: false, + auto_checksum: ChecksumMode::ChecksumNone, + checksum: ChecksumMode::ChecksumNone, + concurrent_stream_parts: false, + internal: AdvancedPutOptions::default(), + custom_header: HeaderMap::new(), + } + } +} + +impl PutObjectOptions { + fn set_matche_tag(&mut self, etag: &str) { + if etag == "*" { + self.custom_header + .insert("If-Match", HeaderValue::from_str("*").expect("err")); + } else { + self.custom_header + .insert("If-Match", HeaderValue::from_str(&format!("\"{}\"", etag)).expect("err")); + } + } + + fn set_matche_tag_except(&mut self, etag: &str) { + if etag == "*" { + self.custom_header + .insert("If-None-Match", HeaderValue::from_str("*").expect("err")); + } else { + self.custom_header + .insert("If-None-Match", HeaderValue::from_str(&format!("\"{etag}\"")).expect("err")); + } + } + + pub fn header(&self) -> HeaderMap { + let mut header = HeaderMap::new(); + + let mut content_type = self.content_type.clone(); + if content_type == "" { + content_type = "application/octet-stream".to_string(); + } + header.insert("Content-Type", HeaderValue::from_str(&content_type).expect("err")); + + if self.content_encoding != "" { + header.insert("Content-Encoding", HeaderValue::from_str(&self.content_encoding).expect("err")); + } + if self.content_disposition != "" { + header.insert("Content-Disposition", HeaderValue::from_str(&self.content_disposition).expect("err")); + } + if self.content_language != "" { + header.insert("Content-Language", HeaderValue::from_str(&self.content_language).expect("err")); + } + if self.cache_control != "" { + header.insert("Cache-Control", HeaderValue::from_str(&self.cache_control).expect("err")); + } + + if self.expires.unix_timestamp() != 0 { + header.insert( + "Expires", + HeaderValue::from_str(&self.expires.format(ISO8601_DATEFORMAT).unwrap()).expect("err"), + ); //rustfs invalid heade + } + + if self.mode.as_str() != "" { + header.insert(X_AMZ_OBJECT_LOCK_MODE, HeaderValue::from_str(self.mode.as_str()).expect("err")); + } + + if self.retain_until_date.unix_timestamp() != 0 { + header.insert( + X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, + HeaderValue::from_str(&self.retain_until_date.format(ISO8601_DATEFORMAT).unwrap()).expect("err"), + ); + } + + if self.legalhold.as_str() != "" { + header.insert(X_AMZ_OBJECT_LOCK_LEGAL_HOLD, HeaderValue::from_str(self.legalhold.as_str()).expect("err")); + } + + if self.storage_class != "" { + header.insert(X_AMZ_STORAGE_CLASS, HeaderValue::from_str(&self.storage_class).expect("err")); + } + + if self.website_redirect_location != "" { + header.insert( + X_AMZ_WEBSITE_REDIRECT_LOCATION, + HeaderValue::from_str(&self.website_redirect_location).expect("err"), + ); + } + + if !self.internal.replication_status.as_str().is_empty() { + header.insert( + X_AMZ_REPLICATION_STATUS, + HeaderValue::from_str(self.internal.replication_status.as_str()).expect("err"), + ); + } + + for (k, v) in &self.user_metadata { + if is_amz_header(k) || is_standard_header(k) || is_storageclass_header(k) || is_rustfs_header(k) || is_minio_header(k) + { + if let Ok(header_name) = HeaderName::from_bytes(k.as_bytes()) { + header.insert(header_name, HeaderValue::from_str(&v).unwrap()); + } + } else { + if let Ok(header_name) = HeaderName::from_bytes(format!("x-amz-meta-{}", k).as_bytes()) { + header.insert(header_name, HeaderValue::from_str(&v).unwrap()); + } + } + } + + for (k, v) in self.custom_header.iter() { + header.insert(k.clone(), v.clone()); + } + + header + } + + fn validate(&self, c: TransitionClient) -> Result<(), std::io::Error> { + //if self.checksum.is_set() { + /*if !self.trailing_header_support { + return Err(Error::from(err_invalid_argument("Checksum requires Client with TrailingHeaders enabled"))); + }*/ + /*else if self.override_signer_type == SignatureType::SignatureV2 { + return Err(Error::from(err_invalid_argument("Checksum cannot be used with v2 signatures"))); + }*/ + //} + + Ok(()) + } +} + +impl TransitionClient { + pub async fn put_object( + self: Arc, + bucket_name: &str, + object_name: &str, + mut reader: ReaderImpl, + object_size: i64, + opts: &PutObjectOptions, + ) -> Result { + if object_size < 0 && opts.disable_multipart { + return Err(std::io::Error::other("object size must be provided with disable multipart upload")); + } + + self.put_object_common(bucket_name, object_name, reader, object_size, opts) + .await + } + + pub async fn put_object_common( + self: Arc, + bucket_name: &str, + object_name: &str, + mut reader: ReaderImpl, + size: i64, + opts: &PutObjectOptions, + ) -> Result { + if size > MAX_MULTIPART_PUT_OBJECT_SIZE { + return Err(std::io::Error::other(err_entity_too_large( + size, + MAX_MULTIPART_PUT_OBJECT_SIZE, + bucket_name, + object_name, + ))); + } + let mut opts = opts.clone(); + opts.auto_checksum.set_default(ChecksumMode::ChecksumCRC32C); + + let mut part_size = opts.part_size as i64; + if opts.part_size == 0 { + part_size = MIN_PART_SIZE; + } + + if SignatureType::SignatureV2 == self.override_signer_type { + if size >= 0 && size < part_size || opts.disable_multipart { + return self.put_object_gcs(bucket_name, object_name, reader, size, &opts).await; + } + return self.put_object_multipart(bucket_name, object_name, reader, size, &opts).await; + } + + if size < 0 { + if opts.disable_multipart { + return Err(std::io::Error::other("no length provided and multipart disabled")); + } + if opts.concurrent_stream_parts && opts.num_threads > 1 { + return self + .put_object_multipart_stream_parallel(bucket_name, object_name, reader, &opts) + .await; + } + return self + .put_object_multipart_stream_no_length(bucket_name, object_name, reader, &opts) + .await; + } + + if size <= part_size || opts.disable_multipart { + return self.put_object_gcs(bucket_name, object_name, reader, size, &opts).await; + } + + self.put_object_multipart_stream(bucket_name, object_name, reader, size, &opts) + .await + } + + pub async fn put_object_multipart_stream_no_length( + &self, + bucket_name: &str, + object_name: &str, + mut reader: ReaderImpl, + opts: &PutObjectOptions, + ) -> Result { + let mut total_uploaded_size: i64 = 0; + + let mut compl_multipart_upload = CompleteMultipartUpload::default(); + + let (total_parts_count, part_size, _) = optimal_part_info(-1, opts.part_size)?; + + let mut opts = opts.clone(); + + if opts.checksum.is_set() { + opts.send_content_md5 = false; + opts.auto_checksum = opts.checksum.clone(); + } + if !opts.send_content_md5 { + //add_auto_checksum_headers(&mut opts); + } + + let upload_id = self.new_upload_id(bucket_name, object_name, &opts).await?; + opts.user_metadata.remove("X-Amz-Checksum-Algorithm"); + + let mut part_number = 1; + let mut parts_info = HashMap::::new(); + let mut buf = Vec::::with_capacity(part_size as usize); + + let mut custom_header = HeaderMap::new(); + + while part_number <= total_parts_count { + buf = match &mut reader { + ReaderImpl::Body(content_body) => content_body.to_vec(), + ReaderImpl::ObjectBody(content_body) => content_body.read_all().await?, + }; + let length = buf.len(); + + let mut md5_base64: String = "".to_string(); + if opts.send_content_md5 { + let mut md5_hasher = self.md5_hasher.lock().unwrap(); + let mut hash = md5_hasher.as_mut().expect("err"); + hash.write(&buf[..length]); + md5_base64 = base64_encode(hash.sum().as_bytes()); + } else { + let csum; + { + let mut crc = opts.auto_checksum.hasher()?; + crc.reset(); + crc.write(&buf[..length]); + csum = crc.sum(); + } + if let Ok(header_name) = HeaderName::from_bytes(opts.auto_checksum.key().as_bytes()) { + custom_header.insert(header_name, base64_encode(csum.as_bytes()).parse().unwrap()); + } else { + warn!("Invalid header name: {}", opts.auto_checksum.key()); + } + } + + //let rd = newHook(bytes.NewReader(buf[..length]), opts.progress); + let rd = ReaderImpl::Body(Bytes::from(buf)); + + let mut p = UploadPartParams { + bucket_name: bucket_name.to_string(), + object_name: object_name.to_string(), + upload_id: upload_id.clone(), + reader: rd, + part_number, + md5_base64, + size: length as i64, + //sse: opts.server_side_encryption, + stream_sha256: !opts.disable_content_sha256, + custom_header: custom_header.clone(), + sha256_hex: Default::default(), + trailer: Default::default(), + }; + let obj_part = self.upload_part(&mut p).await?; + + parts_info.entry(part_number).or_insert(obj_part); + total_uploaded_size += length as i64; + part_number += 1; + } + + let mut all_parts = Vec::::with_capacity(parts_info.len()); + for i in 1..part_number { + let part = parts_info[&i].clone(); + all_parts.push(part.clone()); + compl_multipart_upload.parts.push(CompletePart { + etag: part.etag, + part_num: part.part_num, + checksum_crc32: part.checksum_crc32, + checksum_crc32c: part.checksum_crc32c, + checksum_sha1: part.checksum_sha1, + checksum_sha256: part.checksum_sha256, + checksum_crc64nvme: part.checksum_crc64nvme, + ..Default::default() + }); + } + + compl_multipart_upload.parts.sort(); + + let mut opts = PutObjectOptions { + //server_side_encryption: opts.server_side_encryption, + auto_checksum: opts.auto_checksum, + ..Default::default() + }; + //apply_auto_checksum(&mut opts, all_parts); + + let mut upload_info = self + .complete_multipart_upload(bucket_name, object_name, &upload_id, compl_multipart_upload, &opts) + .await?; + + upload_info.size = total_uploaded_size; + Ok(upload_info) + } +} diff --git a/ecstore/src/client/api_put_object_common.rs b/ecstore/src/client/api_put_object_common.rs new file mode 100644 index 000000000..ff63aa555 --- /dev/null +++ b/ecstore/src/client/api_put_object_common.rs @@ -0,0 +1,97 @@ +#![allow(clippy::map_entry)] +use crate::client::{ + api_error_response::{err_entity_too_large, err_invalid_argument}, + api_put_object::PutObjectOptions, + constants::{ABS_MIN_PART_SIZE, MAX_MULTIPART_PUT_OBJECT_SIZE, MAX_PART_SIZE, MAX_PARTS_COUNT, MIN_PART_SIZE}, + transition_api::ReaderImpl, + transition_api::TransitionClient, +}; + +const NULL_VERSION_ID: &str = "null"; + +pub fn is_object(reader: &ReaderImpl) -> bool { + todo!(); +} + +pub fn is_read_at(reader: ReaderImpl) -> bool { + todo!(); +} + +pub fn optimal_part_info(object_size: i64, configured_part_size: u64) -> Result<(i64, i64, i64), std::io::Error> { + let unknown_size; + let mut object_size = object_size; + if object_size == -1 { + unknown_size = true; + object_size = MAX_MULTIPART_PUT_OBJECT_SIZE; + } else { + unknown_size = false; + } + + if object_size > MAX_MULTIPART_PUT_OBJECT_SIZE { + return Err(std::io::Error::other(err_entity_too_large( + object_size, + MAX_MULTIPART_PUT_OBJECT_SIZE, + "", + "", + ))); + } + + let mut part_size_flt: f64; + if configured_part_size > 0 { + if configured_part_size as i64 > object_size { + return Err(std::io::Error::other(err_entity_too_large( + configured_part_size as i64, + object_size, + "", + "", + ))); + } + + if !unknown_size { + if object_size > (configured_part_size as i64 * MAX_PARTS_COUNT) { + return Err(std::io::Error::other(err_invalid_argument( + "Part size * max_parts(10000) is lesser than input objectSize.", + ))); + } + } + + if (configured_part_size as i64) < ABS_MIN_PART_SIZE { + return Err(std::io::Error::other(err_invalid_argument( + "Input part size is smaller than allowed minimum of 5MiB.", + ))); + } + + if configured_part_size as i64 > MAX_PART_SIZE { + return Err(std::io::Error::other(err_invalid_argument( + "Input part size is bigger than allowed maximum of 5GiB.", + ))); + } + + part_size_flt = configured_part_size as f64; + if unknown_size { + object_size = configured_part_size as i64 * MAX_PARTS_COUNT; + } + } else { + let mut configured_part_size = configured_part_size; + configured_part_size = MIN_PART_SIZE as u64; + part_size_flt = (object_size / MAX_PARTS_COUNT) as f64; + part_size_flt = (part_size_flt / configured_part_size as f64) * configured_part_size as f64; + } + + let total_parts_count = (object_size as f64 / part_size_flt).ceil() as i64; + let part_size = part_size_flt.ceil() as i64; + let last_part_size = object_size - (total_parts_count - 1) * part_size; + Ok((total_parts_count, part_size, last_part_size)) +} + +impl TransitionClient { + pub async fn new_upload_id( + &self, + bucket_name: &str, + object_name: &str, + opts: &PutObjectOptions, + ) -> Result { + let init_multipart_upload_result = self.initiate_multipart_upload(bucket_name, object_name, opts).await?; + Ok(init_multipart_upload_result.upload_id) + } +} diff --git a/ecstore/src/client/api_put_object_multipart.rs b/ecstore/src/client/api_put_object_multipart.rs new file mode 100644 index 000000000..3f0a3f0dc --- /dev/null +++ b/ecstore/src/client/api_put_object_multipart.rs @@ -0,0 +1,413 @@ +#![allow(clippy::map_entry)] +use bytes::Bytes; +use http::{HeaderMap, HeaderName, HeaderValue, StatusCode}; +use s3s::S3ErrorCode; +use std::io::Read; +use std::{collections::HashMap, sync::Arc}; +use time::{OffsetDateTime, format_description}; +use tokio_util::sync::CancellationToken; +use tracing::warn; +use tracing::{error, info}; +use url::form_urlencoded::Serializer; +use uuid::Uuid; + +use reader::hasher::Hasher; +use s3s::header::{X_AMZ_EXPIRATION, X_AMZ_VERSION_ID}; +use s3s::{Body, dto::StreamingBlob}; +//use crate::disk::{Reader, BufferReader}; +use crate::client::{ + api_error_response::{ + err_entity_too_large, err_entity_too_small, err_invalid_argument, http_resp_to_error_response, to_error_response, + }, + api_put_object::PutObjectOptions, + api_put_object_common::optimal_part_info, + api_s3_datatypes::{ + CompleteMultipartUpload, CompleteMultipartUploadResult, CompletePart, InitiateMultipartUploadResult, ObjectPart, + }, + constants::{ABS_MIN_PART_SIZE, ISO8601_DATEFORMAT, MAX_PART_SIZE, MAX_SINGLE_PUT_OBJECT_SIZE}, + transition_api::{ReaderImpl, RequestMetadata, TransitionClient, UploadInfo}, +}; +use crate::{ + checksum::ChecksumMode, + disk::DiskAPI, + store_api::{GetObjectReader, StorageAPI}, +}; +use rustfs_utils::{crypto::base64_encode, path::trim_etag}; + +impl TransitionClient { + pub async fn put_object_multipart( + &self, + bucket_name: &str, + object_name: &str, + mut reader: ReaderImpl, + size: i64, + opts: &PutObjectOptions, + ) -> Result { + let info = self + .put_object_multipart_no_stream(bucket_name, object_name, &mut reader, opts) + .await; + if let Err(err) = &info { + let err_resp = to_error_response(err); + if err_resp.code == S3ErrorCode::AccessDenied && err_resp.message.contains("Access Denied") { + if size > MAX_SINGLE_PUT_OBJECT_SIZE { + return Err(std::io::Error::other(err_entity_too_large( + size, + MAX_SINGLE_PUT_OBJECT_SIZE, + bucket_name, + object_name, + ))); + } + return self.put_object_gcs(bucket_name, object_name, reader, size, opts).await; + } + } + Ok(info?) + } + + pub async fn put_object_multipart_no_stream( + &self, + bucket_name: &str, + object_name: &str, + reader: &mut ReaderImpl, + opts: &PutObjectOptions, + ) -> Result { + let mut total_uploaded_size: i64 = 0; + let mut compl_multipart_upload = CompleteMultipartUpload::default(); + + let ret = optimal_part_info(-1, opts.part_size)?; + let (total_parts_count, part_size, _) = ret; + + let (mut hash_algos, mut hash_sums) = self.hash_materials(opts.send_content_md5, !opts.disable_content_sha256); + let upload_id = self.new_upload_id(bucket_name, object_name, opts).await?; + let mut opts = opts.clone(); + opts.user_metadata.remove("X-Amz-Checksum-Algorithm"); + + let mut part_number = 1; + let mut parts_info = HashMap::::new(); + let mut buf = Vec::::with_capacity(part_size as usize); + let mut custom_header = HeaderMap::new(); + while part_number <= total_parts_count { + match reader { + ReaderImpl::Body(content_body) => { + buf = content_body.to_vec(); + } + ReaderImpl::ObjectBody(content_body) => { + buf = content_body.read_all().await?; + } + } + let length = buf.len(); + + for (k, v) in hash_algos.iter_mut() { + v.write(&buf[..length]); + hash_sums.insert(k.to_string(), Vec::try_from(v.sum().as_bytes()).unwrap()); + } + + //let rd = newHook(bytes.NewReader(buf[..length]), opts.progress); + let rd = Bytes::from(buf.clone()); + + let mut md5_base64: String; + let mut sha256_hex: String; + + //if hash_sums["md5"] != nil { + md5_base64 = base64_encode(&hash_sums["md5"]); + //} + //if hash_sums["sha256"] != nil { + sha256_hex = hex_simd::encode_to_string(hash_sums["sha256"].clone(), hex_simd::AsciiCase::Lower); + //} + if hash_sums.len() == 0 { + let csum; + { + let mut crc = opts.auto_checksum.hasher()?; + crc.reset(); + crc.write(&buf[..length]); + csum = crc.sum(); + } + if let Ok(header_name) = HeaderName::from_bytes(opts.auto_checksum.key().as_bytes()) { + custom_header.insert(header_name, base64_encode(csum.as_bytes()).parse().expect("err")); + } else { + warn!("Invalid header name: {}", opts.auto_checksum.key()); + } + } + + let mut p = UploadPartParams { + bucket_name: bucket_name.to_string(), + object_name: object_name.to_string(), + upload_id: upload_id.clone(), + reader: ReaderImpl::Body(rd), + part_number, + md5_base64, + sha256_hex, + size: length as i64, + //sse: opts.server_side_encryption, + stream_sha256: !opts.disable_content_sha256, + custom_header: custom_header.clone(), + trailer: HeaderMap::new(), + }; + let obj_part = self.upload_part(&mut p).await?; + + parts_info.insert(part_number, obj_part); + total_uploaded_size += length as i64; + part_number += 1; + } + + let mut all_parts = Vec::::with_capacity(parts_info.len()); + for i in 1..part_number { + let part = parts_info[&i].clone(); + all_parts.push(part.clone()); + compl_multipart_upload.parts.push(CompletePart { + etag: part.etag, + part_num: part.part_num, + checksum_crc32: part.checksum_crc32, + checksum_crc32c: part.checksum_crc32c, + checksum_sha1: part.checksum_sha1, + checksum_sha256: part.checksum_sha256, + checksum_crc64nvme: part.checksum_crc64nvme, + ..Default::default() + }); + } + + compl_multipart_upload.parts.sort(); + let mut opts = PutObjectOptions { + //server_side_encryption: opts.server_side_encryption, + auto_checksum: opts.auto_checksum, + ..Default::default() + }; + //apply_auto_checksum(&mut opts, all_parts); + + let mut upload_info = self + .complete_multipart_upload(bucket_name, object_name, &upload_id, compl_multipart_upload, &opts) + .await?; + + upload_info.size = total_uploaded_size; + Ok(upload_info) + } + + pub async fn initiate_multipart_upload( + &self, + bucket_name: &str, + object_name: &str, + opts: &PutObjectOptions, + ) -> Result { + let mut url_values = HashMap::new(); + url_values.insert("uploads".to_string(), "".to_string()); + + if opts.internal.source_version_id != "" { + if !opts.internal.source_version_id.is_empty() { + if let Err(err) = Uuid::parse_str(&opts.internal.source_version_id) { + return Err(std::io::Error::other(err_invalid_argument(&err.to_string()))); + } + } + url_values.insert("versionId".to_string(), opts.internal.source_version_id.clone()); + } + + let mut custom_header = opts.header(); + + let mut req_metadata = RequestMetadata { + bucket_name: bucket_name.to_string(), + object_name: object_name.to_string(), + query_values: url_values, + custom_header, + content_body: ReaderImpl::Body(Bytes::new()), + content_length: 0, + content_md5_base64: "".to_string(), + content_sha256_hex: "".to_string(), + stream_sha256: false, + trailer: HeaderMap::new(), + pre_sign_url: Default::default(), + add_crc: Default::default(), + extra_pre_sign_header: Default::default(), + bucket_location: Default::default(), + expires: Default::default(), + }; + + let resp = self.execute_method(http::Method::POST, &mut req_metadata).await?; + //if resp.is_none() { + if resp.status() != StatusCode::OK { + return Err(std::io::Error::other(http_resp_to_error_response(resp, vec![], bucket_name, object_name))); + } + //} + let initiate_multipart_upload_result = InitiateMultipartUploadResult::default(); + Ok(initiate_multipart_upload_result) + } + + pub async fn upload_part(&self, p: &mut UploadPartParams) -> Result { + if p.size > MAX_PART_SIZE { + return Err(std::io::Error::other(err_entity_too_large( + p.size, + MAX_PART_SIZE, + &p.bucket_name, + &p.object_name, + ))); + } + if p.size <= -1 { + return Err(std::io::Error::other(err_entity_too_small(p.size, &p.bucket_name, &p.object_name))); + } + if p.part_number <= 0 { + return Err(std::io::Error::other(err_invalid_argument( + "Part number cannot be negative or equal to zero.", + ))); + } + if p.upload_id == "" { + return Err(std::io::Error::other(err_invalid_argument("UploadID cannot be empty."))); + } + + let mut url_values = HashMap::new(); + url_values.insert("partNumber".to_string(), p.part_number.to_string()); + url_values.insert("uploadId".to_string(), p.upload_id.clone()); + + let buf = match &mut p.reader { + ReaderImpl::Body(content_body) => content_body.to_vec(), + ReaderImpl::ObjectBody(content_body) => content_body.read_all().await?, + }; + let mut req_metadata = RequestMetadata { + bucket_name: p.bucket_name.clone(), + object_name: p.object_name.clone(), + query_values: url_values, + custom_header: p.custom_header.clone(), + content_body: ReaderImpl::Body(Bytes::from(buf)), + content_length: p.size, + content_md5_base64: p.md5_base64.clone(), + content_sha256_hex: p.sha256_hex.clone(), + stream_sha256: p.stream_sha256, + trailer: p.trailer.clone(), + pre_sign_url: Default::default(), + add_crc: Default::default(), + extra_pre_sign_header: Default::default(), + bucket_location: Default::default(), + expires: Default::default(), + }; + + let resp = self.execute_method(http::Method::PUT, &mut req_metadata).await?; + //defer closeResponse(resp) + //if resp.is_none() { + if resp.status() != StatusCode::OK { + return Err(std::io::Error::other(http_resp_to_error_response( + resp, + vec![], + &p.bucket_name.clone(), + &p.object_name, + ))); + } + //} + let h = resp.headers(); + let mut obj_part = ObjectPart { + checksum_crc32: if let Some(h_checksum_crc32) = h.get(ChecksumMode::ChecksumCRC32.key()) { + h_checksum_crc32.to_str().expect("err").to_string() + } else { + "".to_string() + }, + checksum_crc32c: if let Some(h_checksum_crc32c) = h.get(ChecksumMode::ChecksumCRC32C.key()) { + h_checksum_crc32c.to_str().expect("err").to_string() + } else { + "".to_string() + }, + checksum_sha1: if let Some(h_checksum_sha1) = h.get(ChecksumMode::ChecksumSHA1.key()) { + h_checksum_sha1.to_str().expect("err").to_string() + } else { + "".to_string() + }, + checksum_sha256: if let Some(h_checksum_sha256) = h.get(ChecksumMode::ChecksumSHA256.key()) { + h_checksum_sha256.to_str().expect("err").to_string() + } else { + "".to_string() + }, + checksum_crc64nvme: if let Some(h_checksum_crc64nvme) = h.get(ChecksumMode::ChecksumCRC64NVME.key()) { + h_checksum_crc64nvme.to_str().expect("err").to_string() + } else { + "".to_string() + }, + ..Default::default() + }; + obj_part.size = p.size; + obj_part.part_num = p.part_number; + obj_part.etag = if let Some(h_etag) = h.get("ETag") { + h_etag.to_str().expect("err").trim_matches('"').to_string() + } else { + "".to_string() + }; + Ok(obj_part) + } + + pub async fn complete_multipart_upload( + &self, + bucket_name: &str, + object_name: &str, + upload_id: &str, + complete: CompleteMultipartUpload, + opts: &PutObjectOptions, + ) -> Result { + let mut url_values = HashMap::new(); + url_values.insert("uploadId".to_string(), upload_id.to_string()); + let complete_multipart_upload_bytes = complete.marshal_msg()?.as_bytes().to_vec(); + + let mut headers = opts.header(); + + let complete_multipart_upload_buffer = Bytes::from(complete_multipart_upload_bytes); + let mut req_metadata = RequestMetadata { + bucket_name: bucket_name.to_string(), + object_name: object_name.to_string(), + query_values: url_values, + content_body: ReaderImpl::Body(complete_multipart_upload_buffer), + content_length: 100, //complete_multipart_upload_bytes.len(), + content_sha256_hex: "".to_string(), //hex_simd::encode_to_string(complete_multipart_upload_bytes, hex_simd::AsciiCase::Lower), + custom_header: headers, + stream_sha256: Default::default(), + trailer: Default::default(), + content_md5_base64: "".to_string(), + pre_sign_url: Default::default(), + add_crc: Default::default(), + extra_pre_sign_header: Default::default(), + bucket_location: Default::default(), + expires: Default::default(), + }; + + let resp = self.execute_method(http::Method::POST, &mut req_metadata).await?; + + let b = resp.body().bytes().expect("err").to_vec(); + let complete_multipart_upload_result: CompleteMultipartUploadResult = CompleteMultipartUploadResult::default(); + + let (exp_time, rule_id) = if let Some(h_x_amz_expiration) = resp.headers().get(X_AMZ_EXPIRATION) { + ( + OffsetDateTime::parse(h_x_amz_expiration.to_str().unwrap(), ISO8601_DATEFORMAT).unwrap(), + "".to_string(), + ) + } else { + (OffsetDateTime::now_utc(), "".to_string()) + }; + + let h = resp.headers(); + Ok(UploadInfo { + bucket: complete_multipart_upload_result.bucket, + key: complete_multipart_upload_result.key, + etag: trim_etag(&complete_multipart_upload_result.etag), + version_id: if let Some(h_x_amz_version_id) = h.get(X_AMZ_VERSION_ID) { + h_x_amz_version_id.to_str().expect("err").to_string() + } else { + "".to_string() + }, + location: complete_multipart_upload_result.location, + expiration: exp_time, + expiration_rule_id: rule_id, + checksum_sha256: complete_multipart_upload_result.checksum_sha256, + checksum_sha1: complete_multipart_upload_result.checksum_sha1, + checksum_crc32: complete_multipart_upload_result.checksum_crc32, + checksum_crc32c: complete_multipart_upload_result.checksum_crc32c, + checksum_crc64nvme: complete_multipart_upload_result.checksum_crc64nvme, + ..Default::default() + }) + } +} + +pub struct UploadPartParams { + pub bucket_name: String, + pub object_name: String, + pub upload_id: String, + pub reader: ReaderImpl, + pub part_number: i64, + pub md5_base64: String, + pub sha256_hex: String, + pub size: i64, + //pub sse: encrypt.ServerSide, + pub stream_sha256: bool, + pub custom_header: HeaderMap, + pub trailer: HeaderMap, +} diff --git a/ecstore/src/client/api_put_object_streaming.rs b/ecstore/src/client/api_put_object_streaming.rs new file mode 100644 index 000000000..ce10bbca1 --- /dev/null +++ b/ecstore/src/client/api_put_object_streaming.rs @@ -0,0 +1,526 @@ +#![allow(clippy::map_entry)] +use bytes::Bytes; +use futures::future::join_all; +use http::{HeaderMap, HeaderName, HeaderValue, StatusCode}; +use std::sync::RwLock; +use std::{collections::HashMap, sync::Arc}; +use time::{OffsetDateTime, format_description}; +use tokio::{select, sync::mpsc}; +use tokio_util::sync::CancellationToken; +use tracing::warn; +use uuid::Uuid; + +use crate::checksum::{ChecksumMode, add_auto_checksum_headers, apply_auto_checksum}; +use crate::client::{ + api_error_response::{err_invalid_argument, err_unexpected_eof, http_resp_to_error_response}, + api_put_object::PutObjectOptions, + api_put_object_common::{is_object, optimal_part_info}, + api_put_object_multipart::UploadPartParams, + api_s3_datatypes::{CompleteMultipartUpload, CompletePart, ObjectPart}, + constants::ISO8601_DATEFORMAT, + transition_api::{ReaderImpl, RequestMetadata, TransitionClient, UploadInfo}, +}; +use reader::hasher::Hasher; +use rustfs_utils::{crypto::base64_encode, path::trim_etag}; +use s3s::header::{X_AMZ_EXPIRATION, X_AMZ_VERSION_ID}; + +pub struct UploadedPartRes { + pub error: std::io::Error, + pub part_num: i64, + pub size: i64, + pub part: ObjectPart, +} + +pub struct UploadPartReq { + pub part_num: i64, + pub part: ObjectPart, +} + +impl TransitionClient { + pub async fn put_object_multipart_stream( + self: Arc, + bucket_name: &str, + object_name: &str, + mut reader: ReaderImpl, + size: i64, + opts: &PutObjectOptions, + ) -> Result { + let info: UploadInfo; + if opts.concurrent_stream_parts && opts.num_threads > 1 { + info = self + .put_object_multipart_stream_parallel(bucket_name, object_name, reader, opts) + .await?; + } else if !is_object(&reader) && !opts.send_content_md5 { + info = self + .put_object_multipart_stream_from_readat(bucket_name, object_name, reader, size, opts) + .await?; + } else { + info = self + .put_object_multipart_stream_optional_checksum(bucket_name, object_name, reader, size, opts) + .await?; + } + + Ok(info) + } + + pub async fn put_object_multipart_stream_from_readat( + &self, + bucket_name: &str, + object_name: &str, + mut reader: ReaderImpl, + size: i64, + opts: &PutObjectOptions, + ) -> Result { + let ret = optimal_part_info(size, opts.part_size)?; + let (total_parts_count, part_size, lastpart_size) = ret; + let mut opts = opts.clone(); + if opts.checksum.is_set() { + opts.auto_checksum = opts.checksum.clone(); + } + let with_checksum = self.trailing_header_support; + let upload_id = self.new_upload_id(bucket_name, object_name, &opts).await?; + opts.user_metadata.remove("X-Amz-Checksum-Algorithm"); + + todo!(); + } + + pub async fn put_object_multipart_stream_optional_checksum( + &self, + bucket_name: &str, + object_name: &str, + mut reader: ReaderImpl, + size: i64, + opts: &PutObjectOptions, + ) -> Result { + let mut opts = opts.clone(); + if opts.checksum.is_set() { + opts.auto_checksum = opts.checksum.clone(); + opts.send_content_md5 = false; + } + + if !opts.send_content_md5 { + add_auto_checksum_headers(&mut opts); + } + + let ret = optimal_part_info(size, opts.part_size)?; + let (total_parts_count, mut part_size, lastpart_size) = ret; + let upload_id = self.new_upload_id(bucket_name, object_name, &opts).await?; + opts.user_metadata.remove("X-Amz-Checksum-Algorithm"); + + let mut custom_header = opts.header().clone(); + + let mut total_uploaded_size: i64 = 0; + + let mut parts_info = HashMap::::new(); + let mut buf = Vec::::with_capacity(part_size as usize); + + let mut md5_base64: String = "".to_string(); + for part_number in 1..=total_parts_count { + if part_number == total_parts_count { + part_size = lastpart_size; + } + + match &mut reader { + ReaderImpl::Body(content_body) => { + buf = content_body.to_vec(); + } + ReaderImpl::ObjectBody(content_body) => { + buf = content_body.read_all().await?; + } + } + let length = buf.len(); + + if opts.send_content_md5 { + let mut md5_hasher = self.md5_hasher.lock().unwrap(); + let mut md5_hash = md5_hasher.as_mut().expect("err"); + md5_hash.reset(); + md5_hash.write(&buf[..length]); + md5_base64 = base64_encode(md5_hash.sum().as_bytes()); + } else { + let csum; + { + let mut crc = opts.auto_checksum.hasher()?; + crc.reset(); + crc.write(&buf[..length]); + csum = crc.sum(); + } + if let Ok(header_name) = HeaderName::from_bytes(opts.auto_checksum.key_capitalized().as_bytes()) { + custom_header.insert(header_name, HeaderValue::from_str(&base64_encode(csum.as_bytes())).expect("err")); + } else { + warn!("Invalid header name: {}", opts.auto_checksum.key_capitalized()); + } + } + + let hooked = ReaderImpl::Body(Bytes::from(buf)); //newHook(BufferReader::new(buf), opts.progress); + let mut p = UploadPartParams { + bucket_name: bucket_name.to_string(), + object_name: object_name.to_string(), + upload_id: upload_id.clone(), + reader: hooked, + part_number, + md5_base64: md5_base64.clone(), + size: part_size, + //sse: opts.server_side_encryption, + stream_sha256: !opts.disable_content_sha256, + custom_header: custom_header.clone(), + sha256_hex: "".to_string(), + trailer: HeaderMap::new(), + }; + let obj_part = self.upload_part(&mut p).await?; + + parts_info.entry(part_number).or_insert(obj_part); + + total_uploaded_size += part_size as i64; + } + + if size > 0 { + if total_uploaded_size != size { + return Err(std::io::Error::other(err_unexpected_eof( + total_uploaded_size, + size, + bucket_name, + object_name, + ))); + } + } + + let mut compl_multipart_upload = CompleteMultipartUpload::default(); + + let mut all_parts = Vec::::with_capacity(parts_info.len()); + let part_number = total_parts_count; + for i in 1..part_number { + let part = parts_info[&i].clone(); + + all_parts.push(part.clone()); + compl_multipart_upload.parts.push(CompletePart { + etag: part.etag, + part_num: part.part_num, + checksum_crc32: part.checksum_crc32, + checksum_crc32c: part.checksum_crc32c, + checksum_sha1: part.checksum_sha1, + checksum_sha256: part.checksum_sha256, + checksum_crc64nvme: part.checksum_crc64nvme, + }); + } + + compl_multipart_upload.parts.sort(); + + let mut opts = PutObjectOptions { + //server_side_encryption: opts.server_side_encryption, + auto_checksum: opts.auto_checksum, + ..Default::default() + }; + apply_auto_checksum(&mut opts, &mut all_parts); + let mut upload_info = self + .complete_multipart_upload(bucket_name, object_name, &upload_id, compl_multipart_upload, &opts) + .await?; + + upload_info.size = total_uploaded_size; + Ok(upload_info) + } + + pub async fn put_object_multipart_stream_parallel( + self: Arc, + bucket_name: &str, + object_name: &str, + mut reader: ReaderImpl, /*GetObjectReader*/ + opts: &PutObjectOptions, + ) -> Result { + let mut opts = opts.clone(); + if opts.checksum.is_set() { + opts.send_content_md5 = false; + opts.auto_checksum = opts.checksum.clone(); + } + if !opts.send_content_md5 { + add_auto_checksum_headers(&mut opts); + } + + let ret = optimal_part_info(-1, opts.part_size)?; + let (total_parts_count, part_size, _) = ret; + + let upload_id = self.new_upload_id(bucket_name, object_name, &opts).await?; + opts.user_metadata.remove("X-Amz-Checksum-Algorithm"); + + let mut total_uploaded_size: i64 = 0; + let mut parts_info = Arc::new(RwLock::new(HashMap::::new())); + + let n_buffers = opts.num_threads; + let (bufs_tx, mut bufs_rx) = mpsc::channel(n_buffers as usize); + //let all = Vec::::with_capacity(n_buffers as usize * part_size as usize); + for i in 0..n_buffers { + //bufs_tx.send(&all[i * part_size..i * part_size + part_size]); + bufs_tx.send(Vec::::with_capacity(part_size as usize)); + } + + let mut futures = Vec::with_capacity(total_parts_count as usize); + let (err_tx, mut err_rx) = mpsc::channel(opts.num_threads as usize); + let cancel_token = CancellationToken::new(); + + //reader = newHook(reader, opts.progress); + + for part_number in 1..=total_parts_count { + let mut buf = Vec::::new(); + select! { + buf = bufs_rx.recv() => {} + err = err_rx.recv() => { + //cancel_token.cancel(); + //wg.Wait() + return Err(err.expect("err")); + } + else => (), + } + + if buf.len() != part_size as usize { + return Err(std::io::Error::other(format!( + "read buffer < {} than expected partSize: {}", + buf.len(), + part_size + ))); + } + + match &mut reader { + ReaderImpl::Body(content_body) => { + buf = content_body.to_vec(); + } + ReaderImpl::ObjectBody(content_body) => { + buf = content_body.read_all().await?; + } + } + let length = buf.len(); + + let mut custom_header = HeaderMap::new(); + if !opts.send_content_md5 { + let csum; + { + let mut crc = opts.auto_checksum.hasher()?; + crc.reset(); + crc.write(&buf[..length]); + csum = crc.sum(); + } + if let Ok(header_name) = HeaderName::from_bytes(opts.auto_checksum.key().as_bytes()) { + if let Ok(header_value) = HeaderValue::from_str(&base64_encode(csum.as_bytes())) { + custom_header.insert(header_name, header_value); + } + } else { + warn!("Invalid header name: {}", opts.auto_checksum.key()); + } + } + + let clone_bufs_tx = bufs_tx.clone(); + let clone_parts_info = parts_info.clone(); + let clone_upload_id = upload_id.clone(); + let clone_self = self.clone(); + futures.push(async move { + let mut md5_base64: String = "".to_string(); + + if opts.send_content_md5 { + let mut md5_hasher = clone_self.md5_hasher.lock().unwrap(); + let mut md5_hash = md5_hasher.as_mut().expect("err"); + md5_hash.write(&buf[..length]); + md5_base64 = base64_encode(md5_hash.sum().as_bytes()); + } + + //defer wg.Done() + let mut p = UploadPartParams { + bucket_name: bucket_name.to_string(), + object_name: object_name.to_string(), + upload_id: clone_upload_id, + reader: ReaderImpl::Body(Bytes::from(buf.clone())), + part_number, + md5_base64, + size: length as i64, + //sse: opts.server_side_encryption, + stream_sha256: !opts.disable_content_sha256, + custom_header, + sha256_hex: "".to_string(), + trailer: HeaderMap::new(), + }; + let obj_part = clone_self.upload_part(&mut p).await.expect("err"); + + let mut clone_parts_info = clone_parts_info.write().unwrap(); + clone_parts_info.entry(part_number).or_insert(obj_part); + + clone_bufs_tx.send(buf); + }); + + total_uploaded_size += length as i64; + } + + let results = join_all(futures).await; + + select! { + err = err_rx.recv() => { + return Err(err.expect("err")); + } + else => (), + } + + let mut compl_multipart_upload = CompleteMultipartUpload::default(); + + let mut part_number: i64 = total_parts_count; + let mut all_parts = Vec::::with_capacity(parts_info.read().unwrap().len()); + for i in 1..part_number { + let part = parts_info.read().unwrap()[&i].clone(); + + all_parts.push(part.clone()); + compl_multipart_upload.parts.push(CompletePart { + etag: part.etag, + part_num: part.part_num, + checksum_crc32: part.checksum_crc32, + checksum_crc32c: part.checksum_crc32c, + checksum_sha1: part.checksum_sha1, + checksum_sha256: part.checksum_sha256, + checksum_crc64nvme: part.checksum_crc64nvme, + ..Default::default() + }); + } + + compl_multipart_upload.parts.sort(); + + let mut opts = PutObjectOptions { + //server_side_encryption: opts.server_side_encryption, + auto_checksum: opts.auto_checksum, + ..Default::default() + }; + apply_auto_checksum(&mut opts, &mut all_parts); + + let mut upload_info = self + .complete_multipart_upload(bucket_name, object_name, &upload_id, compl_multipart_upload, &opts) + .await?; + + upload_info.size = total_uploaded_size; + Ok(upload_info) + } + + pub async fn put_object_gcs( + &self, + bucket_name: &str, + object_name: &str, + mut reader: ReaderImpl, + size: i64, + opts: &PutObjectOptions, + ) -> Result { + let mut opts = opts.clone(); + if opts.checksum.is_set() { + opts.send_content_md5 = false; + } + + let mut md5_base64: String = "".to_string(); + let progress_reader = reader; //newHook(reader, opts.progress); + + self.put_object_do(bucket_name, object_name, progress_reader, &md5_base64, "", size, &opts) + .await + } + + pub async fn put_object_do( + &self, + bucket_name: &str, + object_name: &str, + reader: ReaderImpl, + md5_base64: &str, + sha256_hex: &str, + size: i64, + opts: &PutObjectOptions, + ) -> Result { + let custom_header = opts.header(); + + let mut req_metadata = RequestMetadata { + bucket_name: bucket_name.to_string(), + object_name: object_name.to_string(), + custom_header, + content_body: reader, + content_length: size, + content_md5_base64: md5_base64.to_string(), + content_sha256_hex: sha256_hex.to_string(), + stream_sha256: !opts.disable_content_sha256, + add_crc: Default::default(), + bucket_location: Default::default(), + pre_sign_url: Default::default(), + query_values: Default::default(), + extra_pre_sign_header: Default::default(), + expires: Default::default(), + trailer: Default::default(), + }; + let mut add_crc = false; //self.trailing_header_support && md5_base64 == "" && !s3utils.IsGoogleEndpoint(self.endpoint_url) && (opts.disable_content_sha256 || self.secure); + let mut opts = opts.clone(); + if opts.checksum.is_set() { + req_metadata.add_crc = opts.checksum; + } else if add_crc { + for (k, _) in opts.user_metadata { + if k.to_lowercase().starts_with("x-amz-checksum-") { + add_crc = false; + } + } + if add_crc { + opts.auto_checksum.set_default(ChecksumMode::ChecksumCRC32C); + req_metadata.add_crc = opts.auto_checksum; + } + } + + if opts.internal.source_version_id != "" { + if !opts.internal.source_version_id.is_empty() { + if let Err(err) = Uuid::parse_str(&opts.internal.source_version_id) { + return Err(std::io::Error::other(err_invalid_argument(&err.to_string()))); + } + } + let mut url_values = HashMap::new(); + url_values.insert("versionId".to_string(), opts.internal.source_version_id); + req_metadata.query_values = url_values; + } + + let resp = self.execute_method(http::Method::PUT, &mut req_metadata).await?; + + if resp.status() != StatusCode::OK { + return Err(std::io::Error::other(http_resp_to_error_response(resp, vec![], bucket_name, object_name))); + } + + let (exp_time, rule_id) = if let Some(h_x_amz_expiration) = resp.headers().get(X_AMZ_EXPIRATION) { + ( + OffsetDateTime::parse(h_x_amz_expiration.to_str().unwrap(), ISO8601_DATEFORMAT).unwrap(), + "".to_string(), + ) + } else { + (OffsetDateTime::now_utc(), "".to_string()) + }; + let h = resp.headers(); + Ok(UploadInfo { + bucket: bucket_name.to_string(), + key: object_name.to_string(), + etag: trim_etag(h.get("ETag").expect("err").to_str().expect("err")), + version_id: if let Some(h_x_amz_version_id) = h.get(X_AMZ_VERSION_ID) { + h_x_amz_version_id.to_str().expect("err").to_string() + } else { + "".to_string() + }, + size: size, + expiration: exp_time, + expiration_rule_id: rule_id, + checksum_crc32: if let Some(h_checksum_crc32) = h.get(ChecksumMode::ChecksumCRC32.key()) { + h_checksum_crc32.to_str().expect("err").to_string() + } else { + "".to_string() + }, + checksum_crc32c: if let Some(h_checksum_crc32c) = h.get(ChecksumMode::ChecksumCRC32C.key()) { + h_checksum_crc32c.to_str().expect("err").to_string() + } else { + "".to_string() + }, + checksum_sha1: if let Some(h_checksum_sha1) = h.get(ChecksumMode::ChecksumSHA1.key()) { + h_checksum_sha1.to_str().expect("err").to_string() + } else { + "".to_string() + }, + checksum_sha256: if let Some(h_checksum_sha256) = h.get(ChecksumMode::ChecksumSHA256.key()) { + h_checksum_sha256.to_str().expect("err").to_string() + } else { + "".to_string() + }, + checksum_crc64nvme: if let Some(h_checksum_crc64nvme) = h.get(ChecksumMode::ChecksumCRC64NVME.key()) { + h_checksum_crc64nvme.to_str().expect("err").to_string() + } else { + "".to_string() + }, + ..Default::default() + }) + } +} diff --git a/ecstore/src/client/api_remove.rs b/ecstore/src/client/api_remove.rs new file mode 100644 index 000000000..9ca5bc51f --- /dev/null +++ b/ecstore/src/client/api_remove.rs @@ -0,0 +1,472 @@ +#![allow(clippy::map_entry)] +use bytes::Bytes; +use http::{HeaderMap, HeaderValue, Method, StatusCode}; +use s3s::S3ErrorCode; +use s3s::dto::ReplicationStatus; +use s3s::header::X_AMZ_BYPASS_GOVERNANCE_RETENTION; +use std::fmt::Display; +use std::{collections::HashMap, sync::Arc}; +use time::OffsetDateTime; +use tokio::sync::mpsc::{self, Receiver, Sender}; + +use crate::client::{ + api_error_response::{ErrorResponse, http_resp_to_error_response, to_error_response}, + transition_api::{ReaderImpl, RequestMetadata, TransitionClient}, +}; +use crate::{ + disk::DiskAPI, + store_api::{GetObjectReader, ObjectInfo, StorageAPI}, +}; +use reader::hasher::{sum_md5_base64, sum_sha256_hex}; +use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH; + +struct RemoveBucketOptions { + forced_elete: bool, +} + +#[derive(Debug)] +pub struct AdvancedRemoveOptions { + replication_delete_marker: bool, + replication_status: ReplicationStatus, + replication_mtime: OffsetDateTime, + replication_request: bool, + replication_validity_check: bool, +} + +impl Default for AdvancedRemoveOptions { + fn default() -> Self { + Self { + replication_delete_marker: false, + replication_status: ReplicationStatus::from_static(ReplicationStatus::PENDING), + replication_mtime: OffsetDateTime::now_utc(), + replication_request: false, + replication_validity_check: false, + } + } +} + +#[derive(Debug, Default)] +pub struct RemoveObjectOptions { + pub force_delete: bool, + pub governance_bypass: bool, + pub version_id: String, + pub internal: AdvancedRemoveOptions, +} + +impl TransitionClient { + pub async fn remove_bucket_with_options(&self, bucket_name: &str, opts: &RemoveBucketOptions) -> Result<(), std::io::Error> { + let mut headers = HeaderMap::new(); + /*if opts.force_delete { + headers.insert(rustFSForceDelete, "true"); + }*/ + + let resp = self + .execute_method( + Method::DELETE, + &mut RequestMetadata { + bucket_name: bucket_name.to_string(), + content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(), + custom_header: headers, + object_name: "".to_string(), + query_values: Default::default(), + content_body: ReaderImpl::Body(Bytes::new()), + content_length: 0, + content_md5_base64: "".to_string(), + stream_sha256: false, + trailer: HeaderMap::new(), + pre_sign_url: Default::default(), + add_crc: Default::default(), + extra_pre_sign_header: Default::default(), + bucket_location: Default::default(), + expires: Default::default(), + }, + ) + .await?; + + { + let mut bucket_loc_cache = self.bucket_loc_cache.lock().unwrap(); + bucket_loc_cache.delete(bucket_name); + } + Ok(()) + } + + pub async fn remove_bucket(&self, bucket_name: &str) -> Result<(), std::io::Error> { + let resp = self + .execute_method( + http::Method::DELETE, + &mut RequestMetadata { + bucket_name: bucket_name.to_string(), + content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(), + custom_header: Default::default(), + object_name: "".to_string(), + query_values: Default::default(), + content_body: ReaderImpl::Body(Bytes::new()), + content_length: 0, + content_md5_base64: "".to_string(), + stream_sha256: false, + trailer: HeaderMap::new(), + pre_sign_url: Default::default(), + add_crc: Default::default(), + extra_pre_sign_header: Default::default(), + bucket_location: Default::default(), + expires: Default::default(), + }, + ) + .await?; + + { + let mut bucket_loc_cache = self.bucket_loc_cache.lock().unwrap(); + bucket_loc_cache.delete(bucket_name); + } + + Ok(()) + } + + pub async fn remove_object(&self, bucket_name: &str, object_name: &str, opts: RemoveObjectOptions) -> Option { + let res = self.remove_object_inner(bucket_name, object_name, opts).await.expect("err"); + res.err + } + + pub async fn remove_object_inner( + &self, + bucket_name: &str, + object_name: &str, + opts: RemoveObjectOptions, + ) -> Result { + let mut url_values = HashMap::new(); + + if opts.version_id != "" { + url_values.insert("versionId".to_string(), opts.version_id.clone()); + } + + let mut headers = HeaderMap::new(); + + if opts.governance_bypass { + headers.insert(X_AMZ_BYPASS_GOVERNANCE_RETENTION, "true".parse().expect("err")); //amzBypassGovernance + } + + let resp = self + .execute_method( + http::Method::DELETE, + &mut RequestMetadata { + bucket_name: bucket_name.to_string(), + object_name: object_name.to_string(), + content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(), + query_values: url_values, + custom_header: headers, + content_body: ReaderImpl::Body(Bytes::new()), + content_length: 0, + content_md5_base64: "".to_string(), + stream_sha256: false, + trailer: HeaderMap::new(), + pre_sign_url: Default::default(), + add_crc: Default::default(), + extra_pre_sign_header: Default::default(), + bucket_location: Default::default(), + expires: Default::default(), + }, + ) + .await?; + + Ok(RemoveObjectResult { + object_name: object_name.to_string(), + object_version_id: opts.version_id, + delete_marker: resp.headers().get("x-amz-delete-marker").expect("err") == "true", + delete_marker_version_id: resp + .headers() + .get("x-amz-version-id") + .expect("err") + .to_str() + .expect("err") + .to_string(), + ..Default::default() + }) + } + + pub async fn remove_objects_with_result( + self: Arc, + bucket_name: &str, + objects_rx: Receiver, + opts: RemoveObjectsOptions, + ) -> Receiver { + let (result_tx, mut result_rx) = mpsc::channel(1); + + let self_clone = Arc::clone(&self); + let bucket_name_owned = bucket_name.to_string(); + + tokio::spawn(async move { + self_clone + .remove_objects_inner(&bucket_name_owned, objects_rx, &result_tx, opts) + .await; + }); + result_rx + } + + pub async fn remove_objects( + self: Arc, + bucket_name: &str, + objects_rx: Receiver, + opts: RemoveObjectsOptions, + ) -> Receiver { + let (error_tx, mut error_rx) = mpsc::channel(1); + + let self_clone = Arc::clone(&self); + let bucket_name_owned = bucket_name.to_string(); + + let (result_tx, mut result_rx) = mpsc::channel(1); + tokio::spawn(async move { + self_clone + .remove_objects_inner(&bucket_name_owned, objects_rx, &result_tx, opts) + .await; + }); + tokio::spawn(async move { + while let Some(res) = result_rx.recv().await { + if res.err.is_none() { + continue; + } + error_tx + .send(RemoveObjectError { + object_name: res.object_name, + version_id: res.object_version_id, + err: res.err, + ..Default::default() + }) + .await; + } + }); + + error_rx + } + + pub async fn remove_objects_inner( + &self, + bucket_name: &str, + mut objects_rx: Receiver, + result_tx: &Sender, + opts: RemoveObjectsOptions, + ) -> Result<(), std::io::Error> { + let max_entries = 1000; + let mut finish = false; + let mut url_values = HashMap::new(); + url_values.insert("delete".to_string(), "".to_string()); + + loop { + if finish { + break; + } + let mut count = 0; + let mut batch = Vec::::new(); + + while let Some(object) = objects_rx.recv().await { + if has_invalid_xml_char(&object.name) { + let remove_result = self + .remove_object_inner( + bucket_name, + &object.name, + RemoveObjectOptions { + version_id: object.version_id.expect("err").to_string(), + governance_bypass: opts.governance_bypass, + ..Default::default() + }, + ) + .await?; + let remove_result_clone = remove_result.clone(); + if !remove_result.err.is_none() { + match to_error_response(&remove_result.err.expect("err")).code { + S3ErrorCode::InvalidArgument | S3ErrorCode::NoSuchVersion => { + continue; + } + _ => (), + } + result_tx.send(remove_result_clone.clone()).await; + } + + result_tx.send(remove_result_clone).await; + continue; + } + + batch.push(object); + count += 1; + if count >= max_entries { + break; + } + } + if count == 0 { + break; + } + if count < max_entries { + finish = true; + } + + let mut headers = HeaderMap::new(); + if opts.governance_bypass { + headers.insert(X_AMZ_BYPASS_GOVERNANCE_RETENTION, "true".parse().expect("err")); + } + + let remove_bytes = generate_remove_multi_objects_request(&batch); + let resp = self + .execute_method( + http::Method::POST, + &mut RequestMetadata { + bucket_name: bucket_name.to_string(), + query_values: url_values.clone(), + content_body: ReaderImpl::Body(Bytes::from(remove_bytes.clone())), + content_length: remove_bytes.len() as i64, + content_md5_base64: sum_md5_base64(&remove_bytes), + content_sha256_hex: sum_sha256_hex(&remove_bytes), + custom_header: headers, + object_name: "".to_string(), + stream_sha256: false, + trailer: HeaderMap::new(), + pre_sign_url: Default::default(), + add_crc: Default::default(), + extra_pre_sign_header: Default::default(), + bucket_location: Default::default(), + expires: Default::default(), + }, + ) + .await?; + + let body_bytes: Vec = resp.body().bytes().expect("err").to_vec(); + process_remove_multi_objects_response(ReaderImpl::Body(Bytes::from(body_bytes)), result_tx.clone()); + } + Ok(()) + } + + pub async fn remove_incomplete_upload(&self, bucket_name: &str, object_name: &str) -> Result<(), std::io::Error> { + let upload_ids = self.find_upload_ids(bucket_name, object_name)?; + for upload_id in upload_ids { + self.abort_multipart_upload(bucket_name, object_name, &upload_id).await?; + } + + Ok(()) + } + + pub async fn abort_multipart_upload( + &self, + bucket_name: &str, + object_name: &str, + upload_id: &str, + ) -> Result<(), std::io::Error> { + let mut url_values = HashMap::new(); + url_values.insert("uploadId".to_string(), upload_id.to_string()); + + let resp = self + .execute_method( + http::Method::DELETE, + &mut RequestMetadata { + bucket_name: bucket_name.to_string(), + object_name: object_name.to_string(), + query_values: url_values, + content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(), + custom_header: HeaderMap::new(), + content_body: ReaderImpl::Body(Bytes::new()), + content_length: 0, + content_md5_base64: "".to_string(), + stream_sha256: false, + trailer: HeaderMap::new(), + pre_sign_url: Default::default(), + add_crc: Default::default(), + extra_pre_sign_header: Default::default(), + bucket_location: Default::default(), + expires: Default::default(), + }, + ) + .await?; + //if resp.is_some() { + if resp.status() != StatusCode::NO_CONTENT { + let error_response: ErrorResponse; + match resp.status() { + StatusCode::NOT_FOUND => { + error_response = ErrorResponse { + code: S3ErrorCode::NoSuchUpload, + message: "The specified multipart upload does not exist.".to_string(), + bucket_name: bucket_name.to_string(), + key: object_name.to_string(), + request_id: resp + .headers() + .get("x-amz-request-id") + .expect("err") + .to_str() + .expect("err") + .to_string(), + host_id: resp + .headers() + .get("x-amz-id-2") + .expect("err") + .to_str() + .expect("err") + .to_string(), + region: resp + .headers() + .get("x-amz-bucket-region") + .expect("err") + .to_str() + .expect("err") + .to_string(), + ..Default::default() + }; + } + _ => { + return Err(std::io::Error::other(http_resp_to_error_response(resp, vec![], bucket_name, object_name))); + } + } + return Err(std::io::Error::other(error_response)); + } + //} + Ok(()) + } +} + +#[derive(Debug, Default)] +struct RemoveObjectError { + object_name: String, + version_id: String, + err: Option, +} + +impl Display for RemoveObjectError { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + if self.err.is_none() { + return write!(f, "unexpected remove object error result"); + } + write!(f, "{}", self.err.as_ref().expect("err").to_string()) + } +} + +#[derive(Debug, Default)] +pub struct RemoveObjectResult { + pub object_name: String, + pub object_version_id: String, + pub delete_marker: bool, + pub delete_marker_version_id: String, + pub err: Option, +} + +impl Clone for RemoveObjectResult { + fn clone(&self) -> Self { + Self { + object_name: self.object_name.clone(), + object_version_id: self.object_version_id.clone(), + delete_marker: self.delete_marker, + delete_marker_version_id: self.delete_marker_version_id.clone(), + err: None, //err + } + } +} + +pub struct RemoveObjectsOptions { + pub governance_bypass: bool, +} + +pub fn generate_remove_multi_objects_request(objects: &[ObjectInfo]) -> Vec { + todo!(); +} + +pub fn process_remove_multi_objects_response(body: ReaderImpl, result_tx: Sender) { + todo!(); +} + +fn has_invalid_xml_char(str: &str) -> bool { + false +} diff --git a/ecstore/src/client/api_s3_datatypes.rs b/ecstore/src/client/api_s3_datatypes.rs new file mode 100644 index 000000000..7276f8635 --- /dev/null +++ b/ecstore/src/client/api_s3_datatypes.rs @@ -0,0 +1,336 @@ +#![allow(clippy::map_entry)] +use s3s::dto::Owner; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use time::OffsetDateTime; + +use crate::checksum::ChecksumMode; +use crate::client::transition_api::ObjectMultipartInfo; +use rustfs_utils::crypto::base64_decode; + +use super::transition_api; + +pub struct ListAllMyBucketsResult { + pub owner: Owner, +} + +#[derive(Debug, Default, Serialize, Deserialize)] +pub struct CommonPrefix { + pub prefix: String, +} + +#[derive(Debug, Default, Serialize, Deserialize)] +#[serde(default, rename_all = "PascalCase")] +pub struct ListBucketV2Result { + pub common_prefixes: Vec, + pub contents: Vec, + pub delimiter: String, + pub encoding_type: String, + pub is_truncated: bool, + pub max_keys: i64, + pub name: String, + pub next_continuation_token: String, + pub continuation_token: String, + pub prefix: String, + pub fetch_owner: String, + pub start_after: String, +} + +pub struct Version { + etag: String, + is_latest: bool, + key: String, + last_modified: OffsetDateTime, + owner: Owner, + size: i64, + storage_class: String, + version_id: String, + user_metadata: HashMap, + user_tags: HashMap, + is_delete_marker: bool, +} + +pub struct ListVersionsResult { + versions: Vec, + common_prefixes: Vec, + name: String, + prefix: String, + delimiter: String, + max_keys: i64, + encoding_type: String, + is_truncated: bool, + key_marker: String, + version_id_marker: String, + next_key_marker: String, + next_version_id_marker: String, +} + +impl ListVersionsResult { + fn unmarshal_xml() -> Result<(), std::io::Error> { + todo!(); + } +} + +pub struct ListBucketResult { + common_prefixes: Vec, + contents: Vec, + delimiter: String, + encoding_type: String, + is_truncated: bool, + marker: String, + max_keys: i64, + name: String, + next_marker: String, + prefix: String, +} + +pub struct ListMultipartUploadsResult { + bucket: String, + key_marker: String, + upload_id_marker: String, + next_key_marker: String, + next_upload_id_marker: String, + encoding_type: String, + max_uploads: i64, + is_truncated: bool, + uploads: Vec, + prefix: String, + delimiter: String, + common_prefixes: Vec, +} + +pub struct Initiator { + id: String, + display_name: String, +} + +pub struct CopyObjectResult { + pub etag: String, + pub last_modified: OffsetDateTime, +} + +#[derive(Debug, Clone)] +pub struct ObjectPart { + pub etag: String, + pub part_num: i64, + pub last_modified: OffsetDateTime, + pub size: i64, + pub checksum_crc32: String, + pub checksum_crc32c: String, + pub checksum_sha1: String, + pub checksum_sha256: String, + pub checksum_crc64nvme: String, +} + +impl Default for ObjectPart { + fn default() -> Self { + ObjectPart { + etag: Default::default(), + part_num: 0, + last_modified: OffsetDateTime::now_utc(), + size: 0, + checksum_crc32: Default::default(), + checksum_crc32c: Default::default(), + checksum_sha1: Default::default(), + checksum_sha256: Default::default(), + checksum_crc64nvme: Default::default(), + } + } +} + +impl ObjectPart { + fn checksum(&self, t: &ChecksumMode) -> String { + match t { + ChecksumMode::ChecksumCRC32C => { + return self.checksum_crc32c.clone(); + } + ChecksumMode::ChecksumCRC32 => { + return self.checksum_crc32.clone(); + } + ChecksumMode::ChecksumSHA1 => { + return self.checksum_sha1.clone(); + } + ChecksumMode::ChecksumSHA256 => { + return self.checksum_sha256.clone(); + } + ChecksumMode::ChecksumCRC64NVME => { + return self.checksum_crc64nvme.clone(); + } + _ => { + return "".to_string(); + } + } + } + + pub fn checksum_raw(&self, t: &ChecksumMode) -> Result, std::io::Error> { + let b = self.checksum(t); + if b == "" { + return Err(std::io::Error::other("no checksum set")); + } + let decoded = match base64_decode(b.as_bytes()) { + Ok(b) => b, + Err(e) => return Err(std::io::Error::other(e)), + }; + if decoded.len() != t.raw_byte_len() as usize { + return Err(std::io::Error::other("checksum length mismatch")); + } + Ok(decoded) + } +} + +pub struct ListObjectPartsResult { + pub bucket: String, + pub key: String, + pub upload_id: String, + pub initiator: Initiator, + pub owner: Owner, + pub storage_class: String, + pub part_number_marker: i32, + pub next_part_number_marker: i32, + pub max_parts: i32, + pub checksum_algorithm: String, + pub checksum_type: String, + pub is_truncated: bool, + pub object_parts: Vec, + pub encoding_type: String, +} + +#[derive(Debug, Default)] +pub struct InitiateMultipartUploadResult { + pub bucket: String, + pub key: String, + pub upload_id: String, +} + +#[derive(Debug, Default)] +pub struct CompleteMultipartUploadResult { + pub location: String, + pub bucket: String, + pub key: String, + pub etag: String, + pub checksum_crc32: String, + pub checksum_crc32c: String, + pub checksum_sha1: String, + pub checksum_sha256: String, + pub checksum_crc64nvme: String, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, PartialOrd, Ord, serde::Serialize)] +pub struct CompletePart { + //api has + pub etag: String, + pub part_num: i64, + pub checksum_crc32: String, + pub checksum_crc32c: String, + pub checksum_sha1: String, + pub checksum_sha256: String, + pub checksum_crc64nvme: String, +} + +impl CompletePart { + fn checksum(&self, t: &ChecksumMode) -> String { + match t { + ChecksumMode::ChecksumCRC32C => { + return self.checksum_crc32c.clone(); + } + ChecksumMode::ChecksumCRC32 => { + return self.checksum_crc32.clone(); + } + ChecksumMode::ChecksumSHA1 => { + return self.checksum_sha1.clone(); + } + ChecksumMode::ChecksumSHA256 => { + return self.checksum_sha256.clone(); + } + ChecksumMode::ChecksumCRC64NVME => { + return self.checksum_crc64nvme.clone(); + } + _ => { + return "".to_string(); + } + } + } +} + +pub struct CopyObjectPartResult { + pub etag: String, + pub last_modified: OffsetDateTime, +} + +#[derive(Debug, Default, serde::Serialize)] +pub struct CompleteMultipartUpload { + pub parts: Vec, +} + +impl CompleteMultipartUpload { + pub fn marshal_msg(&self) -> Result { + //let buf = serde_json::to_string(self)?; + let buf = match serde_xml_rs::to_string(self) { + Ok(buf) => buf, + Err(e) => { + return Err(std::io::Error::other(e)); + } + }; + + Ok(buf) + } + + pub fn unmarshal(buf: &[u8]) -> Result { + todo!(); + } +} + +pub struct CreateBucketConfiguration { + pub location: String, +} + +#[derive(serde::Serialize)] +pub struct DeleteObject { + //api has + pub key: String, + pub version_id: String, +} + +pub struct DeletedObject { + //s3s has + pub key: String, + pub version_id: String, + pub deletemarker: bool, + pub deletemarker_version_id: String, +} + +pub struct NonDeletedObject { + pub key: String, + pub code: String, + pub message: String, + pub version_id: String, +} + +#[derive(serde::Serialize)] +pub struct DeleteMultiObjects { + pub quiet: bool, + pub objects: Vec, +} + +impl DeleteMultiObjects { + pub fn marshal_msg(&self) -> Result { + //let buf = serde_json::to_string(self)?; + let buf = match serde_xml_rs::to_string(self) { + Ok(buf) => buf, + Err(e) => { + return Err(std::io::Error::other(e)); + } + }; + + Ok(buf) + } + + pub fn unmarshal(buf: &[u8]) -> Result { + todo!(); + } +} + +pub struct DeleteMultiObjectsResult { + pub deleted_objects: Vec, + pub undeleted_objects: Vec, +} diff --git a/ecstore/src/client/bucket_cache.rs b/ecstore/src/client/bucket_cache.rs new file mode 100644 index 000000000..6216cd5e0 --- /dev/null +++ b/ecstore/src/client/bucket_cache.rs @@ -0,0 +1,221 @@ +#![allow(clippy::map_entry)] +use http::Request; +use hyper::StatusCode; +use hyper::body::Incoming; +use std::{collections::HashMap, sync::Arc}; +use tracing::warn; +use tracing::{debug, error, info}; + +use crate::client::{ + api_error_response::{http_resp_to_error_response, to_error_response}, + transition_api::{Document, TransitionClient}, +}; +use crate::signer; +use reader::hasher::{Hasher, Sha256}; +use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH; +use s3s::Body; +use s3s::S3ErrorCode; + +use super::constants::UNSIGNED_PAYLOAD; +use super::credentials::SignatureType; + +pub struct BucketLocationCache { + items: HashMap, +} + +impl BucketLocationCache { + pub fn new() -> BucketLocationCache { + BucketLocationCache { items: HashMap::new() } + } + + pub fn get(&self, bucket_name: &str) -> Option { + self.items.get(bucket_name).map(|s| s.clone()) + } + + pub fn set(&mut self, bucket_name: &str, location: &str) { + self.items.insert(bucket_name.to_string(), location.to_string()); + } + + pub fn delete(&mut self, bucket_name: &str) { + self.items.remove(bucket_name); + } +} + +impl TransitionClient { + pub async fn get_bucket_location(&self, bucket_name: &str) -> Result { + Ok(self.get_bucket_location_inner(bucket_name).await?) + } + + async fn get_bucket_location_inner(&self, bucket_name: &str) -> Result { + if self.region != "" { + return Ok(self.region.clone()); + } + + let mut location; + { + let mut bucket_loc_cache = self.bucket_loc_cache.lock().unwrap(); + let ret = bucket_loc_cache.get(bucket_name); + if let Some(location) = ret { + return Ok(location); + } + //location = ret?; + } + + let req = self.get_bucket_location_request(bucket_name)?; + + let mut resp = self.doit(req).await?; + location = process_bucket_location_response(resp, bucket_name).await?; + { + let mut bucket_loc_cache = self.bucket_loc_cache.lock().unwrap(); + bucket_loc_cache.set(bucket_name, &location); + } + Ok(location) + } + + fn get_bucket_location_request(&self, bucket_name: &str) -> Result, std::io::Error> { + let mut url_values = HashMap::new(); + url_values.insert("location".to_string(), "".to_string()); + + let mut target_url = self.endpoint_url.clone(); + let scheme = self.endpoint_url.scheme(); + let h = target_url.host().expect("host is none."); + let default_port = if scheme == "https" { 443 } else { 80 }; + let p = target_url.port().unwrap_or(default_port); + + let is_virtual_style = self.is_virtual_host_style_request(&target_url, bucket_name); + + let mut url_str: String = "".to_string(); + + if is_virtual_style { + url_str = scheme.to_string(); + url_str.push_str("://"); + url_str.push_str(bucket_name); + url_str.push_str("."); + url_str.push_str(target_url.host_str().expect("err")); + url_str.push_str("/?location"); + } else { + let mut path = bucket_name.to_string(); + path.push_str("/"); + target_url.set_path(&path); + { + let mut q = target_url.query_pairs_mut(); + for (k, v) in url_values { + q.append_pair(&k, &urlencoding::encode(&v)); + } + } + url_str = target_url.to_string(); + } + + let mut req_builder = Request::builder().method(http::Method::GET).uri(url_str); + + self.set_user_agent(&mut req_builder); + + let value; + { + let mut creds_provider = self.creds_provider.lock().unwrap(); + value = match creds_provider.get_with_context(Some(self.cred_context())) { + Ok(v) => v, + Err(err) => { + return Err(std::io::Error::other(err)); + } + }; + } + + let mut signer_type = value.signer_type.clone(); + let mut access_key_id = value.access_key_id; + let mut secret_access_key = value.secret_access_key; + let mut session_token = value.session_token; + + if self.override_signer_type != SignatureType::SignatureDefault { + signer_type = self.override_signer_type.clone(); + } + + if value.signer_type == SignatureType::SignatureAnonymous { + signer_type = SignatureType::SignatureAnonymous + } + + if signer_type == SignatureType::SignatureAnonymous { + let req = match req_builder.body(Body::empty()) { + Ok(req) => return Ok(req), + Err(err) => { + return Err(std::io::Error::other(err)); + } + }; + } + + if signer_type == SignatureType::SignatureV2 { + let req_builder = signer::sign_v2(req_builder, 0, &access_key_id, &secret_access_key, is_virtual_style); + let req = match req_builder.body(Body::empty()) { + Ok(req) => return Ok(req), + Err(err) => { + return Err(std::io::Error::other(err)); + } + }; + } + + let mut content_sha256 = EMPTY_STRING_SHA256_HASH.to_string(); + if self.secure { + content_sha256 = UNSIGNED_PAYLOAD.to_string(); + } + + req_builder + .headers_mut() + .expect("err") + .insert("X-Amz-Content-Sha256", content_sha256.parse().unwrap()); + let req_builder = signer::sign_v4(req_builder, 0, &access_key_id, &secret_access_key, &session_token, "us-east-1"); + let req = match req_builder.body(Body::empty()) { + Ok(req) => return Ok(req), + Err(err) => { + return Err(std::io::Error::other(err)); + } + }; + } +} + +async fn process_bucket_location_response(mut resp: http::Response, bucket_name: &str) -> Result { + //if resp != nil { + if resp.status() != StatusCode::OK { + let err_resp = http_resp_to_error_response(resp, vec![], bucket_name, ""); + match err_resp.code { + S3ErrorCode::NotImplemented => { + match err_resp.server.as_str() { + "AmazonSnowball" => { + return Ok("snowball".to_string()); + } + "cloudflare" => { + return Ok("us-east-1".to_string()); + } + _ => { + return Err(std::io::Error::other(err_resp)); + } + } + } + S3ErrorCode::AuthorizationHeaderMalformed | + //S3ErrorCode::InvalidRegion | + S3ErrorCode::AccessDenied => { + if err_resp.region == "" { + return Ok("us-east-1".to_string()); + } + return Ok(err_resp.region); + } + _ => { + return Err(std::io::Error::other(err_resp)); + } + } + } + //} + + let b = resp.body_mut().store_all_unlimited().await.unwrap().to_vec(); + let Document(location_constraint) = serde_xml_rs::from_str::(&String::from_utf8(b).unwrap()).unwrap(); + + let mut location = location_constraint; + if location == "" { + location = "us-east-1".to_string(); + } + + if location == "EU" { + location = "eu-west-1".to_string(); + } + + Ok(location) +} diff --git a/ecstore/src/client/constants.rs b/ecstore/src/client/constants.rs new file mode 100644 index 000000000..42ee4c15b --- /dev/null +++ b/ecstore/src/client/constants.rs @@ -0,0 +1,25 @@ +#![allow(clippy::map_entry)] +use lazy_static::lazy_static; +use std::{collections::HashMap, sync::Arc}; +use time::{format_description::FormatItem, macros::format_description}; + +pub const ABS_MIN_PART_SIZE: i64 = 1024 * 1024 * 5; +pub const MAX_PARTS_COUNT: i64 = 10000; +pub const MAX_PART_SIZE: i64 = 1024 * 1024 * 1024 * 5; +pub const MIN_PART_SIZE: i64 = 1024 * 1024 * 16; + +pub const MAX_SINGLE_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 5; +pub const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5; + +pub const UNSIGNED_PAYLOAD: &str = "UNSIGNED-PAYLOAD"; +pub const UNSIGNED_PAYLOAD_TRAILER: &str = "STREAMING-UNSIGNED-PAYLOAD-TRAILER"; + +pub const TOTAL_WORKERS: i64 = 4; + +pub const SIGN_V4_ALGORITHM: &str = "AWS4-HMAC-SHA256"; +pub const ISO8601_DATEFORMAT: &[FormatItem<'_>] = + format_description!("[year]-[month]-[day]T[hour]:[minute]:[second].[subsecond]Z"); + +const GetObjectAttributesTags: &str = "ETag,Checksum,StorageClass,ObjectSize,ObjectParts"; +const GetObjectAttributesMaxParts: i64 = 1000; +const RUSTFS_BUCKET_SOURCE_MTIME: &str = "X-RustFs-Source-Mtime"; diff --git a/ecstore/src/client/credentials.rs b/ecstore/src/client/credentials.rs new file mode 100644 index 000000000..f4a13513e --- /dev/null +++ b/ecstore/src/client/credentials.rs @@ -0,0 +1,165 @@ +use std::fmt::{Display, Formatter}; + +use time::OffsetDateTime; + +#[derive(Debug, Default, Clone, Eq, PartialEq)] +pub enum SignatureType { + #[default] + SignatureDefault, + SignatureV4, + SignatureV2, + SignatureV4Streaming, + SignatureAnonymous, +} + +#[derive(Debug, Clone, Default)] +pub struct Credentials { + creds: Value, + force_refresh: bool, + provider: P, +} + +impl Credentials

{ + pub fn new(provider: P) -> Self { + Self { + provider: provider, + force_refresh: true, + ..Default::default() + } + } + + pub fn get(&mut self) -> Result { + self.get_with_context(None) + } + + pub fn get_with_context(&mut self, mut cc: Option) -> Result { + if self.is_expired() { + let creds = self.provider.retrieve_with_cred_context(cc.expect("err")); + self.creds = creds; + self.force_refresh = false; + } + + Ok(self.creds.clone()) + } + + fn expire(&mut self) { + self.force_refresh = true; + } + + pub fn is_expired(&self) -> bool { + self.force_refresh || self.provider.is_expired() + } +} + +#[derive(Debug, Clone)] +pub struct Value { + pub access_key_id: String, + pub secret_access_key: String, + pub session_token: String, + pub expiration: OffsetDateTime, + pub signer_type: SignatureType, +} + +impl Default for Value { + fn default() -> Self { + Self { + access_key_id: "".to_string(), + secret_access_key: "".to_string(), + session_token: "".to_string(), + expiration: OffsetDateTime::now_utc(), + signer_type: SignatureType::SignatureDefault, + } + } +} + +pub struct CredContext { + //pub client: SendRequest, + pub endpoint: String, +} + +trait Provider { + fn retrieve(&self) -> Value; + fn retrieve_with_cred_context(&self, _: CredContext) -> Value; + fn is_expired(&self) -> bool; +} + +#[derive(Debug, Clone, Default)] +pub struct Static(pub Value); + +impl Provider for Static { + fn retrieve(&self) -> Value { + if self.0.access_key_id == "" || self.0.secret_access_key == "" { + return Value { + signer_type: SignatureType::SignatureAnonymous, + ..Default::default() + }; + } + self.0.clone() + } + + fn retrieve_with_cred_context(&self, _: CredContext) -> Value { + self.retrieve() + } + + fn is_expired(&self) -> bool { + false + } +} + +#[derive(Debug, Clone, Default)] +pub struct STSError { + pub r#type: String, + pub code: String, + pub message: String, +} + +#[derive(Debug, Clone, thiserror::Error)] +pub struct ErrorResponse { + pub sts_error: STSError, + pub request_id: String, +} + +impl Display for ErrorResponse { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.error()) + } +} + +impl ErrorResponse { + fn error(&self) -> String { + if self.sts_error.message == "" { + return format!("Error response code {}.", self.sts_error.code); + } + return self.sts_error.message.clone(); + } +} + +struct Error { + code: String, + message: String, + bucket_name: String, + key: String, + resource: String, + request_id: String, + host_id: String, + region: String, + server: String, + status_code: i64, +} + +impl Display for Error { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + if self.message == "" { + return write!(f, "{}", format!("Error response code {}.", self.code)); + } + write!(f, "{}", self.message) + } +} + +pub fn xml_decoder(body: &[u8]) -> Result { + todo!(); +} + +pub fn xml_decode_and_body(body_reader: &[u8]) -> Result<(Vec, T), std::io::Error> { + todo!(); +} diff --git a/ecstore/src/client/hook_reader.rs b/ecstore/src/client/hook_reader.rs new file mode 100644 index 000000000..c8351e4dd --- /dev/null +++ b/ecstore/src/client/hook_reader.rs @@ -0,0 +1,45 @@ +#![allow(clippy::map_entry)] +use std::{collections::HashMap, sync::Arc}; + +use crate::{ + disk::{ + error::{is_unformatted_disk, DiskError}, + format::{DistributionAlgoVersion, FormatV3}, + new_disk, DiskAPI, DiskInfo, DiskOption, DiskStore, + }, + store_api::{ + BucketInfo, BucketOptions, CompletePart, DeleteBucketOptions, DeletedObject, GetObjectReader, HTTPRangeSpec, + ListMultipartsInfo, ListObjectVersionsInfo, ListObjectsV2Info, MakeBucketOptions, MultipartInfo, MultipartUploadResult, + ObjectIO, ObjectInfo, ObjectOptions, ObjectToDelete, PartInfo, PutObjReader, StorageAPI, + }, + credentials::{Credentials, SignatureType,}, + api_put_object_multipart::UploadPartParams, +}; + +use http::HeaderMap; +use tokio_util::sync::CancellationToken; +use tracing::warn; +use tracing::{error, info}; +use url::Url; + +struct HookReader { + source: GetObjectReader, + hook: GetObjectReader, +} + +impl HookReader { + pub fn new(source: GetObjectReader, hook: GetObjectReader) -> HookReader { + HookReader { + source, + hook, + } + } + + fn seek(&self, offset: i64, whence: i64) -> Result { + todo!(); + } + + fn read(&self, b: &[u8]) -> Result { + todo!(); + } +} \ No newline at end of file diff --git a/ecstore/src/client/mod.rs b/ecstore/src/client/mod.rs new file mode 100644 index 000000000..e8a45e8e5 --- /dev/null +++ b/ecstore/src/client/mod.rs @@ -0,0 +1,18 @@ +pub mod admin_handler_utils; +pub mod api_bucket_policy; +pub mod api_error_response; +pub mod api_get_object; +pub mod api_get_options; +pub mod api_list; +pub mod api_put_object; +pub mod api_put_object_common; +pub mod api_put_object_multipart; +pub mod api_put_object_streaming; +pub mod api_remove; +pub mod api_s3_datatypes; +pub mod bucket_cache; +pub mod constants; +pub mod credentials; +pub mod object_api_utils; +pub mod object_handlers_common; +pub mod transition_api; diff --git a/ecstore/src/client/object_api_utils.rs b/ecstore/src/client/object_api_utils.rs new file mode 100644 index 000000000..e824968b8 --- /dev/null +++ b/ecstore/src/client/object_api_utils.rs @@ -0,0 +1,141 @@ +#![allow(clippy::map_entry)] +use http::HeaderMap; +use std::io::Cursor; +use std::{collections::HashMap, sync::Arc}; +use tokio::io::BufReader; + +use crate::error::ErrorResponse; +use crate::store_api::{GetObjectReader, HTTPRangeSpec, ObjectInfo, ObjectOptions}; +use rustfs_filemeta::fileinfo::ObjectPartInfo; +use rustfs_rio::HashReader; +use s3s::S3ErrorCode; + +//#[derive(Clone)] +pub struct PutObjReader { + pub reader: HashReader, + pub raw_reader: HashReader, + //pub sealMD5Fn: SealMD5CurrFn, +} + +impl PutObjReader { + pub fn new(raw_reader: HashReader) -> Self { + todo!(); + } + + fn size(&self) -> usize { + //self.reader.size() + todo!(); + } + + fn md5_current_hex_string(&self) -> String { + todo!(); + } + + fn with_encryption(&mut self, enc_reader: HashReader) -> Result<(), std::io::Error> { + self.reader = enc_reader; + + Ok(()) + } +} + +pub type ObjReaderFn = Arc>>, HeaderMap) -> GetObjectReader + 'static>; + +fn part_number_to_rangespec(oi: ObjectInfo, part_number: usize) -> Option { + if oi.size == 0 || oi.parts.len() == 0 { + return None; + } + + let mut start: i64 = 0; + let mut end: i64 = -1; + let mut i = 0; + while i < oi.parts.len() && i < part_number { + start = end + 1; + end = start + oi.parts[i].actual_size as i64 - 1; + i += 1; + } + + Some(HTTPRangeSpec { + start: start as usize, + end: Some(end as usize), + is_suffix_length: false, + }) +} + +fn get_compressed_offsets(oi: ObjectInfo, offset: i64) -> (i64, i64, i64, i64, u64) { + let mut skip_length: i64 = 0; + let mut cumulative_actual_size: i64 = 0; + let mut first_part_idx: i64 = 0; + let mut compressed_offset: i64 = 0; + let mut part_skip: i64 = 0; + let mut decrypt_skip: i64 = 0; + let mut seq_num: u64 = 0; + for (i, part) in oi.parts.iter().enumerate() { + cumulative_actual_size += part.actual_size as i64; + if cumulative_actual_size <= offset { + compressed_offset += part.size as i64; + } else { + first_part_idx = i as i64; + skip_length = cumulative_actual_size - part.actual_size as i64; + break; + } + } + skip_length = offset - skip_length; + + let parts: &[ObjectPartInfo] = &oi.parts; + if skip_length > 0 + && parts.len() > first_part_idx as usize + && parts[first_part_idx as usize].index.as_ref().expect("err").len() > 0 + { + todo!(); + } + + (compressed_offset, part_skip, first_part_idx, decrypt_skip, seq_num) +} + +pub fn new_getobjectreader( + rs: HTTPRangeSpec, + oi: &ObjectInfo, + opts: &ObjectOptions, + h: &HeaderMap, +) -> Result<(ObjReaderFn, i64, i64), ErrorResponse> { + //let (_, mut is_encrypted) = crypto.is_encrypted(oi.user_defined)?; + let mut is_encrypted = false; + let is_compressed = false; //oi.is_compressed_ok(); + + let mut get_fn: ObjReaderFn; + + let (off, length) = match rs.get_offset_length(oi.size) { + Ok(x) => x, + Err(err) => { + return Err(ErrorResponse { + code: S3ErrorCode::InvalidRange, + message: err.to_string(), + key: None, + bucket_name: None, + region: None, + request_id: None, + host_id: "".to_string(), + }); + } + }; + get_fn = Arc::new(move |input_reader: BufReader>>, _: HeaderMap| { + //Box::pin({ + /*let r = GetObjectReader { + object_info: oi.clone(), + stream: StreamingBlob::new(HashReader::new(input_reader, 10, None, None, 10)), + }; + r*/ + todo!(); + //}) + }); + + Ok((get_fn, off as i64, length as i64)) +} + +pub fn extract_etag(metadata: &HashMap) -> String { + if let Some(etag) = metadata.get("etag") { + etag.clone() + } else { + metadata["md5Sum"].clone() + } +} diff --git a/ecstore/src/client/object_handlers_common.rs b/ecstore/src/client/object_handlers_common.rs new file mode 100644 index 000000000..8e5b87618 --- /dev/null +++ b/ecstore/src/client/object_handlers_common.rs @@ -0,0 +1,33 @@ +use crate::StorageAPI; +use crate::bucket::lifecycle::lifecycle; +use crate::bucket::versioning::VersioningApi; +use crate::bucket::versioning_sys::BucketVersioningSys; +use crate::store::ECStore; +use crate::store_api::{ObjectOptions, ObjectToDelete}; +use lock::local_locker::MAX_DELETE_LIST; + +pub async fn delete_object_versions(api: ECStore, bucket: &str, to_del: &[ObjectToDelete], lc_event: lifecycle::Event) { + let mut remaining = to_del; + loop { + if remaining.len() <= 0 { + break; + }; + let mut to_del = remaining; + if to_del.len() > MAX_DELETE_LIST { + remaining = &to_del[MAX_DELETE_LIST..]; + to_del = &to_del[..MAX_DELETE_LIST]; + } else { + remaining = &[]; + } + let vc = BucketVersioningSys::get(bucket).await.expect("err!"); + let deleted_objs = api.delete_objects( + bucket, + to_del.to_vec(), + ObjectOptions { + //prefix_enabled_fn: vc.prefix_enabled(""), + version_suspended: vc.suspended(), + ..Default::default() + }, + ); + } +} diff --git a/ecstore/src/client/transition_api.rs b/ecstore/src/client/transition_api.rs new file mode 100644 index 000000000..830feffbb --- /dev/null +++ b/ecstore/src/client/transition_api.rs @@ -0,0 +1,1025 @@ +#![allow(clippy::map_entry)] +use bytes::Bytes; +use futures::Future; +use http::{HeaderMap, HeaderName}; +use http::{ + HeaderValue, Response, StatusCode, + request::{Builder, Request}, +}; +use hyper_rustls::{ConfigBuilderExt, HttpsConnector}; +use hyper_util::{client::legacy::Client, client::legacy::connect::HttpConnector, rt::TokioExecutor}; +use rand::Rng; +use serde::{Deserialize, Serialize}; +use std::io::Cursor; +use std::pin::Pin; +use std::sync::atomic::{AtomicI32, Ordering}; +use std::task::{Context, Poll}; +use std::{ + collections::HashMap, + sync::{Arc, Mutex}, +}; +use time::Duration; +use time::OffsetDateTime; +use tokio::io::BufReader; +use tracing::{debug, error}; +use url::{Url, form_urlencoded}; +use uuid::Uuid; + +use crate::client::bucket_cache::BucketLocationCache; +use crate::client::{ + api_error_response::{err_invalid_argument, http_resp_to_error_response, to_error_response}, + api_get_options::GetObjectOptions, + api_put_object::PutObjectOptions, + api_put_object_multipart::UploadPartParams, + api_s3_datatypes::{ + CompleteMultipartUpload, CompletePart, ListBucketResult, ListBucketV2Result, ListMultipartUploadsResult, + ListObjectPartsResult, ObjectPart, + }, + constants::{UNSIGNED_PAYLOAD, UNSIGNED_PAYLOAD_TRAILER}, + credentials::{CredContext, Credentials, SignatureType, Static}, +}; +use crate::signer; +use crate::{checksum::ChecksumMode, store_api::GetObjectReader}; +use reader::hasher::{MD5, Sha256}; +use rustfs_rio::HashReader; +use rustfs_utils::{ + net::get_endpoint_url, + retry::{MAX_RETRY, new_retry_timer}, +}; +use s3s::S3ErrorCode; +use s3s::dto::ReplicationStatus; +use s3s::{Body, dto::Owner}; + +const C_USER_AGENT_PREFIX: &str = "RustFS (linux; x86)"; +const C_USER_AGENT: &str = "RustFS (linux; x86)"; + +const SUCCESS_STATUS: [StatusCode; 3] = [StatusCode::OK, StatusCode::NO_CONTENT, StatusCode::PARTIAL_CONTENT]; + +const C_UNKNOWN: i32 = -1; +const C_OFFLINE: i32 = 0; +const C_ONLINE: i32 = 1; + +//pub type ReaderImpl = Box; +pub enum ReaderImpl { + Body(Bytes), + ObjectBody(GetObjectReader), +} + +pub type ReadCloser = BufReader>>; + +pub struct TransitionClient { + pub endpoint_url: Url, + pub creds_provider: Arc>>, + pub override_signer_type: SignatureType, + /*app_info: TODO*/ + pub secure: bool, + pub http_client: Client, Body>, + //pub http_trace: Httptrace.ClientTrace, + pub bucket_loc_cache: Arc>, + pub is_trace_enabled: Arc>, + pub trace_errors_only: Arc>, + //pub trace_output: io.Writer, + pub s3_accelerate_endpoint: Arc>, + pub s3_dual_stack_enabled: Arc>, + pub region: String, + pub random: u64, + pub lookup: BucketLookupType, + //pub lookupFn: func(u url.URL, bucketName string) BucketLookupType, + pub md5_hasher: Arc>>, + pub sha256_hasher: Option, + pub health_status: AtomicI32, + pub trailing_header_support: bool, + pub max_retries: i64, +} + +#[derive(Debug, Default)] +pub struct Options { + pub creds: Credentials, + pub secure: bool, + //pub transport: http.RoundTripper, + //pub trace: *httptrace.ClientTrace, + pub region: String, + pub bucket_lookup: BucketLookupType, + //pub custom_region_via_url: func(u url.URL) string, + //pub bucket_lookup_via_url: func(u url.URL, bucketName string) BucketLookupType, + pub trailing_headers: bool, + pub custom_md5: Option, + pub custom_sha256: Option, + pub max_retries: i64, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +enum BucketLookupType { + #[default] + BucketLookupAuto, + BucketLookupDNS, + BucketLookupPath, +} + +impl TransitionClient { + pub async fn new(endpoint: &str, opts: Options) -> Result { + let clnt = Self::private_new(endpoint, opts).await?; + + Ok(clnt) + } + + async fn private_new(endpoint: &str, opts: Options) -> Result { + let endpoint_url = get_endpoint_url(endpoint, opts.secure)?; + + //let jar = cookiejar.New(cookiejar.Options{PublicSuffixList: publicsuffix.List})?; + + //#[cfg(feature = "ring")] + //let _ = rustls::crypto::ring::default_provider().install_default(); + //#[cfg(feature = "aws-lc-rs")] + let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); + + let scheme = endpoint_url.scheme(); + let client; + //if scheme == "https" { + // client = Client::builder(TokioExecutor::new()).build_http(); + //} else { + let tls = rustls::ClientConfig::builder().with_native_roots()?.with_no_client_auth(); + let https = hyper_rustls::HttpsConnectorBuilder::new() + .with_tls_config(tls) + .https_or_http() + .enable_http1() + .build(); + client = Client::builder(TokioExecutor::new()).build(https); + //} + + let mut clnt = TransitionClient { + endpoint_url, + creds_provider: Arc::new(Mutex::new(opts.creds)), + override_signer_type: SignatureType::SignatureDefault, + secure: opts.secure, + http_client: client, + bucket_loc_cache: Arc::new(Mutex::new(BucketLocationCache::new())), + is_trace_enabled: Arc::new(Mutex::new(false)), + trace_errors_only: Arc::new(Mutex::new(false)), + s3_accelerate_endpoint: Arc::new(Mutex::new("".to_string())), + s3_dual_stack_enabled: Arc::new(Mutex::new(false)), + region: opts.region, + random: rand::rng().random_range(10..=50), + lookup: opts.bucket_lookup, + md5_hasher: Arc::new(Mutex::new(opts.custom_md5)), + sha256_hasher: opts.custom_sha256, + health_status: AtomicI32::new(C_UNKNOWN), + trailing_header_support: opts.trailing_headers, + max_retries: opts.max_retries, + }; + + { + let mut md5_hasher = clnt.md5_hasher.lock().unwrap(); + if md5_hasher.is_none() { + *md5_hasher = Some(MD5::new()); + } + } + if clnt.sha256_hasher.is_none() { + clnt.sha256_hasher = Some(Sha256::new()); + } + + clnt.trailing_header_support = opts.trailing_headers && clnt.override_signer_type == SignatureType::SignatureV4; + + if opts.max_retries > 0 { + clnt.max_retries = opts.max_retries; + } + + Ok(clnt) + } + + fn endpoint_url(&self) -> Url { + self.endpoint_url.clone() + } + + fn set_appinfo(&self, app_name: &str, app_version: &str) { + /*if app_name != "" && app_version != "" { + self.appInfo.app_name = app_name + self.appInfo.app_version = app_version + }*/ + } + + fn trace_errors_only_off(&self) { + let mut trace_errors_only = self.trace_errors_only.lock().unwrap(); + *trace_errors_only = false; + } + + fn trace_off(&self) { + let mut is_trace_enabled = self.is_trace_enabled.lock().unwrap(); + *is_trace_enabled = false; + let mut trace_errors_only = self.trace_errors_only.lock().unwrap(); + *trace_errors_only = false; + } + + fn set_s3_transfer_accelerate(&self, accelerate_endpoint: &str) { + todo!(); + } + + fn set_s3_enable_dual_stack(&self, enabled: bool) { + todo!(); + } + + pub fn hash_materials( + &self, + is_md5_requested: bool, + is_sha256_requested: bool, + ) -> (HashMap, HashMap>) { + todo!(); + } + + fn is_online(&self) -> bool { + !self.is_offline() + } + + fn mark_offline(&self) { + self.health_status + .compare_exchange(C_ONLINE, C_OFFLINE, Ordering::SeqCst, Ordering::SeqCst); + } + + fn is_offline(&self) -> bool { + self.health_status.load(Ordering::SeqCst) == C_OFFLINE + } + + fn health_check(hc_duration: Duration) { + todo!(); + } + + fn dump_http(&self, req: &http::Request, resp: &http::Response) -> Result<(), std::io::Error> { + let mut resp_trace: Vec; + + //info!("{}{}", self.trace_output, "---------END-HTTP---------"); + + Ok(()) + } + + pub async fn doit(&self, req: http::Request) -> Result, std::io::Error> { + let req_method; + let req_uri; + let req_headers; + let resp; + let http_client = self.http_client.clone(); + { + //let mut http_client = http_client.lock().unwrap(); + req_method = req.method().clone(); + req_uri = req.uri().clone(); + req_headers = req.headers().clone(); + + debug!("endpoint_url: {}", self.endpoint_url.as_str().to_string()); + resp = http_client.request(req); + } + let resp = resp + .await /*.map_err(Into::into)*/ + .map(|res| res.map(Body::from)); + debug!("http_client url: {} {}", req_method, req_uri); + debug!("http_client headers: {:?}", req_headers); + if let Err(err) = resp { + error!("http_client call error: {:?}", err); + return Err(std::io::Error::other(err)); + } + + let mut resp = resp.unwrap(); + debug!("http_resp: {:?}", resp); + + //if self.is_trace_enabled && !(self.trace_errors_only && resp.status() == StatusCode::OK) { + if resp.status() != StatusCode::OK { + //self.dump_http(&cloned_req, &resp)?; + let b = resp.body_mut().store_all_unlimited().await.unwrap().to_vec(); + debug!("err_body: {}", String::from_utf8(b).unwrap()); + } + + Ok(resp) + } + + pub async fn execute_method( + &self, + method: http::Method, + metadata: &mut RequestMetadata, + ) -> Result, std::io::Error> { + if self.is_offline() { + let mut s = self.endpoint_url.to_string(); + s.push_str(" is offline."); + return Err(std::io::Error::other(s)); + } + + let mut retryable: bool; + //let mut body_seeker: BufferReader; + let mut req_retry = self.max_retries; + let mut resp: http::Response; + + //if metadata.content_body != nil { + //body_seeker = BufferReader::new(metadata.content_body.read_all().await?); + retryable = true; + if !retryable { + req_retry = 1; + } + //} + + //let mut retry_timer = RetryTimer::new(); + //while let Some(v) = retry_timer.next().await { + for _ in [1; 1] + /*new_retry_timer(req_retry, DefaultRetryUnit, DefaultRetryCap, MaxJitter)*/ + { + let req = self.new_request(method, metadata).await?; + + resp = self.doit(req).await?; + + for http_status in SUCCESS_STATUS { + if http_status == resp.status() { + return Ok(resp); + } + } + + let b = resp.body_mut().store_all_unlimited().await.unwrap().to_vec(); + let err_response = http_resp_to_error_response(resp, b.clone(), &metadata.bucket_name, &metadata.object_name); + + if self.region == "" { + match err_response.code { + S3ErrorCode::AuthorizationHeaderMalformed | S3ErrorCode::InvalidArgument /*S3ErrorCode::InvalidRegion*/ => { + //break; + return Err(std::io::Error::other(err_response)); + } + S3ErrorCode::AccessDenied => { + if err_response.region == "" { + return Err(std::io::Error::other(err_response)); + } + if metadata.bucket_name != "" { + let mut bucket_loc_cache = self.bucket_loc_cache.lock().unwrap(); + let location = bucket_loc_cache.get(&metadata.bucket_name); + if location.is_some() && location.unwrap() != err_response.region { + bucket_loc_cache.set(&metadata.bucket_name, &err_response.region); + //continue; + } + } else { + if err_response.region != metadata.bucket_location { + metadata.bucket_location = err_response.region.clone(); + //continue; + } + } + return Err(std::io::Error::other(err_response)); + } + _ => { + return Err(std::io::Error::other(err_response)); + } + } + } + + break; + } + + Err(std::io::Error::other("resp err")) + } + + async fn new_request( + &self, + method: http::Method, + metadata: &mut RequestMetadata, + ) -> Result, std::io::Error> { + let location = metadata.bucket_location.clone(); + if location == "" { + if metadata.bucket_name != "" { + let location = self.get_bucket_location(&metadata.bucket_name).await?; + } + } + + let is_makebucket = metadata.object_name == "" && method == http::Method::PUT && metadata.query_values.len() == 0; + let is_virtual_host = self.is_virtual_host_style_request(&self.endpoint_url, &metadata.bucket_name) && !is_makebucket; + + let target_url = self.make_target_url( + &metadata.bucket_name, + &metadata.object_name, + &location, + is_virtual_host, + &metadata.query_values, + )?; + + let mut req_builder = Request::builder().method(method).uri(target_url.to_string()); + + let value; + { + let mut creds_provider = self.creds_provider.lock().unwrap(); + value = creds_provider.get_with_context(Some(self.cred_context()))?; + } + + let mut signer_type = value.signer_type.clone(); + let access_key_id = value.access_key_id; + let secret_access_key = value.secret_access_key; + let session_token = value.session_token; + + if self.override_signer_type != SignatureType::SignatureDefault { + signer_type = self.override_signer_type.clone(); + } + + if value.signer_type == SignatureType::SignatureAnonymous { + signer_type = SignatureType::SignatureAnonymous; + } + + if metadata.expires != 0 && metadata.pre_sign_url { + if signer_type == SignatureType::SignatureAnonymous { + return Err(std::io::Error::other(err_invalid_argument( + "Presigned URLs cannot be generated with anonymous credentials.", + ))); + } + if metadata.extra_pre_sign_header.is_some() { + if signer_type == SignatureType::SignatureV2 { + return Err(std::io::Error::other(err_invalid_argument( + "Extra signed headers for Presign with Signature V2 is not supported.", + ))); + } + for (k, v) in metadata.extra_pre_sign_header.as_ref().unwrap() { + req_builder = req_builder.header(k, v); + } + } + if signer_type == SignatureType::SignatureV2 { + req_builder = + signer::pre_sign_v2(req_builder, &access_key_id, &secret_access_key, metadata.expires, is_virtual_host); + } else if signer_type == SignatureType::SignatureV4 { + req_builder = signer::pre_sign_v4( + req_builder, + &access_key_id, + &secret_access_key, + &session_token, + &location, + metadata.expires, + OffsetDateTime::now_utc(), + ); + } + let req = match req_builder.body(Body::empty()) { + Ok(req) => req, + Err(err) => { + return Err(std::io::Error::other(err)); + } + }; + return Ok(req); + } + + self.set_user_agent(&mut req_builder); + + for (k, v) in metadata.custom_header.clone() { + req_builder.headers_mut().expect("err").insert(k.expect("err"), v); + } + + //req.content_length = metadata.content_length; + if metadata.content_length <= -1 { + let chunked_value = HeaderValue::from_str(&vec!["chunked"].join(",")).expect("err"); + req_builder + .headers_mut() + .expect("err") + .insert(http::header::TRANSFER_ENCODING, chunked_value); + } + + if metadata.content_md5_base64.len() > 0 { + let md5_value = HeaderValue::from_str(&metadata.content_md5_base64).expect("err"); + req_builder.headers_mut().expect("err").insert("Content-Md5", md5_value); + } + + if signer_type == SignatureType::SignatureAnonymous { + let req = match req_builder.body(Body::empty()) { + Ok(req) => req, + Err(err) => { + return Err(std::io::Error::other(err)); + } + }; + return Ok(req); + } + + if signer_type == SignatureType::SignatureV2 { + req_builder = + signer::sign_v2(req_builder, metadata.content_length, &access_key_id, &secret_access_key, is_virtual_host); + } else if metadata.stream_sha256 && !self.secure { + if metadata.trailer.len() > 0 { + //req.Trailer = metadata.trailer; + for (_, v) in &metadata.trailer { + req_builder = req_builder.header(http::header::TRAILER, v.clone()); + } + } + //req_builder = signer::streaming_sign_v4(req_builder, &access_key_id, + // &secret_access_key, &session_token, &location, metadata.content_length, OffsetDateTime::now_utc(), self.sha256_hasher()); + } else { + let mut sha_header = UNSIGNED_PAYLOAD.to_string(); + if metadata.content_sha256_hex != "" { + sha_header = metadata.content_sha256_hex.clone(); + if metadata.trailer.len() > 0 { + return Err(std::io::Error::other("internal error: content_sha256_hex with trailer not supported")); + } + } else if metadata.trailer.len() > 0 { + sha_header = UNSIGNED_PAYLOAD_TRAILER.to_string(); + } + req_builder = req_builder + .header::("X-Amz-Content-Sha256".parse().unwrap(), sha_header.parse().expect("err")); + + req_builder = signer::sign_v4_trailer( + req_builder, + &access_key_id, + &secret_access_key, + &session_token, + &location, + metadata.trailer.clone(), + ); + } + + let req; + if metadata.content_length == 0 { + req = req_builder.body(Body::empty()); + } else { + match &mut metadata.content_body { + ReaderImpl::Body(content_body) => { + req = req_builder.body(Body::from(content_body.clone())); + } + ReaderImpl::ObjectBody(content_body) => { + req = req_builder.body(Body::from(content_body.read_all().await?)); + } + } + //req = req_builder.body(s3s::Body::from(metadata.content_body.read_all().await?)); + } + + match req { + Ok(req) => Ok(req), + Err(err) => Err(std::io::Error::other(err)), + } + } + + pub fn set_user_agent(&self, req: &mut Builder) { + let mut headers = req.headers_mut().expect("err"); + headers.insert("User-Agent", C_USER_AGENT.parse().expect("err")); + /*if self.app_info.app_name != "" && self.app_info.app_version != "" { + headers.insert("User-Agent", C_USER_AGENT+" "+self.app_info.app_name+"/"+self.app_info.app_version); + }*/ + } + + fn make_target_url( + &self, + bucket_name: &str, + object_name: &str, + bucket_location: &str, + is_virtual_host_style: bool, + query_values: &HashMap, + ) -> Result { + let scheme = self.endpoint_url.scheme(); + let host = self.endpoint_url.host().unwrap(); + let default_port = if scheme == "https" { 443 } else { 80 }; + let port = self.endpoint_url.port().unwrap_or(default_port); + + let mut url_str = format!("{scheme}://{host}:{port}/"); + + if bucket_name != "" { + if is_virtual_host_style { + url_str = format!("{scheme}://{bucket_name}.{host}:{port}/"); + if object_name != "" { + url_str.push_str(object_name); + } + } else { + url_str.push_str(bucket_name); + url_str.push_str("/"); + if object_name != "" { + url_str.push_str(object_name); + } + } + } + + if query_values.len() > 0 { + let mut encoded = form_urlencoded::Serializer::new(String::new()); + for (k, v) in query_values { + encoded.append_pair(&k, &v); + } + url_str.push_str("?"); + url_str.push_str(&encoded.finish()); + } + + Url::parse(&url_str).map_err(|e| std::io::Error::other(e.to_string())) + } + + pub fn is_virtual_host_style_request(&self, url: &Url, bucket_name: &str) -> bool { + if bucket_name == "" { + return false; + } + + if self.lookup == BucketLookupType::BucketLookupDNS { + return true; + } + + if self.lookup == BucketLookupType::BucketLookupPath { + return false; + } + + false + } + + pub fn cred_context(&self) -> CredContext { + CredContext { + //client: http_client, + endpoint: self.endpoint_url.to_string(), + } + } +} + +struct LockedRandSource { + src: u64, //rand.Source, +} + +impl LockedRandSource { + fn int63(&self) -> i64 { + /*let n = self.src.int63(); + n*/ + todo!(); + } + + fn seed(&self, seed: i64) { + //self.src.seed(seed); + todo!(); + } +} + +pub struct RequestMetadata { + pub pre_sign_url: bool, + pub bucket_name: String, + pub object_name: String, + pub query_values: HashMap, + pub custom_header: HeaderMap, + pub extra_pre_sign_header: Option, + pub expires: i64, + pub bucket_location: String, + pub content_body: ReaderImpl, + pub content_length: i64, + pub content_md5_base64: String, + pub content_sha256_hex: String, + pub stream_sha256: bool, + pub add_crc: ChecksumMode, + pub trailer: HeaderMap, +} + +pub struct TransitionCore(pub Arc); + +impl TransitionCore { + pub async fn new(endpoint: &str, opts: Options) -> Result { + let client = TransitionClient::new(endpoint, opts).await?; + Ok(Self(Arc::new(client))) + } + + pub fn list_objects( + &self, + bucket: &str, + prefix: &str, + marker: &str, + delimiter: &str, + max_keys: i64, + ) -> Result { + let client = self.0.clone(); + client.list_objects_query(bucket, prefix, marker, delimiter, max_keys, HeaderMap::new()) + } + + pub async fn list_objects_v2( + &self, + bucket_name: &str, + object_prefix: &str, + start_after: &str, + continuation_token: &str, + delimiter: &str, + max_keys: i64, + ) -> Result { + let client = self.0.clone(); + client + .list_objects_v2_query( + bucket_name, + object_prefix, + continuation_token, + true, + false, + delimiter, + start_after, + max_keys, + HeaderMap::new(), + ) + .await + } + + /*pub fn copy_object(&self, source_bucket: &str, source_object: &str, dest_bucket: &str, dest_object: &str, metadata: HashMap, src_opts: CopySrcOptions, dst_opts: PutObjectOptions) -> Result { + self.0.copy_object_do(source_bucket, source_object, dest_bucket, dest_object, metadata, src_opts, dst_opts) + }*/ + + pub fn copy_object_part( + &self, + src_bucket: &str, + src_object: &str, + dest_bucket: &str, + dest_object: &str, + upload_id: &str, + part_id: i32, + start_offset: i32, + length: i64, + metadata: HashMap, + ) -> Result { + //self.0.copy_object_part_do(src_bucket, src_object, dest_bucket, dest_object, upload_id, + // part_id, start_offset, length, metadata) + todo!(); + } + + pub async fn put_object( + &self, + bucket: &str, + object: &str, + data: ReaderImpl, + size: i64, + md5_base64: &str, + sha256_hex: &str, + opts: &PutObjectOptions, + ) -> Result { + let hook_reader = data; //newHook(data, opts.progress); + let client = self.0.clone(); + client + .put_object_do(bucket, object, hook_reader, md5_base64, sha256_hex, size, opts) + .await + } + + pub async fn new_multipart_upload( + &self, + bucket: &str, + object: &str, + opts: PutObjectOptions, + ) -> Result { + let client = self.0.clone(); + let result = client.initiate_multipart_upload(bucket, object, &opts).await?; + Ok(result.upload_id) + } + + pub fn list_multipart_uploads( + &self, + bucket: &str, + prefix: &str, + key_marker: &str, + upload_id_marker: &str, + delimiter: &str, + max_uploads: i64, + ) -> Result { + let client = self.0.clone(); + client.list_multipart_uploads_query(bucket, key_marker, upload_id_marker, prefix, delimiter, max_uploads) + } + + pub async fn put_object_part( + &self, + bucket: &str, + object: &str, + upload_id: &str, + part_id: i64, + data: ReaderImpl, + size: i64, + opts: PutObjectPartOptions, + ) -> Result { + let mut p = UploadPartParams { + bucket_name: bucket.to_string(), + object_name: object.to_string(), + upload_id: upload_id.to_string(), + reader: data, + part_number: part_id, + md5_base64: opts.md5_base64, + sha256_hex: opts.sha256_hex, + size: size, + //sse: opts.sse, + stream_sha256: !opts.disable_content_sha256, + custom_header: opts.custom_header, + trailer: opts.trailer, + }; + let client = self.0.clone(); + client.upload_part(&mut p).await + } + + pub async fn list_object_parts( + &self, + bucket: &str, + object: &str, + upload_id: &str, + part_number_marker: i64, + max_parts: i64, + ) -> Result { + let client = self.0.clone(); + client + .list_object_parts_query(bucket, object, upload_id, part_number_marker, max_parts) + .await + } + + pub async fn complete_multipart_upload( + &self, + bucket: &str, + object: &str, + upload_id: &str, + parts: &[CompletePart], + opts: PutObjectOptions, + ) -> Result { + let client = self.0.clone(); + let res = client + .complete_multipart_upload(bucket, object, upload_id, CompleteMultipartUpload { parts: parts.to_vec() }, &opts) + .await?; + Ok(res) + } + + pub async fn abort_multipart_upload(&self, bucket_name: &str, object: &str, upload_id: &str) -> Result<(), std::io::Error> { + let client = self.0.clone(); + client.abort_multipart_upload(bucket_name, object, upload_id).await + } + + pub async fn get_bucket_policy(&self, bucket_name: &str) -> Result { + let client = self.0.clone(); + client.get_bucket_policy(bucket_name).await + } + + pub async fn put_bucket_policy(&self, bucket_name: &str, bucket_policy: &str) -> Result<(), std::io::Error> { + let client = self.0.clone(); + client.put_bucket_policy(bucket_name, bucket_policy).await + } + + pub async fn get_object( + &self, + bucket_name: &str, + object_name: &str, + opts: &GetObjectOptions, + ) -> Result<(ObjectInfo, HeaderMap, ReadCloser), std::io::Error> { + let client = self.0.clone(); + client.get_object_inner(bucket_name, object_name, opts).await + } +} + +pub struct PutObjectPartOptions { + pub md5_base64: String, + pub sha256_hex: String, + //pub sse: encrypt.ServerSide, + pub custom_header: HeaderMap, + pub trailer: HeaderMap, + pub disable_content_sha256: bool, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct ObjectInfo { + pub etag: String, + pub name: String, + pub mod_time: OffsetDateTime, + pub size: usize, + pub content_type: Option, + #[serde(skip)] + pub metadata: HeaderMap, + pub user_metadata: HashMap, + pub user_tags: String, + pub user_tag_count: i64, + #[serde(skip)] + pub owner: Owner, + //pub grant: Vec, + pub storage_class: String, + pub is_latest: bool, + pub is_delete_marker: bool, + pub version_id: Uuid, + + #[serde(skip, default = "replication_status_default")] + pub replication_status: ReplicationStatus, + pub replication_ready: bool, + pub expiration: OffsetDateTime, + pub expiration_rule_id: String, + pub num_versions: usize, + + pub restore: RestoreInfo, + + pub checksum_crc32: String, + pub checksum_crc32c: String, + pub checksum_sha1: String, + pub checksum_sha256: String, + pub checksum_crc64nvme: String, + pub checksum_mode: String, +} + +fn replication_status_default() -> ReplicationStatus { + ReplicationStatus::from_static(ReplicationStatus::PENDING) +} + +impl Default for ObjectInfo { + fn default() -> Self { + Self { + etag: "".to_string(), + name: "".to_string(), + mod_time: OffsetDateTime::now_utc(), + size: 0, + content_type: None, + metadata: HeaderMap::new(), + user_metadata: HashMap::new(), + user_tags: "".to_string(), + user_tag_count: 0, + owner: Owner::default(), + storage_class: "".to_string(), + is_latest: false, + is_delete_marker: false, + version_id: Uuid::nil(), + replication_status: ReplicationStatus::from_static(ReplicationStatus::PENDING), + replication_ready: false, + expiration: OffsetDateTime::now_utc(), + expiration_rule_id: "".to_string(), + num_versions: 0, + restore: RestoreInfo::default(), + checksum_crc32: "".to_string(), + checksum_crc32c: "".to_string(), + checksum_sha1: "".to_string(), + checksum_sha256: "".to_string(), + checksum_crc64nvme: "".to_string(), + checksum_mode: "".to_string(), + } + } +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct RestoreInfo { + ongoing_restore: bool, + expiry_time: OffsetDateTime, +} + +impl Default for RestoreInfo { + fn default() -> Self { + Self { + ongoing_restore: false, + expiry_time: OffsetDateTime::now_utc(), + } + } +} + +pub struct ObjectMultipartInfo { + pub initiated: OffsetDateTime, + //pub initiator: initiator, + //pub owner: owner, + pub storage_class: String, + pub key: String, + pub size: i64, + pub upload_id: String, + //pub err error, +} + +pub struct UploadInfo { + pub bucket: String, + pub key: String, + pub etag: String, + pub size: i64, + pub last_modified: OffsetDateTime, + pub location: String, + pub version_id: String, + pub expiration: OffsetDateTime, + pub expiration_rule_id: String, + pub checksum_crc32: String, + pub checksum_crc32c: String, + pub checksum_sha1: String, + pub checksum_sha256: String, + pub checksum_crc64nvme: String, + pub checksum_mode: String, +} + +impl Default for UploadInfo { + fn default() -> Self { + Self { + bucket: "".to_string(), + key: "".to_string(), + etag: "".to_string(), + size: 0, + last_modified: OffsetDateTime::now_utc(), + location: "".to_string(), + version_id: "".to_string(), + expiration: OffsetDateTime::now_utc(), + expiration_rule_id: "".to_string(), + checksum_crc32: "".to_string(), + checksum_crc32c: "".to_string(), + checksum_sha1: "".to_string(), + checksum_sha256: "".to_string(), + checksum_crc64nvme: "".to_string(), + checksum_mode: "".to_string(), + } + } +} + +pub fn to_object_info(bucket_name: &str, object_name: &str, h: &HeaderMap) -> Result { + todo!() +} + +type BoxFuture<'a, T> = Pin + Send + 'a>>; + +//#[derive(Clone)] +pub struct SendRequest { + inner: hyper::client::conn::http1::SendRequest, +} + +impl From> for SendRequest { + fn from(inner: hyper::client::conn::http1::SendRequest) -> Self { + Self { inner } + } +} + +impl tower::Service> for SendRequest { + type Response = Response; + type Error = std::io::Error; + type Future = BoxFuture<'static, Result>; + + fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { + self.inner.poll_ready(cx).map_err(std::io::Error::other) + } + + fn call(&mut self, req: Request) -> Self::Future { + //let req = hyper::Request::builder().uri("/").body(http_body_util::Empty::::new()).unwrap(); + //let req = hyper::Request::builder().uri("/").body(Body::empty()).unwrap(); + + let fut = self.inner.send_request(req); + + Box::pin(async move { fut.await.map_err(std::io::Error::other).map(|res| res.map(Body::from)) }) + } +} + +#[derive(Serialize, Deserialize)] +pub struct Document(pub String); diff --git a/ecstore/src/cmd/bucket_replication.rs b/ecstore/src/cmd/bucket_replication.rs index df36cd63c..c44f64149 100644 --- a/ecstore/src/cmd/bucket_replication.rs +++ b/ecstore/src/cmd/bucket_replication.rs @@ -56,6 +56,7 @@ use tracing::{debug, error, info, warn}; use uuid::Uuid; use xxhash_rust::xxh3::xxh3_64; // use bucket_targets::{self, GLOBAL_Bucket_Target_Sys}; +use crate::bucket::lifecycle::bucket_lifecycle_ops::TransitionedObject; #[derive(Serialize, Deserialize, Debug)] struct MRFReplicateEntry { @@ -2013,6 +2014,7 @@ impl ReplicateObjectInfo { data_blocks: 0, version_id: Uuid::try_parse(&self.version_id).ok(), delete_marker: self.delete_marker, + transitioned_object: TransitionedObject::default(), user_tags: self.user_tags.clone(), parts: Vec::new(), is_latest: true, diff --git a/ecstore/src/disk/local.rs b/ecstore/src/disk/local.rs index f45cb4acb..a5d3d053d 100644 --- a/ecstore/src/disk/local.rs +++ b/ecstore/src/disk/local.rs @@ -2149,7 +2149,7 @@ impl DiskAPI for LocalDisk { // Check if the current bucket has a configured lifecycle policy if let Ok((lc, _)) = metadata_sys::get_lifecycle_config(&cache.info.name).await { if lc_has_active_rules(&lc, "") { - cache.info.life_cycle = Some(lc); + cache.info.lifecycle = Some(lc); } } @@ -2258,9 +2258,9 @@ impl DiskAPI for LocalDisk { } } - for frer_version in fivs.free_versions.iter() { + for free_version in fivs.free_versions.iter() { let _obj_info = ObjectInfo::from_file_info( - frer_version, + free_version, &item.bucket, &item.object_path().to_string_lossy(), versioned, diff --git a/ecstore/src/error.rs b/ecstore/src/error.rs index 4fb5c7eed..45768e0ba 100644 --- a/ecstore/src/error.rs +++ b/ecstore/src/error.rs @@ -1,3 +1,5 @@ +use s3s::{S3Error, S3ErrorCode}; + use rustfs_utils::path::decode_dir_object; use crate::disk::error::DiskError; @@ -724,6 +726,254 @@ pub fn to_object_err(err: Error, params: Vec<&str>) -> Error { } } +pub fn is_network_or_host_down(err: &str, expect_timeouts: bool) -> bool { + err.contains("Connection closed by foreign host") + || err.contains("TLS handshake timeout") + || err.contains("i/o timeout") + || err.contains("connection timed out") + || err.contains("connection reset by peer") + || err.contains("broken pipe") + || err.to_lowercase().contains("503 service unavailable") + || err.contains("use of closed network connection") + || err.contains("An existing connection was forcibly closed by the remote host") + || err.contains("client error (Connect)") +} + +#[derive(Debug, Default, PartialEq, Eq)] +pub struct GenericError { + pub bucket: String, + pub object: String, + pub version_id: String, + //pub err: Error, +} + +#[derive(Debug, thiserror::Error, PartialEq, Eq)] +pub enum ObjectApiError { + #[error("Operation timed out")] + OperationTimedOut, + + #[error("etag of the object has changed")] + InvalidETag, + + #[error("BackendDown")] + BackendDown(String), + + #[error("Unsupported headers in Metadata")] + UnsupportedMetadata, + + #[error("Method not allowed: {}/{}", .0.bucket, .0.object)] + MethodNotAllowed(GenericError), + + #[error("The operation is not valid for the current state of the object {}/{}({})", .0.bucket, .0.object, .0.version_id)] + InvalidObjectState(GenericError), +} + +#[derive(Debug, thiserror::Error, PartialEq, Eq)] +#[error("{}", .message)] +pub struct ErrorResponse { + pub code: S3ErrorCode, + pub message: String, + pub key: Option, + pub bucket_name: Option, + pub region: Option, + pub request_id: Option, + pub host_id: String, +} + +pub fn error_resp_to_object_err(err: ErrorResponse, params: Vec<&str>) -> std::io::Error { + let mut bucket = ""; + let mut object = ""; + let mut version_id = ""; + if params.len() >= 1 { + bucket = params[0]; + } + if params.len() >= 2 { + object = params[1]; + } + if params.len() >= 3 { + version_id = params[2]; + } + + if is_network_or_host_down(&err.to_string(), false) { + return std::io::Error::other(ObjectApiError::BackendDown(format!("{}", err))); + } + + let mut err_ = std::io::Error::other(err.to_string()); + let r_err = err; + let mut err = err_; + let bucket = bucket.to_string(); + let object = object.to_string(); + let version_id = version_id.to_string(); + + match r_err.code { + /*S3ErrorCode::SlowDownWrite => { + err = StorageError::InsufficientWriteQuorum;//{bucket: bucket, object: object}; + }*/ + /*S3ErrorCode::SlowDown/* SlowDownRead */ => { + err = std::io::Error::other(StorageError::InsufficientReadQuorum.to_string()); + }*/ + /*S3ErrorCode::PreconditionFailed => { + err = Error::from(StorageError::PreConditionFailed); + }*/ + /*S3ErrorCode::InvalidRange => { + err = Error::from(StorageError::InvalidRange); + }*/ + /*S3ErrorCode::BucketAlreadyOwnedByYou => { + err = Error::from(StorageError::BucketAlreadyOwnedByYou); + }*/ + S3ErrorCode::BucketNotEmpty => { + err = std::io::Error::other(StorageError::BucketNotEmpty("".to_string()).to_string()); + } + /*S3ErrorCode::NoSuchBucketPolicy => { + err = Error::from(StorageError::BucketPolicyNotFound); + }*/ + /*S3ErrorCode::NoSuchLifecycleConfiguration => { + err = Error::from(StorageError::BucketLifecycleNotFound); + }*/ + S3ErrorCode::InvalidBucketName => { + err = std::io::Error::other(StorageError::BucketNameInvalid(bucket)); + } + S3ErrorCode::InvalidPart => { + err = std::io::Error::other(StorageError::InvalidPart(0, bucket, object /* , version_id */)); + } + S3ErrorCode::NoSuchBucket => { + err = std::io::Error::other(StorageError::BucketNotFound(bucket)); + } + S3ErrorCode::NoSuchKey => { + if object != "" { + err = std::io::Error::other(StorageError::ObjectNotFound(bucket, object)); + } else { + err = std::io::Error::other(StorageError::BucketNotFound(bucket)); + } + } + S3ErrorCode::NoSuchVersion => { + if object != "" { + err = std::io::Error::other(StorageError::ObjectNotFound(bucket, object)); //, version_id); + } else { + err = std::io::Error::other(StorageError::BucketNotFound(bucket)); + } + } + /*S3ErrorCode::XRustFsInvalidObjectName => { + err = Error::from(StorageError::ObjectNameInvalid(bucket, object)); + }*/ + S3ErrorCode::AccessDenied => { + err = std::io::Error::other(StorageError::PrefixAccessDenied(bucket, object)); + } + /*S3ErrorCode::XAmzContentSHA256Mismatch => { + err = hash.SHA256Mismatch{}; + }*/ + S3ErrorCode::NoSuchUpload => { + err = std::io::Error::other(StorageError::InvalidUploadID(bucket, object, version_id)); + } + /*S3ErrorCode::EntityTooSmall => { + err = std::io::Error::other(StorageError::PartTooSmall); + }*/ + /*S3ErrorCode::ReplicationPermissionCheck => { + err = std::io::Error::other(StorageError::ReplicationPermissionCheck); + }*/ + _ => { + err = std::io::Error::other("err"); + } + } + + err +} + +pub fn storage_to_object_err(err: Error, params: Vec<&str>) -> S3Error { + let storage_err = &err; + let mut bucket: String = "".to_string(); + let mut object: String = "".to_string(); + if params.len() >= 1 { + bucket = params[0].to_string(); + } + if params.len() >= 2 { + object = decode_dir_object(params[1]); + } + return match storage_err { + /*StorageError::NotImplemented => s3_error!(NotImplemented), + StorageError::InvalidArgument(bucket, object, version_id) => { + s3_error!(InvalidArgument, "Invalid arguments provided for {}/{}-{}", bucket, object, version_id) + }*/ + StorageError::MethodNotAllowed => S3Error::with_message( + S3ErrorCode::MethodNotAllowed, + ObjectApiError::MethodNotAllowed(GenericError { + bucket: bucket, + object: object, + ..Default::default() + }) + .to_string(), + ), + /*StorageError::BucketNotFound(bucket) => { + s3_error!(NoSuchBucket, "bucket not found {}", bucket) + } + StorageError::BucketNotEmpty(bucket) => s3_error!(BucketNotEmpty, "bucket not empty {}", bucket), + StorageError::BucketNameInvalid(bucket) => s3_error!(InvalidBucketName, "invalid bucket name {}", bucket), + StorageError::ObjectNameInvalid(bucket, object) => { + s3_error!(InvalidArgument, "invalid object name {}/{}", bucket, object) + } + StorageError::BucketExists(bucket) => s3_error!(BucketAlreadyExists, "{}", bucket), + StorageError::StorageFull => s3_error!(ServiceUnavailable, "Storage reached its minimum free drive threshold."), + StorageError::SlowDown => s3_error!(SlowDown, "Please reduce your request rate"), + StorageError::PrefixAccessDenied(bucket, object) => { + s3_error!(AccessDenied, "PrefixAccessDenied {}/{}", bucket, object) + } + StorageError::InvalidUploadIDKeyCombination(bucket, object) => { + s3_error!(InvalidArgument, "Invalid UploadID KeyCombination: {}/{}", bucket, object) + } + StorageError::MalformedUploadID(bucket) => s3_error!(InvalidArgument, "Malformed UploadID: {}", bucket), + StorageError::ObjectNameTooLong(bucket, object) => { + s3_error!(InvalidArgument, "Object name too long: {}/{}", bucket, object) + } + StorageError::ObjectNamePrefixAsSlash(bucket, object) => { + s3_error!(InvalidArgument, "Object name contains forward slash as prefix: {}/{}", bucket, object) + } + StorageError::ObjectNotFound(bucket, object) => s3_error!(NoSuchKey, "{}/{}", bucket, object), + StorageError::VersionNotFound(bucket, object, version_id) => { + s3_error!(NoSuchVersion, "{}/{}/{}", bucket, object, version_id) + } + StorageError::InvalidUploadID(bucket, object, version_id) => { + s3_error!(InvalidPart, "Invalid upload id: {}/{}-{}", bucket, object, version_id) + } + StorageError::InvalidVersionID(bucket, object, version_id) => { + s3_error!(InvalidArgument, "Invalid version id: {}/{}-{}", bucket, object, version_id) + } + // extended + StorageError::DataMovementOverwriteErr(bucket, object, version_id) => s3_error!( + InvalidArgument, + "invalid data movement operation, source and destination pool are the same for : {}/{}-{}", + bucket, + object, + version_id + ), + // extended + StorageError::ObjectExistsAsDirectory(bucket, object) => { + s3_error!(InvalidArgument, "Object exists on :{} as directory {}", bucket, object) + } + StorageError::InsufficientReadQuorum => { + s3_error!(SlowDown, "Storage resources are insufficient for the read operation") + } + StorageError::InsufficientWriteQuorum => { + s3_error!(SlowDown, "Storage resources are insufficient for the write operation") + } + StorageError::DecommissionNotStarted => s3_error!(InvalidArgument, "Decommission Not Started"), + + StorageError::VolumeNotFound(bucket) => { + s3_error!(NoSuchBucket, "bucket not found {}", bucket) + } + StorageError::InvalidPart(bucket, object, version_id) => { + s3_error!( + InvalidPart, + "Specified part could not be found. PartNumber {}, Expected {}, got {}", + bucket, + object, + version_id + ) + } + StorageError::DoneForNow => s3_error!(InternalError, "DoneForNow"),*/ + _ => s3s::S3Error::with_message(S3ErrorCode::Custom("err".into()), err.to_string()), + }; +} + #[cfg(test)] mod tests { use super::*; diff --git a/ecstore/src/event/mod.rs b/ecstore/src/event/mod.rs new file mode 100644 index 000000000..fc878d340 --- /dev/null +++ b/ecstore/src/event/mod.rs @@ -0,0 +1,3 @@ +pub mod name; +pub mod targetid; +pub mod targetlist; diff --git a/ecstore/src/event/name.rs b/ecstore/src/event/name.rs new file mode 100644 index 000000000..8ecabe9f6 --- /dev/null +++ b/ecstore/src/event/name.rs @@ -0,0 +1,147 @@ +#[derive(Default)] +pub enum EventName { + ObjectAccessedGet, + ObjectAccessedGetRetention, + ObjectAccessedGetLegalHold, + ObjectAccessedHead, + ObjectAccessedAttributes, + ObjectCreatedCompleteMultipartUpload, + ObjectCreatedCopy, + ObjectCreatedPost, + ObjectCreatedPut, + ObjectCreatedPutRetention, + ObjectCreatedPutLegalHold, + ObjectCreatedPutTagging, + ObjectCreatedDeleteTagging, + ObjectRemovedDelete, + ObjectRemovedDeleteMarkerCreated, + ObjectRemovedDeleteAllVersions, + ObjectRemovedNoOP, + BucketCreated, + BucketRemoved, + ObjectReplicationFailed, + ObjectReplicationComplete, + ObjectReplicationMissedThreshold, + ObjectReplicationReplicatedAfterThreshold, + ObjectReplicationNotTracked, + ObjectRestorePost, + ObjectRestoreCompleted, + ObjectTransitionFailed, + ObjectTransitionComplete, + ObjectManyVersions, + ObjectLargeVersions, + PrefixManyFolders, + ILMDelMarkerExpirationDelete, + objectSingleTypesEnd, + ObjectAccessedAll, + ObjectCreatedAll, + ObjectRemovedAll, + ObjectReplicationAll, + ObjectRestoreAll, + ObjectTransitionAll, + ObjectScannerAll, + #[default] + Everything, +} + +impl EventName { + fn expand(&self) -> Vec { + todo!(); + } + + fn mask(&self) -> u64 { + todo!(); + } +} + +impl AsRef for EventName { + fn as_ref(&self) -> &str { + match self { + EventName::BucketCreated => "s3:BucketCreated:*", + EventName::BucketRemoved => "s3:BucketRemoved:*", + EventName::ObjectAccessedAll => "s3:ObjectAccessed:*", + EventName::ObjectAccessedGet => "s3:ObjectAccessed:Get", + EventName::ObjectAccessedGetRetention => "s3:ObjectAccessed:GetRetention", + EventName::ObjectAccessedGetLegalHold => "s3:ObjectAccessed:GetLegalHold", + EventName::ObjectAccessedHead => "s3:ObjectAccessed:Head", + EventName::ObjectAccessedAttributes => "s3:ObjectAccessed:Attributes", + EventName::ObjectCreatedAll => "s3:ObjectCreated:*", + EventName::ObjectCreatedCompleteMultipartUpload => "s3:ObjectCreated:CompleteMultipartUpload", + EventName::ObjectCreatedCopy => "s3:ObjectCreated:Copy", + EventName::ObjectCreatedPost => "s3:ObjectCreated:Post", + EventName::ObjectCreatedPut => "s3:ObjectCreated:Put", + EventName::ObjectCreatedPutTagging => "s3:ObjectCreated:PutTagging", + EventName::ObjectCreatedDeleteTagging => "s3:ObjectCreated:DeleteTagging", + EventName::ObjectCreatedPutRetention => "s3:ObjectCreated:PutRetention", + EventName::ObjectCreatedPutLegalHold => "s3:ObjectCreated:PutLegalHold", + EventName::ObjectRemovedAll => "s3:ObjectRemoved:*", + EventName::ObjectRemovedDelete => "s3:ObjectRemoved:Delete", + EventName::ObjectRemovedDeleteMarkerCreated => "s3:ObjectRemoved:DeleteMarkerCreated", + EventName::ObjectRemovedNoOP => "s3:ObjectRemoved:NoOP", + EventName::ObjectRemovedDeleteAllVersions => "s3:ObjectRemoved:DeleteAllVersions", + EventName::ILMDelMarkerExpirationDelete => "s3:LifecycleDelMarkerExpiration:Delete", + EventName::ObjectReplicationAll => "s3:Replication:*", + EventName::ObjectReplicationFailed => "s3:Replication:OperationFailedReplication", + EventName::ObjectReplicationComplete => "s3:Replication:OperationCompletedReplication", + EventName::ObjectReplicationNotTracked => "s3:Replication:OperationNotTracked", + EventName::ObjectReplicationMissedThreshold => "s3:Replication:OperationMissedThreshold", + EventName::ObjectReplicationReplicatedAfterThreshold => "s3:Replication:OperationReplicatedAfterThreshold", + EventName::ObjectRestoreAll => "s3:ObjectRestore:*", + EventName::ObjectRestorePost => "s3:ObjectRestore:Post", + EventName::ObjectRestoreCompleted => "s3:ObjectRestore:Completed", + EventName::ObjectTransitionAll => "s3:ObjectTransition:*", + EventName::ObjectTransitionFailed => "s3:ObjectTransition:Failed", + EventName::ObjectTransitionComplete => "s3:ObjectTransition:Complete", + EventName::ObjectManyVersions => "s3:Scanner:ManyVersions", + EventName::ObjectLargeVersions => "s3:Scanner:LargeVersions", + EventName::PrefixManyFolders => "s3:Scanner:BigPrefix", + _ => "", + } + } +} + +impl From<&str> for EventName { + fn from(s: &str) -> Self { + match s { + "s3:BucketCreated:*" => EventName::BucketCreated, + "s3:BucketRemoved:*" => EventName::BucketRemoved, + "s3:ObjectAccessed:*" => EventName::ObjectAccessedAll, + "s3:ObjectAccessed:Get" => EventName::ObjectAccessedGet, + "s3:ObjectAccessed:GetRetention" => EventName::ObjectAccessedGetRetention, + "s3:ObjectAccessed:GetLegalHold" => EventName::ObjectAccessedGetLegalHold, + "s3:ObjectAccessed:Head" => EventName::ObjectAccessedHead, + "s3:ObjectAccessed:Attributes" => EventName::ObjectAccessedAttributes, + "s3:ObjectCreated:*" => EventName::ObjectCreatedAll, + "s3:ObjectCreated:CompleteMultipartUpload" => EventName::ObjectCreatedCompleteMultipartUpload, + "s3:ObjectCreated:Copy" => EventName::ObjectCreatedCopy, + "s3:ObjectCreated:Post" => EventName::ObjectCreatedPost, + "s3:ObjectCreated:Put" => EventName::ObjectCreatedPut, + "s3:ObjectCreated:PutRetention" => EventName::ObjectCreatedPutRetention, + "s3:ObjectCreated:PutLegalHold" => EventName::ObjectCreatedPutLegalHold, + "s3:ObjectCreated:PutTagging" => EventName::ObjectCreatedPutTagging, + "s3:ObjectCreated:DeleteTagging" => EventName::ObjectCreatedDeleteTagging, + "s3:ObjectRemoved:*" => EventName::ObjectRemovedAll, + "s3:ObjectRemoved:Delete" => EventName::ObjectRemovedDelete, + "s3:ObjectRemoved:DeleteMarkerCreated" => EventName::ObjectRemovedDeleteMarkerCreated, + "s3:ObjectRemoved:NoOP" => EventName::ObjectRemovedNoOP, + "s3:ObjectRemoved:DeleteAllVersions" => EventName::ObjectRemovedDeleteAllVersions, + "s3:LifecycleDelMarkerExpiration:Delete" => EventName::ILMDelMarkerExpirationDelete, + "s3:Replication:*" => EventName::ObjectReplicationAll, + "s3:Replication:OperationFailedReplication" => EventName::ObjectReplicationFailed, + "s3:Replication:OperationCompletedReplication" => EventName::ObjectReplicationComplete, + "s3:Replication:OperationMissedThreshold" => EventName::ObjectReplicationMissedThreshold, + "s3:Replication:OperationReplicatedAfterThreshold" => EventName::ObjectReplicationReplicatedAfterThreshold, + "s3:Replication:OperationNotTracked" => EventName::ObjectReplicationNotTracked, + "s3:ObjectRestore:*" => EventName::ObjectRestoreAll, + "s3:ObjectRestore:Post" => EventName::ObjectRestorePost, + "s3:ObjectRestore:Completed" => EventName::ObjectRestoreCompleted, + "s3:ObjectTransition:Failed" => EventName::ObjectTransitionFailed, + "s3:ObjectTransition:Complete" => EventName::ObjectTransitionComplete, + "s3:ObjectTransition:*" => EventName::ObjectTransitionAll, + "s3:Scanner:ManyVersions" => EventName::ObjectManyVersions, + "s3:Scanner:LargeVersions" => EventName::ObjectLargeVersions, + "s3:Scanner:BigPrefix" => EventName::PrefixManyFolders, + _ => EventName::Everything, + } + } +} diff --git a/ecstore/src/event/targetid.rs b/ecstore/src/event/targetid.rs new file mode 100644 index 000000000..a2a01289e --- /dev/null +++ b/ecstore/src/event/targetid.rs @@ -0,0 +1,10 @@ +pub struct TargetID { + id: String, + name: String, +} + +impl TargetID { + fn to_string(&self) -> String { + format!("{}:{}", self.id, self.name) + } +} diff --git a/ecstore/src/event/targetlist.rs b/ecstore/src/event/targetlist.rs new file mode 100644 index 000000000..30602c09e --- /dev/null +++ b/ecstore/src/event/targetlist.rs @@ -0,0 +1,31 @@ +use std::sync::atomic::AtomicI64; + +use super::targetid::TargetID; + +#[derive(Default)] +pub struct TargetList { + pub current_send_calls: AtomicI64, + pub total_events: AtomicI64, + pub events_skipped: AtomicI64, + pub events_errors_total: AtomicI64, + //pub targets: HashMap, + //pub queue: AsyncEvent, + //pub targetStats: HashMap, +} + +impl TargetList { + pub fn new() -> TargetList { + TargetList::default() + } +} + +struct TargetStat { + current_send_calls: i64, + total_events: i64, + failed_events: i64, +} + +struct TargetIDResult { + id: TargetID, + err: std::io::Error, +} diff --git a/ecstore/src/event_notification.rs b/ecstore/src/event_notification.rs new file mode 100644 index 000000000..3639f7f49 --- /dev/null +++ b/ecstore/src/event_notification.rs @@ -0,0 +1,59 @@ +use std::collections::HashMap; +use std::sync::Arc; +use tokio::sync::RwLock; + +use crate::bucket::metadata::BucketMetadata; +use crate::event::name::EventName; +use crate::event::targetlist::TargetList; +use crate::store::ECStore; +use crate::store_api::ObjectInfo; + +pub struct EventNotifier { + target_list: TargetList, + //bucket_rules_map: HashMap>, +} + +impl EventNotifier { + pub fn new() -> Arc> { + Arc::new(RwLock::new(Self { + target_list: TargetList::new(), + //bucket_rules_map: HashMap::new(), + })) + } + + fn get_arn_list(&self) -> Vec { + todo!(); + } + + fn set(&self, bucket: &str, meta: BucketMetadata) { + todo!(); + } + + fn init_bucket_targets(&self, api: ECStore) -> Result<(), std::io::Error> { + /*if err := self.target_list.Add(globalNotifyTargetList.Targets()...); err != nil { + return err + } + self.target_list = self.target_list.Init(runtime.GOMAXPROCS(0)) // TODO: make this configurable (y4m4) + nil*/ + todo!(); + } + + fn send(&self, args: EventArgs) { + todo!(); + } +} + +#[derive(Debug, Default)] +pub struct EventArgs { + pub event_name: String, + pub bucket_name: String, + pub object: ObjectInfo, + pub req_params: HashMap, + pub resp_elements: HashMap, + pub host: String, + pub user_agent: String, +} + +impl EventArgs {} + +pub fn send_event(args: EventArgs) {} diff --git a/ecstore/src/global.rs b/ecstore/src/global.rs index 7f8011bd4..dcd260a82 100644 --- a/ecstore/src/global.rs +++ b/ecstore/src/global.rs @@ -1,9 +1,12 @@ use crate::heal::mrf::MRFState; use crate::{ + bucket::lifecycle::bucket_lifecycle_ops::LifecycleSys, disk::DiskStore, endpoints::{EndpointServerPools, PoolEndpoints, SetupType}, + event_notification::EventNotifier, heal::{background_heal_ops::HealRoutine, heal_ops::AllHealState}, store::ECStore, + tier::tier::TierConfigMgr, }; use lazy_static::lazy_static; use policy::auth::Credentials; @@ -20,24 +23,32 @@ pub const DISK_MIN_INODES: u64 = 1000; pub const DISK_FILL_FRACTION: f64 = 0.99; pub const DISK_RESERVE_FRACTION: f64 = 0.15; +pub const DEFAULT_PORT: u16 = 9000; + lazy_static! { - static ref GLOBAL_RUSTFS_PORT: OnceLock = OnceLock::new(); - pub static ref GLOBAL_OBJECT_API: OnceLock> = OnceLock::new(); - pub static ref GLOBAL_LOCAL_DISK: Arc>>> = Arc::new(RwLock::new(Vec::new())); - pub static ref GLOBAL_IsErasure: RwLock = RwLock::new(false); - pub static ref GLOBAL_IsDistErasure: RwLock = RwLock::new(false); - pub static ref GLOBAL_IsErasureSD: RwLock = RwLock::new(false); - pub static ref GLOBAL_LOCAL_DISK_MAP: Arc>>> = Arc::new(RwLock::new(HashMap::new())); - pub static ref GLOBAL_LOCAL_DISK_SET_DRIVES: Arc> = Arc::new(RwLock::new(Vec::new())); - pub static ref GLOBAL_Endpoints: OnceLock = OnceLock::new(); - pub static ref GLOBAL_RootDiskThreshold: RwLock = RwLock::new(0); - pub static ref GLOBAL_BackgroundHealRoutine: Arc = HealRoutine::new(); - pub static ref GLOBAL_BackgroundHealState: Arc = AllHealState::new(false); - pub static ref GLOBAL_ALlHealState: Arc = AllHealState::new(false); - pub static ref GLOBAL_MRFState: Arc = Arc::new(MRFState::new()); - static ref globalDeploymentIDPtr: OnceLock = OnceLock::new(); - pub static ref GLOBAL_BOOT_TIME: OnceCell = OnceCell::new(); -} +static ref GLOBAL_RUSTFS_PORT: OnceLock = OnceLock::new(); +pub static ref GLOBAL_OBJECT_API: OnceLock> = OnceLock::new(); +pub static ref GLOBAL_LOCAL_DISK: Arc>>> = Arc::new(RwLock::new(Vec::new())); +pub static ref GLOBAL_IsErasure: RwLock = RwLock::new(false); +pub static ref GLOBAL_IsDistErasure: RwLock = RwLock::new(false); +pub static ref GLOBAL_IsErasureSD: RwLock = RwLock::new(false); +pub static ref GLOBAL_LOCAL_DISK_MAP: Arc>>> = Arc::new(RwLock::new(HashMap::new())); +pub static ref GLOBAL_LOCAL_DISK_SET_DRIVES: Arc> = Arc::new(RwLock::new(Vec::new())); +pub static ref GLOBAL_Endpoints: OnceLock = OnceLock::new(); +pub static ref GLOBAL_RootDiskThreshold: RwLock = RwLock::new(0); +pub static ref GLOBAL_BackgroundHealRoutine: Arc = HealRoutine::new(); +pub static ref GLOBAL_BackgroundHealState: Arc = AllHealState::new(false); +pub static ref GLOBAL_TierConfigMgr: Arc> = TierConfigMgr::new(); +pub static ref GLOBAL_LifecycleSys: Arc = LifecycleSys::new(); +pub static ref GLOBAL_EventNotifier: Arc> = EventNotifier::new(); +//pub static ref GLOBAL_RemoteTargetTransport +pub static ref GLOBAL_ALlHealState: Arc = AllHealState::new(false); +pub static ref GLOBAL_MRFState: Arc = Arc::new(MRFState::new()); +static ref globalDeploymentIDPtr: OnceLock = OnceLock::new(); +pub static ref GLOBAL_BOOT_TIME: OnceCell = OnceCell::new(); +pub static ref GLOBAL_LocalNodeName: String = "127.0.0.1:9000".to_string(); +pub static ref GLOBAL_LocalNodeNameHex: String = rustfs_utils::crypto::hex(GLOBAL_LocalNodeName.as_bytes()); +pub static ref GLOBAL_NodeNamesHex: HashMap = HashMap::new();} static GLOBAL_ACTIVE_CRED: OnceLock = OnceLock::new(); diff --git a/ecstore/src/heal/data_scanner.rs b/ecstore/src/heal/data_scanner.rs index 78f80c8a1..c83a3331e 100644 --- a/ecstore/src/heal/data_scanner.rs +++ b/ecstore/src/heal/data_scanner.rs @@ -12,13 +12,35 @@ use std::{ time::{Duration, SystemTime}, }; +use common::defer; +use time::{self, OffsetDateTime}; + use super::{ data_scanner_metric::{ScannerMetric, ScannerMetrics, globalScannerMetrics}, data_usage::{DATA_USAGE_BLOOM_NAME_PATH, store_data_usage_in_backend}, data_usage_cache::{DataUsageCache, DataUsageEntry, DataUsageHash}, heal_commands::{HEAL_DEEP_SCAN, HEAL_NORMAL_SCAN, HealScanMode}, }; -use crate::{bucket::metadata_sys, cmd::bucket_replication::queue_replication_heal}; +use crate::bucket::{ + object_lock::objectlock_sys::{BucketObjectLockSys, enforce_retention_for_deletion}, + utils::is_meta_bucketname, +}; +use crate::cmd::bucket_replication::queue_replication_heal; +use crate::event::name::EventName; +use crate::{ + bucket::{ + lifecycle::{ + bucket_lifecycle_audit::{LcAuditEvent, LcEventSrc}, + bucket_lifecycle_ops::{self, GLOBAL_ExpiryState, GLOBAL_TransitionState, LifecycleOps, expire_transitioned_object}, + lifecycle::{self, ExpirationOptions, Lifecycle}, + }, + metadata_sys, + }, + event_notification::{EventArgs, send_event}, + global::GLOBAL_LocalNodeName, + heal::data_scanner, + store_api::{ObjectOptions, ObjectToDelete, StorageAPI}, +}; use crate::{ bucket::{versioning::VersioningApi, versioning_sys::BucketVersioningSys}, cmd::bucket_replication::ReplicationStatusType, @@ -55,8 +77,12 @@ use lazy_static::lazy_static; use rand::Rng; use rmp_serde::{Deserializer, Serializer}; use rustfs_filemeta::{FileInfo, MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams}; +use rustfs_utils::path::encode_dir_object; use rustfs_utils::path::{SLASH_SEPARATOR, path_join, path_to_bucket_object, path_to_bucket_object_with_base_path}; -use s3s::dto::{BucketLifecycleConfiguration, ExpirationStatus, LifecycleRule, ReplicationConfiguration, ReplicationRuleStatus}; +use s3s::dto::{ + BucketLifecycleConfiguration, DefaultRetention, ExpirationStatus, LifecycleRule, ReplicationConfiguration, + ReplicationRuleStatus, VersioningConfiguration, +}; use serde::{Deserialize, Serialize}; use tokio::{ sync::{ @@ -515,8 +541,68 @@ impl ScannerItem { path_join(&[PathBuf::from(self.prefix.clone()), PathBuf::from(self.object_name.clone())]) } - pub async fn apply_versions_actions(&self, fives: &[FileInfo]) -> Result> { - let obj_infos = self.apply_newer_noncurrent_version_limit(fives).await?; + async fn apply_lifecycle(&self, oi: &ObjectInfo) -> (lifecycle::IlmAction, i64) { + let mut size = oi.get_actual_size().expect("err!"); + if self.debug { + info!("apply_lifecycle debug"); + } + if self.lifecycle.is_none() { + return (lifecycle::IlmAction::NoneAction, size as i64); + } + + let version_id = oi.version_id; + + let mut vc = None; + let mut lr = None; + let mut rcfg = None; + if !is_meta_bucketname(&self.bucket) { + vc = Some(BucketVersioningSys::get(&self.bucket).await.unwrap()); + lr = BucketObjectLockSys::get(&self.bucket).await; + rcfg = if let Ok(replication_config) = metadata_sys::get_replication_config(&self.bucket).await { + Some(replication_config) + } else { + None + }; + } + + let lc_evt = eval_action_from_lifecycle(self.lifecycle.as_ref().expect("err"), lr, rcfg, oi).await; + if self.debug { + if !version_id.is_none() { + info!( + "lifecycle: {} (version-id={}), Initial scan: {}", + self.object_path().to_string_lossy().to_string(), + version_id.expect("err"), + lc_evt.action + ); + } else { + info!( + "lifecycle: {} Initial scan: {}", + self.object_path().to_string_lossy().to_string(), + lc_evt.action + ); + } + } + + match lc_evt.action { + lifecycle::IlmAction::DeleteVersionAction + | lifecycle::IlmAction::DeleteAllVersionsAction + | lifecycle::IlmAction::DelMarkerDeleteAllVersionsAction => { + size = 0; + } + lifecycle::IlmAction::DeleteAction => { + if !vc.unwrap().prefix_enabled(&oi.name) { + size = 0 + } + } + _ => (), + } + + apply_lifecycle_action(&lc_evt, &LcEventSrc::Scanner, oi).await; + (lc_evt.action, size as i64) + } + + pub async fn apply_versions_actions(&self, fivs: &[FileInfo]) -> Result> { + let obj_infos = self.apply_newer_noncurrent_version_limit(fivs).await?; if obj_infos.len() >= SCANNER_EXCESS_OBJECT_VERSIONS.load(Ordering::SeqCst) as usize { // todo } @@ -533,16 +619,24 @@ impl ScannerItem { Ok(obj_infos) } - pub async fn apply_newer_noncurrent_version_limit(&self, fives: &[FileInfo]) -> Result> { + pub async fn apply_newer_noncurrent_version_limit(&self, fivs: &[FileInfo]) -> Result> { // let done = ScannerMetrics::time(ScannerMetric::ApplyNonCurrent); + + let lock_enabled = if let Some(rcfg) = BucketObjectLockSys::get(&self.bucket).await { + rcfg.mode.is_some() + } else { + false + }; + let vcfg = BucketVersioningSys::get(&self.bucket).await?; + let versioned = match BucketVersioningSys::get(&self.bucket).await { Ok(vcfg) => vcfg.versioned(self.object_path().to_str().unwrap_or_default()), Err(_) => false, }; - let mut object_infos = Vec::with_capacity(fives.len()); + let mut object_infos = Vec::with_capacity(fivs.len()); if self.lifecycle.is_none() { - for info in fives.iter() { + for info in fivs.iter() { object_infos.push(ObjectInfo::from_file_info( info, &self.bucket, @@ -553,6 +647,72 @@ impl ScannerItem { return Ok(object_infos); } + let event = self + .lifecycle + .as_ref() + .expect("lifecycle err.") + .noncurrent_versions_expiration_limit(&lifecycle::ObjectOpts { + name: self.object_path().to_string_lossy().to_string(), + ..Default::default() + }) + .await; + let lim = event.newer_noncurrent_versions; + if lim == 0 || fivs.len() <= lim + 1 { + for fi in fivs.iter() { + object_infos.push(ObjectInfo::from_file_info( + fi, + &self.bucket, + &self.object_path().to_string_lossy(), + versioned, + )); + } + return Ok(object_infos); + } + + let overflow_versions = &fivs[lim + 1..]; + for fi in fivs[..lim + 1].iter() { + object_infos.push(ObjectInfo::from_file_info( + fi, + &self.bucket, + &self.object_path().to_string_lossy(), + versioned, + )); + } + + let mut to_del = Vec::::with_capacity(overflow_versions.len()); + for fi in overflow_versions.iter() { + let obj = ObjectInfo::from_file_info(fi, &self.bucket, &self.object_path().to_string_lossy(), versioned); + if lock_enabled && enforce_retention_for_deletion(&obj) { + //if enforce_retention_for_deletion(&obj) { + if self.debug { + if obj.version_id.is_some() { + info!("lifecycle: {} v({}) is locked, not deleting\n", obj.name, obj.version_id.expect("err")); + } else { + info!("lifecycle: {} is locked, not deleting\n", obj.name); + } + } + object_infos.push(obj); + continue; + } + + if OffsetDateTime::now_utc().unix_timestamp() + < lifecycle::expected_expiry_time(obj.successor_mod_time.expect("err"), event.noncurrent_days as i32) + .unix_timestamp() + { + object_infos.push(obj); + continue; + } + + to_del.push(ObjectToDelete { + object_name: obj.name, + version_id: obj.version_id, + }); + } + + if to_del.len() > 0 { + let mut expiry_state = GLOBAL_ExpiryState.write().await; + expiry_state.enqueue_by_newer_noncurrent(&self.bucket, to_del, event).await; + } // done().await; Ok(object_infos) @@ -560,7 +720,9 @@ impl ScannerItem { pub async fn apply_actions(&mut self, oi: &ObjectInfo, _size_s: &mut SizeSummary) -> (bool, i64) { let done = ScannerMetrics::time(ScannerMetric::Ilm); - //todo: lifecycle + + let (action, size) = self.apply_lifecycle(oi).await; + info!( "apply_actions {} {} {:?} {:?}", oi.bucket.clone(), @@ -580,6 +742,10 @@ impl ScannerItem { self.heal_replication(&oi, _size_s).await; done(); + if action.delete_all() { + return (true, 0); + } + (false, oi.size) } @@ -756,9 +922,9 @@ impl FolderScanner { let (_, prefix) = path_to_bucket_object_with_base_path(&self.root, &folder.name); // Todo: lifeCycle - let active_life_cycle = if let Some(lc) = self.old_cache.info.life_cycle.as_ref() { + let active_life_cycle = if let Some(lc) = self.old_cache.info.lifecycle.as_ref() { if lc_has_active_rules(lc, &prefix) { - self.old_cache.info.life_cycle.clone() + self.old_cache.info.lifecycle.clone() } else { None } @@ -1400,7 +1566,165 @@ pub async fn scan_data_folder( Ok(s.new_cache) } -// pub fn eval_action_from_lifecycle(lc: &BucketLifecycleConfiguration, lr: &ObjectLockConfiguration, rcfg: &ReplicationConfiguration, obj: &ObjectInfo) +pub async fn eval_action_from_lifecycle( + lc: &BucketLifecycleConfiguration, + lr: Option, + rcfg: Option<(ReplicationConfiguration, OffsetDateTime)>, + oi: &ObjectInfo, +) -> lifecycle::Event { + let event = lc.eval(&oi.to_lifecycle_opts()).await; + //if serverDebugLog { + info!("lifecycle: Secondary scan: {}", event.action); + //} + + let lock_enabled = if let Some(lr) = lr { lr.mode.is_some() } else { false }; + + match event.action { + lifecycle::IlmAction::DeleteAllVersionsAction | lifecycle::IlmAction::DelMarkerDeleteAllVersionsAction => { + if lock_enabled { + return lifecycle::Event::default(); + } + } + lifecycle::IlmAction::DeleteVersionAction | lifecycle::IlmAction::DeleteRestoredVersionAction => { + if oi.version_id.is_none() { + return lifecycle::Event::default(); + } + if lock_enabled && enforce_retention_for_deletion(oi) { + //if serverDebugLog { + if !oi.version_id.is_none() { + info!("lifecycle: {} v({}) is locked, not deleting", oi.name, oi.version_id.expect("err")); + } else { + info!("lifecycle: {} is locked, not deleting", oi.name); + } + //} + return lifecycle::Event::default(); + } + if let Some(rcfg) = rcfg { + if rep_has_active_rules(&rcfg.0, &oi.name, true) { + return lifecycle::Event::default(); + } + } + } + _ => (), + } + + event +} + +async fn apply_transition_rule(event: &lifecycle::Event, src: &LcEventSrc, oi: &ObjectInfo) -> bool { + if oi.delete_marker || oi.is_dir { + return false; + } + GLOBAL_TransitionState.queue_transition_task(oi, event, src).await; + true +} + +pub async fn apply_expiry_on_transitioned_object( + api: Arc, + oi: &ObjectInfo, + lc_event: &lifecycle::Event, + src: &LcEventSrc, +) -> bool { + let time_ilm = ScannerMetrics::time_ilm(lc_event.action.clone()); + if let Err(_err) = expire_transitioned_object(api, oi, lc_event, src).await { + return false; + } + time_ilm(1); + + true +} + +pub async fn apply_expiry_on_non_transitioned_objects( + api: Arc, + oi: &ObjectInfo, + lc_event: &lifecycle::Event, + src: &LcEventSrc, +) -> bool { + let mut opts = ObjectOptions { + expiration: ExpirationOptions { expire: true }, + ..Default::default() + }; + + if lc_event.action.delete_versioned() { + opts.version_id = Some(oi.version_id.expect("err").to_string()); + } + + opts.versioned = BucketVersioningSys::prefix_enabled(&oi.bucket, &oi.name).await; + opts.version_suspended = BucketVersioningSys::prefix_suspended(&oi.bucket, &oi.name).await; + + if lc_event.action.delete_all() { + opts.delete_prefix = true; + opts.delete_prefix_object = true; + } + let dobj: ObjectInfo; + //let err: Error; + + let time_ilm = ScannerMetrics::time_ilm(lc_event.action.clone()); + + let mut dobj = api + .delete_object(&oi.bucket, &encode_dir_object(&oi.name), opts) + .await + .unwrap(); + if dobj.name == "" { + dobj = oi.clone(); + } + + //let tags = LcAuditEvent::new(lc_event.clone(), src.clone()).tags(); + //tags["version-id"] = dobj.version_id; + + let mut event_name = EventName::ObjectRemovedDelete; + if oi.delete_marker { + event_name = EventName::ObjectRemovedDeleteMarkerCreated; + } + match lc_event.action { + lifecycle::IlmAction::DeleteAllVersionsAction => event_name = EventName::ObjectRemovedDeleteAllVersions, + lifecycle::IlmAction::DelMarkerDeleteAllVersionsAction => event_name = EventName::ILMDelMarkerExpirationDelete, + _ => (), + } + send_event(EventArgs { + event_name: event_name.as_ref().to_string(), + bucket_name: dobj.bucket.clone(), + object: dobj, + user_agent: "Internal: [ILM-Expiry]".to_string(), + host: GLOBAL_LocalNodeName.to_string(), + ..Default::default() + }); + + if lc_event.action != lifecycle::IlmAction::NoneAction { + let mut num_versions = 1_u64; + if lc_event.action.delete_all() { + num_versions = oi.num_versions as u64; + } + time_ilm(num_versions); + } + + true +} + +async fn apply_expiry_rule(event: &lifecycle::Event, src: &LcEventSrc, oi: &ObjectInfo) -> bool { + let mut expiry_state = GLOBAL_ExpiryState.write().await; + expiry_state.enqueue_by_days(oi, event, src).await; + true +} + +pub async fn apply_lifecycle_action(event: &lifecycle::Event, src: &LcEventSrc, oi: &ObjectInfo) -> bool { + let mut success = false; + match event.action { + lifecycle::IlmAction::DeleteVersionAction + | lifecycle::IlmAction::DeleteAction + | lifecycle::IlmAction::DeleteRestoredAction + | lifecycle::IlmAction::DeleteRestoredVersionAction + | lifecycle::IlmAction::DeleteAllVersionsAction + | lifecycle::IlmAction::DelMarkerDeleteAllVersionsAction => { + success = apply_expiry_rule(event, src, oi).await; + } + lifecycle::IlmAction::TransitionAction | lifecycle::IlmAction::TransitionVersionAction => { + success = apply_transition_rule(event, src, oi).await; + } + _ => (), + } + success +} #[cfg(test)] mod tests { diff --git a/ecstore/src/heal/data_scanner_metric.rs b/ecstore/src/heal/data_scanner_metric.rs index 6d97106ad..ced1e6b87 100644 --- a/ecstore/src/heal/data_scanner_metric.rs +++ b/ecstore/src/heal/data_scanner_metric.rs @@ -15,14 +15,13 @@ use tokio::sync::{Mutex, RwLock}; use tracing::debug; use super::data_scanner::CurrentScannerCycle; +use crate::bucket::lifecycle::lifecycle; lazy_static! { pub static ref globalScannerMetrics: Arc = Arc::new(ScannerMetrics::new()); } -/// Scanner metric types, matching the Go version exactly -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] -#[repr(u8)] +#[derive(Clone, Debug, PartialEq, PartialOrd)] pub enum ScannerMetric { // START Realtime metrics, that only records // last minute latencies and total operation count. @@ -196,7 +195,8 @@ pub struct ScannerMetrics { // All fields must be accessed atomically and aligned. operations: Vec, latency: Vec, - + actions: Vec, + actions_latency: Vec, // Current paths contains disk -> tracker mappings current_paths: Arc>>>, @@ -215,6 +215,8 @@ impl ScannerMetrics { Self { operations, latency, + actions: (0..ScannerMetric::Last as usize).map(|_| AtomicU64::new(0)).collect(), + actions_latency: vec![LockedLastMinuteLatency::default(); ScannerMetric::LastRealtime as usize], current_paths: Arc::new(RwLock::new(HashMap::new())), cycle_info: Arc::new(RwLock::new(None)), } @@ -222,16 +224,17 @@ impl ScannerMetrics { /// Log scanner action with custom metadata - compatible with existing usage pub fn log(metric: ScannerMetric) -> impl Fn(&HashMap) { + let metric = metric as usize; let start_time = SystemTime::now(); move |_custom: &HashMap| { let duration = SystemTime::now().duration_since(start_time).unwrap_or_default(); // Update operation count - globalScannerMetrics.operations[metric as usize].fetch_add(1, Ordering::Relaxed); + globalScannerMetrics.operations[metric].fetch_add(1, Ordering::Relaxed); // Update latency for realtime metrics (spawn async task for this) - if (metric as usize) < ScannerMetric::LastRealtime as usize { - let metric_index = metric as usize; + if (metric) < ScannerMetric::LastRealtime as usize { + let metric_index = metric; tokio::spawn(async move { globalScannerMetrics.latency[metric_index].add(duration).await; }); @@ -239,23 +242,24 @@ impl ScannerMetrics { // Log trace metrics if metric as u8 > ScannerMetric::StartTrace as u8 { - debug!(metric = metric.as_str(), duration_ms = duration.as_millis(), "Scanner trace metric"); + //debug!(metric = metric.as_str(), duration_ms = duration.as_millis(), "Scanner trace metric"); } } } /// Time scanner action with size - returns function that takes size pub fn time_size(metric: ScannerMetric) -> impl Fn(u64) { + let metric = metric as usize; let start_time = SystemTime::now(); move |size: u64| { let duration = SystemTime::now().duration_since(start_time).unwrap_or_default(); // Update operation count - globalScannerMetrics.operations[metric as usize].fetch_add(1, Ordering::Relaxed); + globalScannerMetrics.operations[metric].fetch_add(1, Ordering::Relaxed); // Update latency for realtime metrics with size (spawn async task) - if (metric as usize) < ScannerMetric::LastRealtime as usize { - let metric_index = metric as usize; + if (metric) < ScannerMetric::LastRealtime as usize { + let metric_index = metric; tokio::spawn(async move { globalScannerMetrics.latency[metric_index].add_size(duration, size).await; }); @@ -265,16 +269,17 @@ impl ScannerMetrics { /// Time a scanner action - returns a closure to call when done pub fn time(metric: ScannerMetric) -> impl Fn() { + let metric = metric as usize; let start_time = SystemTime::now(); move || { let duration = SystemTime::now().duration_since(start_time).unwrap_or_default(); // Update operation count - globalScannerMetrics.operations[metric as usize].fetch_add(1, Ordering::Relaxed); + globalScannerMetrics.operations[metric].fetch_add(1, Ordering::Relaxed); // Update latency for realtime metrics (spawn async task) - if (metric as usize) < ScannerMetric::LastRealtime as usize { - let metric_index = metric as usize; + if (metric) < ScannerMetric::LastRealtime as usize { + let metric_index = metric; tokio::spawn(async move { globalScannerMetrics.latency[metric_index].add(duration).await; }); @@ -284,17 +289,18 @@ impl ScannerMetrics { /// Time N scanner actions - returns function that takes count, then returns completion function pub fn time_n(metric: ScannerMetric) -> Box Box + Send + Sync> { + let metric = metric as usize; let start_time = SystemTime::now(); Box::new(move |count: usize| { Box::new(move || { let duration = SystemTime::now().duration_since(start_time).unwrap_or_default(); // Update operation count - globalScannerMetrics.operations[metric as usize].fetch_add(count as u64, Ordering::Relaxed); + globalScannerMetrics.operations[metric].fetch_add(count as u64, Ordering::Relaxed); // Update latency for realtime metrics (spawn async task) - if (metric as usize) < ScannerMetric::LastRealtime as usize { - let metric_index = metric as usize; + if (metric) < ScannerMetric::LastRealtime as usize { + let metric_index = metric; tokio::spawn(async move { globalScannerMetrics.latency[metric_index].add(duration).await; }); @@ -303,31 +309,51 @@ impl ScannerMetrics { }) } + pub fn time_ilm(a: lifecycle::IlmAction) -> Box Box + Send + Sync> { + let a_clone = a as usize; + if a_clone == lifecycle::IlmAction::NoneAction as usize || a_clone >= lifecycle::IlmAction::ActionCount as usize { + return Box::new(move |_: u64| Box::new(move || {})); + } + let start = SystemTime::now(); + Box::new(move |versions: u64| { + Box::new(move || { + let duration = SystemTime::now().duration_since(start).unwrap_or(Duration::from_secs(0)); + tokio::spawn(async move { + globalScannerMetrics.actions[a_clone].fetch_add(versions, Ordering::Relaxed); + globalScannerMetrics.actions_latency[a_clone].add(duration).await; + }); + }) + }) + } + /// Increment time with specific duration pub async fn inc_time(metric: ScannerMetric, duration: Duration) { + let metric = metric as usize; // Update operation count - globalScannerMetrics.operations[metric as usize].fetch_add(1, Ordering::Relaxed); + globalScannerMetrics.operations[metric].fetch_add(1, Ordering::Relaxed); // Update latency for realtime metrics - if (metric as usize) < ScannerMetric::LastRealtime as usize { - globalScannerMetrics.latency[metric as usize].add(duration).await; + if (metric) < ScannerMetric::LastRealtime as usize { + globalScannerMetrics.latency[metric].add(duration).await; } } /// Get lifetime operation count for a metric pub fn lifetime(&self, metric: ScannerMetric) -> u64 { - if (metric as usize) >= ScannerMetric::Last as usize { + let metric = metric as usize; + if (metric) >= ScannerMetric::Last as usize { return 0; } - self.operations[metric as usize].load(Ordering::Relaxed) + self.operations[metric].load(Ordering::Relaxed) } /// Get last minute statistics for a metric pub async fn last_minute(&self, metric: ScannerMetric) -> AccElem { - if (metric as usize) >= ScannerMetric::LastRealtime as usize { + let metric = metric as usize; + if (metric) >= ScannerMetric::LastRealtime as usize { return AccElem::default(); } - self.latency[metric as usize].total().await + self.latency[metric].total().await } /// Set current cycle information diff --git a/ecstore/src/heal/data_usage_cache.rs b/ecstore/src/heal/data_usage_cache.rs index 2e329f898..a88da15af 100644 --- a/ecstore/src/heal/data_usage_cache.rs +++ b/ecstore/src/heal/data_usage_cache.rs @@ -129,6 +129,59 @@ const OBJECTS_VERSION_COUNT_INTERVALS: [ObjectHistogramInterval; DATA_USAGE_VERS }, ]; +#[derive(Clone, Copy, Default)] +pub struct TierStats { + pub total_size: u64, + pub num_versions: i32, + pub num_objects: i32, +} + +impl TierStats { + pub fn add(&self, u: &TierStats) -> TierStats { + TierStats { + total_size: self.total_size + u.total_size, + num_versions: self.num_versions + u.num_versions, + num_objects: self.num_objects + u.num_objects, + } + } +} + +#[derive(Clone)] +struct AllTierStats { + tiers: HashMap, +} + +impl AllTierStats { + pub fn new() -> Self { + Self { tiers: HashMap::new() } + } + + fn add_sizes(&mut self, tiers: HashMap) { + for (tier, st) in tiers { + self.tiers.insert(tier.clone(), self.tiers[&tier].add(&st)); + } + } + + fn merge(&mut self, other: AllTierStats) { + for (tier, st) in other.tiers { + self.tiers.insert(tier.clone(), self.tiers[&tier].add(&st)); + } + } + + fn populate_stats(&self, stats: &mut HashMap) { + for (tier, st) in &self.tiers { + stats.insert( + tier.clone(), + TierStats { + total_size: st.total_size.clone(), + num_versions: st.num_versions.clone(), + num_objects: st.num_objects.clone(), + }, + ); + } + } +} + // sizeHistogram is a size histogram. #[derive(Clone, Debug, Serialize, Deserialize)] pub struct SizeHistogram(Vec); @@ -346,8 +399,7 @@ pub struct DataUsageCacheInfo { pub last_update: Option, pub skip_healing: bool, #[serde(skip)] - pub life_cycle: Option, - // pub life_cycle: + pub lifecycle: Option, #[serde(skip)] pub updates: Option>, #[serde(skip)] diff --git a/ecstore/src/lib.rs b/ecstore/src/lib.rs index cfd8fae76..bca6aa68e 100644 --- a/ecstore/src/lib.rs +++ b/ecstore/src/lib.rs @@ -28,6 +28,13 @@ mod store_init; pub mod store_list_objects; pub mod store_utils; +pub mod checksum; +pub mod client; +pub mod event; +pub mod event_notification; +pub mod signer; +pub mod tier; + pub use global::new_object_layer_fn; pub use global::set_global_endpoints; pub use global::update_erasure_type; diff --git a/ecstore/src/set_disk.rs b/ecstore/src/set_disk.rs index 30cb40d01..45379adca 100644 --- a/ecstore/src/set_disk.rs +++ b/ecstore/src/set_disk.rs @@ -1,4 +1,6 @@ use crate::bitrot::{create_bitrot_reader, create_bitrot_writer}; +use crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE; +use crate::client::{object_api_utils::extract_etag, transition_api::ReaderImpl}; use crate::disk::error_reduce::{OBJECT_OP_IGNORED_ERRS, reduce_read_quorum_errs, reduce_write_quorum_errs}; use crate::disk::{ self, CHECK_PART_DISK_NOT_FOUND, CHECK_PART_FILE_CORRUPT, CHECK_PART_FILE_NOT_FOUND, CHECK_PART_SUCCESS, @@ -6,12 +8,15 @@ use crate::disk::{ }; use crate::erasure_coding; use crate::erasure_coding::bitrot_verify; +use crate::error::ObjectApiError; use crate::error::{Error, Result}; use crate::global::GLOBAL_MRFState; +use crate::global::{GLOBAL_LocalNodeName, GLOBAL_TierConfigMgr}; use crate::heal::data_usage_cache::DataUsageCache; use crate::heal::heal_ops::{HealEntryFn, HealSequence}; use crate::store_api::ObjectToDelete; use crate::{ + bucket::lifecycle::bucket_lifecycle_ops::{gen_transition_objname, get_transitioned_object_reader, put_restore_opts}, cache_value::metacache_set::{ListPathRawOptions, list_path_raw}, config::{GLOBAL_StorageClass, storageclass}, disk::{ @@ -20,6 +25,8 @@ use crate::{ UpdateMetadataOpts, endpoint::Endpoint, error::DiskError, format::FormatV3, new_disk, }, error::{StorageError, to_object_err}, + event::name::EventName, + event_notification::{EventArgs, send_event}, global::{ GLOBAL_BackgroundHealState, GLOBAL_LOCAL_DISK_MAP, GLOBAL_LOCAL_DISK_SET_DRIVES, get_global_deployment_id, is_dist_erasure, @@ -68,7 +75,8 @@ use rustfs_utils::{ crypto::{base64_decode, base64_encode, hex}, path::{SLASH_SEPARATOR, encode_dir_object, has_suffix, path_join_buf}, }; -use sha2::Sha256; +use s3s::header::X_AMZ_RESTORE; +use sha2::{Digest, Sha256}; use std::hash::Hash; use std::mem; use std::time::SystemTime; @@ -3040,33 +3048,33 @@ impl SetDisks { let task = if !skip_background_task { Some(tokio::spawn(async move { - let last_save = Some(SystemTime::now()); - let mut need_loop = true; - while need_loop { - select! { - _ = ticker.tick() => { - if !cache.info.last_update.eq(&last_save) { + let last_save = Some(SystemTime::now()); + let mut need_loop = true; + while need_loop { + select! { + _ = ticker.tick() => { + if !cache.info.last_update.eq(&last_save) { + let _ = cache.save(DATA_USAGE_CACHE_NAME).await; + let _ = updates.send(cache.clone()).await; + } + } + result = buckets_results_rx.recv() => { + match result { + Some(result) => { + cache.replace(&result.name, &result.parent, result.entry); + cache.info.last_update = Some(SystemTime::now()); + }, + None => { + need_loop = false; + cache.info.next_cycle = want_cycle; + cache.info.last_update = Some(SystemTime::now()); let _ = cache.save(DATA_USAGE_CACHE_NAME).await; let _ = updates.send(cache.clone()).await; } } - result = buckets_results_rx.recv() => { - match result { - Some(result) => { - cache.replace(&result.name, &result.parent, result.entry); - cache.info.last_update = Some(SystemTime::now()); - }, - None => { - need_loop = false; - cache.info.next_cycle = want_cycle; - cache.info.last_update = Some(SystemTime::now()); - let _ = cache.save(DATA_USAGE_CACHE_NAME).await; - let _ = updates.send(cache.clone()).await; - } - } - } } } + } })) } else { None @@ -3175,7 +3183,7 @@ impl SetDisks { info!("ns_scanner start"); let _ = join_all(futures).await; if let Some(task) = task { - let _ = task.await; + let _ = task.await; } info!("ns_scanner completed"); Ok(()) @@ -3702,6 +3710,45 @@ impl SetDisks { Ok(()) } + + pub async fn update_restore_metadata( + &self, + bucket: &str, + object: &str, + obj_info: &ObjectInfo, + opts: &ObjectOptions, + ) -> Result<()> { + let mut oi = obj_info.clone(); + oi.metadata_only = true; + + if let Some(user_defined) = &mut oi.user_defined { + user_defined.remove(X_AMZ_RESTORE.as_str()); + } + + let version_id = oi.version_id.clone().map(|v| v.to_string()); + let obj = self + .copy_object( + bucket, + object, + bucket, + object, + &mut oi, + &ObjectOptions { + version_id: version_id.clone(), + ..Default::default() + }, + &ObjectOptions { + version_id: version_id, + ..Default::default() + }, + ) + .await; + if let Err(err) = obj { + //storagelogif(ctx, fmt.Errorf("Unable to update transition restore metadata for %s/%s(%s): %s", bucket, object, oi.VersionID, err)) + return Err(err); + } + Ok(()) + } } #[async_trait::async_trait] @@ -3997,7 +4044,7 @@ impl ObjectIO for SetDisks { if opts.data_movement { fi.set_data_moved(); - } + } } let (online_disks, _, op_old_dir) = Self::rename_data( @@ -4188,6 +4235,48 @@ impl StorageAPI for SetDisks { src_opts.versioned || src_opts.version_suspended, )) } + #[tracing::instrument(skip(self))] + async fn delete_object_version(&self, bucket: &str, object: &str, fi: &FileInfo, force_del_marker: bool) -> Result<()> { + let disks = self.get_disks(0, 0).await?; + let write_quorum = disks.len() / 2 + 1; + + let mut futures = Vec::with_capacity(disks.len()); + let mut errs = Vec::with_capacity(disks.len()); + + for disk in disks.iter() { + futures.push(async move { + if let Some(disk) = disk { + match disk + .delete_version(&bucket, &object, fi.clone(), force_del_marker, DeleteOptions::default()) + .await + { + Ok(r) => Ok(r), + Err(e) => Err(e), + } + } else { + Err(DiskError::DiskNotFound) + } + }); + } + + let results = join_all(futures).await; + for result in results { + match result { + Ok(_) => { + errs.push(None); + } + Err(e) => { + errs.push(Some(e)); + } + } + } + + if let Some(err) = reduce_write_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, write_quorum) { + return Err(err.into()); + } + Ok(()) + } + #[tracing::instrument(skip(self))] async fn delete_objects( &self, @@ -4381,6 +4470,22 @@ impl StorageAPI for SetDisks { Ok(oi) } + #[tracing::instrument(skip(self))] + async fn add_partial(&self, bucket: &str, object: &str, version_id: &str) -> Result<()> { + GLOBAL_MRFState + .add_partial(PartialOperation { + bucket: bucket.to_string(), + object: object.to_string(), + version_id: Some(version_id.to_string()), + queued: Utc::now(), + set_index: self.set_index, + pool_index: self.pool_index, + ..Default::default() + }) + .await; + Ok(()) + } + #[tracing::instrument(skip(self))] async fn put_object_metadata(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result { // TODO: nslock @@ -4465,6 +4570,218 @@ impl StorageAPI for SetDisks { Ok(oi.user_tags) } + #[tracing::instrument(level = "debug", skip(self))] + async fn transition_object(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> { + let mut tier_config_mgr = GLOBAL_TierConfigMgr.write().await; + let tgt_client = match tier_config_mgr.get_driver(&opts.transition.tier).await { + Ok(client) => client, + Err(err) => { + return Err(Error::other(err.to_string())); + } + }; + + /*if !opts.no_lock { + let lk = self.new_ns_lock(bucket, object); + let lkctx = lk.get_lock(globalDeleteOperationTimeout)?; + //ctx = lkctx.Context() + //defer lk.Unlock(lkctx) + }*/ + + let (mut fi, mut meta_arr, mut online_disks) = self.get_object_fileinfo(&bucket, &object, &opts, true).await?; + /*if err != nil { + return Err(to_object_err(err, vec![bucket, object])); + }*/ + /*if fi.deleted { + if opts.version_id.is_none() { + return Err(to_object_err(DiskError::FileNotFound, vec![bucket, object])); + } + return Err(to_object_err(ERR_METHOD_NOT_ALLOWED, vec![bucket, object])); + }*/ + if !opts.mod_time.expect("err").unix_timestamp() == fi.mod_time.as_ref().expect("err").unix_timestamp() + || !(opts.transition.etag == extract_etag(&fi.metadata)) + { + return Err(to_object_err(Error::from(DiskError::FileNotFound), vec![bucket, object])); + } + if fi.transition_status == TRANSITION_COMPLETE { + return Ok(()); + } + + /*if fi.xlv1 { + if let Err(err) = self.heal_object(bucket, object, "", &HealOpts {no_lock: true, ..Default::default()}) { + return err.expect("err"); + } + (fi, meta_arr, online_disks) = self.get_object_fileinfo(&bucket, &object, &opts, true); + if err != nil { + return to_object_err(err, vec![bucket, object]); + } + }*/ + //let traceFn = GLOBAL_LifecycleSys.trace(fi.to_object_info(bucket, object, opts.Versioned || opts.VersionSuspended)); + + let dest_obj = gen_transition_objname(bucket); + if let Err(err) = dest_obj { + //traceFn(ILMTransition, nil, err) + return Err(to_object_err(err, vec![])); + } + let dest_obj = dest_obj.unwrap(); + + let oi = ObjectInfo::from_file_info(&fi, &bucket, &object, opts.versioned || opts.version_suspended); + + let (pr, mut pw) = tokio::io::duplex(fi.erasure.block_size); + //let h = HeaderMap::new(); + //let reader = ReaderImpl::ObjectBody(GetObjectReader {stream: StreamingBlob::wrap(tokio_util::io::ReaderStream::new(pr)), object_info: oi}); + let reader = ReaderImpl::ObjectBody(GetObjectReader { + stream: Box::new(pr), + object_info: oi, + }); + + let cloned_bucket = bucket.to_string(); + let cloned_object = object.to_string(); + let cloned_fi = fi.clone(); + let set_index = self.set_index; + let pool_index = self.pool_index; + tokio::spawn(async move { + if let Err(e) = Self::get_object_with_fileinfo( + &cloned_bucket, + &cloned_object, + 0, + cloned_fi.size, + &mut pw, + cloned_fi, + meta_arr, + &online_disks, + set_index, + pool_index, + ) + .await + { + error!("get_object_with_fileinfo err {:?}", e); + }; + }); + + let rv = tgt_client + .put_with_meta(&dest_obj, reader, fi.size as i64, { + let mut m = HashMap::::new(); + m.insert("name".to_string(), object.to_string()); + m + }) + .await; + //pr.CloseWithError(err); + if let Err(err) = rv { + //traceFn(ILMTransition, nil, err) + return Err(StorageError::Io(err)); + } + let rv = rv.unwrap(); + fi.transition_status = TRANSITION_COMPLETE.to_string(); + fi.transitioned_objname = dest_obj; + fi.transition_tier = opts.transition.tier.clone(); + fi.transition_version_id = if rv == "" { None } else { Some(Uuid::parse_str(&rv)?) }; + let mut event_name = EventName::ObjectTransitionComplete.as_ref(); + + let disks = self.get_disks(0, 0).await?; + + if let Err(err) = self.delete_object_version(bucket, object, &fi, false).await { + event_name = EventName::ObjectTransitionFailed.as_ref(); + } + + for disk in disks.iter() { + if let Some(disk) = disk { + if disk.is_online().await { + continue; + } + } + self.add_partial(bucket, object, &opts.version_id.as_ref().expect("err")) + .await; + break; + } + + let obj_info = ObjectInfo::from_file_info(&fi, bucket, object, opts.versioned || opts.version_suspended); + send_event(EventArgs { + event_name: event_name.to_string(), + bucket_name: bucket.to_string(), + object: obj_info, + user_agent: "Internal: [ILM-Transition]".to_string(), + host: GLOBAL_LocalNodeName.to_string(), + ..Default::default() + }); + //let tags = opts.lifecycle_audit_event.tags(); + //auditLogLifecycle(ctx, objInfo, ILMTransition, tags, traceFn) + Ok(()) + } + + #[tracing::instrument(level = "debug", skip(self))] + async fn restore_transitioned_object(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> { + let set_restore_header_fn = async move |oi: &mut ObjectInfo, rerr: Option| -> Result<()> { + if rerr.is_none() { + return Ok(()); + } + self.update_restore_metadata(bucket, object, oi, opts).await?; + Err(rerr.unwrap()) + }; + let mut oi = ObjectInfo::default(); + let fi = self.get_object_fileinfo(&bucket, &object, &opts, true).await; + if let Err(err) = fi { + return set_restore_header_fn(&mut oi, Some(to_object_err(err, vec![bucket, object]))).await; + } + let (mut actual_fi, _, _) = fi.unwrap(); + + oi = ObjectInfo::from_file_info(&actual_fi, bucket, object, opts.versioned || opts.version_suspended); + let ropts = put_restore_opts(bucket, object, &opts.transition.restore_request, &oi); + /*if oi.parts.len() == 1 { + let mut rs: HTTPRangeSpec; + let gr = get_transitioned_object_reader(bucket, object, rs, HeaderMap::new(), oi, opts); + //if err != nil { + // return set_restore_header_fn(&mut oi, Some(toObjectErr(err, bucket, object))); + //} + //defer gr.Close() + let hash_reader = HashReader::new(gr, gr.obj_info.size, "", "", gr.obj_info.size); + let p_reader = PutObjReader::new(StreamingBlob::from(Box::pin(hash_reader)), hash_reader.size()); + if let Err(err) = self.put_object(bucket, object, &mut p_reader, &ropts).await { + return set_restore_header_fn(&mut oi, Some(to_object_err(err, vec![bucket, object]))); + } else { + return Ok(()); + } + } + + let res = self.new_multipart_upload(bucket, object, &ropts).await?; + //if err != nil { + // return set_restore_header_fn(&mut oi, err); + //} + + let mut uploaded_parts: Vec = vec![]; + let mut rs: HTTPRangeSpec; + let gr = get_transitioned_object_reader(bucket, object, rs, HeaderMap::new(), oi, opts).await?; + //if err != nil { + // return set_restore_header_fn(&mut oi, err); + //} + + for part_info in oi.parts { + //let hr = HashReader::new(LimitReader(gr, part_info.size), part_info.size, "", "", part_info.size); + let hr = HashReader::new(gr, part_info.size as i64, part_info.size as i64, None, false); + //if err != nil { + // return set_restore_header_fn(&mut oi, err); + //} + let mut p_reader = PutObjReader::new(hr, hr.size()); + let p_info = self.put_object_part(bucket, object, &res.upload_id, part_info.number, &mut p_reader, &ObjectOptions::default()).await?; + //if let Err(err) = p_info { + // return set_restore_header_fn(&mut oi, err); + //} + if p_info.size != part_info.size { + return set_restore_header_fn(&mut oi, Some(Error::from(ObjectApiError::InvalidObjectState(GenericError{bucket: bucket.to_string(), object: object.to_string(), ..Default::default()})))); + } + uploaded_parts.push(CompletePart { + part_num: p_info.part_num, + etag: p_info.etag, + }); + } + if let Err(err) = self.complete_multipart_upload(bucket, object, &res.upload_id, uploaded_parts, &ObjectOptions { + mod_time: oi.mod_time, + ..Default::default() + }).await { + set_restore_header_fn(&mut oi, Some(err)); + }*/ + Ok(()) + } + #[tracing::instrument(level = "debug", skip(self))] async fn put_object_tags(&self, bucket: &str, object: &str, tags: &str, opts: &ObjectOptions) -> Result { let (mut fi, _, disks) = self.get_object_fileinfo(bucket, object, opts, false).await?; @@ -5846,7 +6163,7 @@ mod tests { // Test that all CHECK_PART constants have expected values assert_eq!(CHECK_PART_UNKNOWN, 0); assert_eq!(CHECK_PART_SUCCESS, 1); - assert_eq!(CHECK_PART_FILE_NOT_FOUND, 4); // 实际值是 4,不是 2 + assert_eq!(CHECK_PART_FILE_NOT_FOUND, 4); // 实际值是4,不是2 assert_eq!(CHECK_PART_VOLUME_NOT_FOUND, 3); assert_eq!(CHECK_PART_FILE_CORRUPT, 5); } @@ -6111,7 +6428,7 @@ mod tests { assert_eq!(conv_part_err_to_int(&Some(disk_err)), CHECK_PART_FILE_NOT_FOUND); let other_err = DiskError::other("other error"); - assert_eq!(conv_part_err_to_int(&Some(other_err)), CHECK_PART_UNKNOWN); // other 错误应该返回 UNKNOWN,不是 SUCCESS + assert_eq!(conv_part_err_to_int(&Some(other_err)), CHECK_PART_UNKNOWN); // other错误应该返回UNKNOWN,不是SUCCESS } #[test] diff --git a/ecstore/src/sets.rs b/ecstore/src/sets.rs index 15ec3c14f..676af12a0 100644 --- a/ecstore/src/sets.rs +++ b/ecstore/src/sets.rs @@ -29,6 +29,7 @@ use futures::future::join_all; use http::HeaderMap; use lock::{LockApi, namespace_lock::NsLockMap, new_lock_api}; use madmin::heal_commands::{HealDriveInfo, HealResultItem}; +use rustfs_filemeta::FileInfo; use rustfs_utils::{crc_hash, path::path_join_buf, sip_hash}; use tokio::sync::RwLock; use uuid::Uuid; @@ -459,6 +460,11 @@ impl StorageAPI for Sets { )) } + #[tracing::instrument(skip(self))] + async fn delete_object_version(&self, bucket: &str, object: &str, fi: &FileInfo, force_del_marker: bool) -> Result<()> { + unimplemented!() + } + #[tracing::instrument(skip(self))] async fn delete_object(&self, bucket: &str, object: &str, opts: ObjectOptions) -> Result { if opts.delete_prefix && !opts.delete_prefix_object { @@ -572,6 +578,24 @@ impl StorageAPI for Sets { self.get_disks_by_key(object).new_multipart_upload(bucket, object, opts).await } + #[tracing::instrument(skip(self))] + async fn transition_object(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> { + self.get_disks_by_key(object).transition_object(bucket, object, opts).await + } + + #[tracing::instrument(skip(self))] + async fn add_partial(&self, bucket: &str, object: &str, version_id: &str) -> Result<()> { + self.get_disks_by_key(object).add_partial(bucket, object, version_id).await; + Ok(()) + } + + #[tracing::instrument(skip(self))] + async fn restore_transitioned_object(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> { + self.get_disks_by_key(object) + .restore_transitioned_object(bucket, object, opts) + .await + } + #[tracing::instrument(skip(self))] async fn copy_object_part( &self, diff --git a/ecstore/src/signer/mod.rs b/ecstore/src/signer/mod.rs new file mode 100644 index 000000000..7d87ba047 --- /dev/null +++ b/ecstore/src/signer/mod.rs @@ -0,0 +1,13 @@ +pub mod ordered_qs; +pub mod request_signature_streaming; +pub mod request_signature_streaming_unsigned_trailer; +pub mod request_signature_v2; +pub mod request_signature_v4; +pub mod utils; + +pub use request_signature_streaming::streaming_sign_v4; +pub use request_signature_v2::pre_sign_v2; +pub use request_signature_v2::sign_v2; +pub use request_signature_v4::pre_sign_v4; +pub use request_signature_v4::sign_v4; +pub use request_signature_v4::sign_v4_trailer; diff --git a/ecstore/src/signer/ordered_qs.rs b/ecstore/src/signer/ordered_qs.rs new file mode 100644 index 000000000..7cc97e2b8 --- /dev/null +++ b/ecstore/src/signer/ordered_qs.rs @@ -0,0 +1,109 @@ +//! Ordered query strings + +use crate::signer::utils::stable_sort_by_first; + +/// Immutable query string container +#[derive(Debug, Default, Clone)] +pub struct OrderedQs { + /// Ascending query strings + qs: Vec<(String, String)>, +} + +/// [`OrderedQs`] +#[derive(Debug, thiserror::Error)] +#[error("ParseOrderedQsError: {inner}")] +pub struct ParseOrderedQsError { + /// url decode error + inner: serde_urlencoded::de::Error, +} + +impl OrderedQs { + /// Constructs [`OrderedQs`] from vec + /// + /// + strings must be url-decoded + #[cfg(test)] + #[must_use] + pub fn from_vec_unchecked(mut v: Vec<(String, String)>) -> Self { + stable_sort_by_first(&mut v); + Self { qs: v } + } + + /// Parses [`OrderedQs`] from query + /// + /// # Errors + /// Returns [`ParseOrderedQsError`] if query cannot be decoded + pub fn parse(query: &str) -> Result { + let result = serde_urlencoded::from_str::>(query); + let mut v = result.map_err(|e| ParseOrderedQsError { inner: e })?; + stable_sort_by_first(&mut v); + Ok(Self { qs: v }) + } + + #[must_use] + pub fn has(&self, name: &str) -> bool { + self.qs.binary_search_by_key(&name, |x| x.0.as_str()).is_ok() + } + + /// Gets query values by name. Time `O(logn)` + pub fn get_all(&self, name: &str) -> impl Iterator + use<'_> { + let qs = self.qs.as_slice(); + + let lower_bound = qs.partition_point(|x| x.0.as_str() < name); + let upper_bound = qs.partition_point(|x| x.0.as_str() <= name); + + qs[lower_bound..upper_bound].iter().map(|x| x.1.as_str()) + } + + pub fn get_unique(&self, name: &str) -> Option<&str> { + let qs = self.qs.as_slice(); + let lower_bound = qs.partition_point(|x| x.0.as_str() < name); + + let mut iter = qs[lower_bound..].iter(); + let pair = iter.next()?; + + if let Some(following) = iter.next() { + if following.0 == name { + return None; + } + } + + (pair.0.as_str() == name).then_some(pair.1.as_str()) + } +} + +impl AsRef<[(String, String)]> for OrderedQs { + fn as_ref(&self) -> &[(String, String)] { + self.qs.as_ref() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn tag() { + { + let query = "tagging"; + let qs = OrderedQs::parse(query).unwrap(); + assert_eq!(qs.as_ref(), &[("tagging".to_owned(), String::new())]); + + assert_eq!(qs.get_unique("taggin"), None); + assert_eq!(qs.get_unique("tagging"), Some("")); + assert_eq!(qs.get_unique("taggingg"), None); + } + + { + let query = "tagging&tagging"; + let qs = OrderedQs::parse(query).unwrap(); + assert_eq!( + qs.as_ref(), + &[("tagging".to_owned(), String::new()), ("tagging".to_owned(), String::new())] + ); + + assert_eq!(qs.get_unique("taggin"), None); + assert_eq!(qs.get_unique("tagging"), None); + assert_eq!(qs.get_unique("taggingg"), None); + } + } +} diff --git a/ecstore/src/signer/request_signature_streaming.rs b/ecstore/src/signer/request_signature_streaming.rs new file mode 100644 index 000000000..90a4fe66e --- /dev/null +++ b/ecstore/src/signer/request_signature_streaming.rs @@ -0,0 +1,80 @@ +use bytes::{Bytes, BytesMut}; +use futures::prelude::*; +use futures::task; +use http::HeaderMap; +use http::Uri; +use http::header::TRAILER; +use http::request::{self, Request}; +use hyper::Method; +use lazy_static::lazy_static; +use std::collections::HashMap; +use std::fmt::Write; +use std::pin::Pin; +use std::sync::Mutex; +use stdx::str::StrExt; +use time::{OffsetDateTime, format_description, macros::datetime, macros::format_description}; +use tracing::{debug, error, info, warn}; + +use super::request_signature_v4::{SERVICE_TYPE_S3, get_scope, get_signature, get_signing_key}; +use crate::client::constants::UNSIGNED_PAYLOAD; +use rustfs_utils::{ + crypto::{hex, hex_sha256, hex_sha256_chunk, hmac_sha256}, + hash::EMPTY_STRING_SHA256_HASH, +}; + +const STREAMING_SIGN_ALGORITHM: &str = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD"; +const STREAMING_SIGN_TRAILER_ALGORITHM: &str = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER"; +const STREAMING_PAYLOAD_HDR: &str = "AWS4-HMAC-SHA256-PAYLOAD"; +const STREAMING_TRAILER_HDR: &str = "AWS4-HMAC-SHA256-TRAILER"; +const PAYLOAD_CHUNK_SIZE: i64 = 64 * 1024; +const CHUNK_SIGCONST_LEN: i64 = 17; +const SIGNATURESTR_LEN: i64 = 64; +const CRLF_LEN: i64 = 2; +const TRAILER_KV_SEPARATOR: &str = ":"; +const TRAILER_SIGNATURE: &str = "x-amz-trailer-signature"; + +lazy_static! { + static ref ignored_streaming_headers: HashMap = { + let mut m = >::new(); + m.insert("authorization".to_string(), true); + m.insert("user-agent".to_string(), true); + m.insert("content-type".to_string(), true); + m + }; +} + +fn build_chunk_string_to_sign(t: OffsetDateTime, region: &str, previous_sig: &str, chunk_check_sum: &str) -> String { + let mut string_to_sign_parts = >::new(); + string_to_sign_parts.push(STREAMING_PAYLOAD_HDR.to_string()); + let format = format_description!("[year][month][day]T[hour][minute][second]Z"); + string_to_sign_parts.push(t.format(&format).unwrap()); + string_to_sign_parts.push(get_scope(region, t, SERVICE_TYPE_S3)); + string_to_sign_parts.push(previous_sig.to_string()); + string_to_sign_parts.push(EMPTY_STRING_SHA256_HASH.to_string()); + string_to_sign_parts.push(chunk_check_sum.to_string()); + string_to_sign_parts.join("\n") +} + +fn build_chunk_signature( + chunk_check_sum: &str, + req_time: OffsetDateTime, + region: &str, + previous_signature: &str, + secret_access_key: &str, +) -> String { + let chunk_string_to_sign = build_chunk_string_to_sign(req_time, region, previous_signature, chunk_check_sum); + let signing_key = get_signing_key(secret_access_key, region, req_time, SERVICE_TYPE_S3); + get_signature(signing_key, &chunk_string_to_sign) +} + +pub fn streaming_sign_v4( + req: request::Builder, + access_key_id: &str, + secret_access_key: &str, + session_token: &str, + region: &str, + data_len: i64, + req_time: OffsetDateTime, /*, sh256: md5simd.Hasher*/ +) -> request::Builder { + todo!(); +} diff --git a/ecstore/src/signer/request_signature_streaming_unsigned_trailer.rs b/ecstore/src/signer/request_signature_streaming_unsigned_trailer.rs new file mode 100644 index 000000000..6a635343e --- /dev/null +++ b/ecstore/src/signer/request_signature_streaming_unsigned_trailer.rs @@ -0,0 +1,11 @@ +use http::request; +use time::OffsetDateTime; + +pub fn streaming_unsigned_v4( + mut req: request::Builder, + session_token: &str, + data_len: i64, + req_time: OffsetDateTime, +) -> request::Builder { + todo!(); +} diff --git a/ecstore/src/signer/request_signature_v2.rs b/ecstore/src/signer/request_signature_v2.rs new file mode 100644 index 000000000..12aebc1de --- /dev/null +++ b/ecstore/src/signer/request_signature_v2.rs @@ -0,0 +1,231 @@ +use bytes::{Bytes, BytesMut}; +use http::request; +use hyper::Uri; +use std::collections::HashMap; +use std::fmt::Write; +use time::{OffsetDateTime, format_description, macros::format_description}; + +use rustfs_utils::crypto::{base64_encode, hex, hmac_sha1}; + +use super::utils::get_host_addr; + +const SIGN_V4_ALGORITHM: &str = "AWS4-HMAC-SHA256"; +const SIGN_V2_ALGORITHM: &str = "AWS"; + +fn encode_url2path(req: &request::Builder, virtual_host: bool) -> String { + let mut path = "".to_string(); + + //path = serde_urlencoded::to_string(req.uri_ref().unwrap().path().unwrap()).unwrap(); + path = req.uri_ref().unwrap().path().to_string(); + path +} + +pub fn pre_sign_v2( + mut req: request::Builder, + access_key_id: &str, + secret_access_key: &str, + expires: i64, + virtual_host: bool, +) -> request::Builder { + if access_key_id == "" || secret_access_key == "" { + return req; + } + + let d = OffsetDateTime::now_utc(); + let d = d.replace_time(time::Time::from_hms(0, 0, 0).unwrap()); + let epoch_expires = d.unix_timestamp() + expires; + + let mut headers = req.headers_mut().expect("err"); + let expires_str = headers.get("Expires"); + if expires_str.is_none() { + headers.insert("Expires", format!("{:010}", epoch_expires).parse().unwrap()); + } + + let string_to_sign = pre_string_to_sign_v2(&req, virtual_host); + let signature = hex(hmac_sha1(secret_access_key, string_to_sign)); + + let result = serde_urlencoded::from_str::>(req.uri_ref().unwrap().query().unwrap()); + let mut query = result.unwrap_or_default(); + if get_host_addr(&req).contains(".storage.googleapis.com") { + query.insert("GoogleAccessId".to_string(), access_key_id.to_string()); + } else { + query.insert("AWSAccessKeyId".to_string(), access_key_id.to_string()); + } + + query.insert("Expires".to_string(), format!("{:010}", epoch_expires)); + + let uri = req.uri_ref().unwrap().clone(); + let mut parts = req.uri_ref().unwrap().clone().into_parts(); + parts.path_and_query = Some( + format!("{}?{}&Signature={}", uri.path(), serde_urlencoded::to_string(&query).unwrap(), signature) + .parse() + .unwrap(), + ); + let req = req.uri(Uri::from_parts(parts).unwrap()); + + req +} + +fn post_pre_sign_signature_v2(policy_base64: &str, secret_access_key: &str) -> String { + let signature = hex(hmac_sha1(secret_access_key, policy_base64)); + signature +} + +pub fn sign_v2( + mut req: request::Builder, + content_len: i64, + access_key_id: &str, + secret_access_key: &str, + virtual_host: bool, +) -> request::Builder { + if access_key_id == "" || secret_access_key == "" { + return req; + } + + let d = OffsetDateTime::now_utc(); + let d2 = d.replace_time(time::Time::from_hms(0, 0, 0).unwrap()); + + let string_to_sign = string_to_sign_v2(&req, virtual_host); + let mut headers = req.headers_mut().expect("err"); + + let date = headers.get("Date").unwrap(); + if date.to_str().unwrap() == "" { + headers.insert( + "Date", + d2.format(&format_description::well_known::Rfc2822) + .unwrap() + .to_string() + .parse() + .unwrap(), + ); + } + + let mut auth_header = format!("{} {}:", SIGN_V2_ALGORITHM, access_key_id); + let auth_header = format!("{}{}", auth_header, base64_encode(&hmac_sha1(secret_access_key, string_to_sign))); + + headers.insert("Authorization", auth_header.parse().unwrap()); + + req +} + +fn pre_string_to_sign_v2(req: &request::Builder, virtual_host: bool) -> String { + let mut buf = BytesMut::new(); + write_pre_sign_v2_headers(&mut buf, &req); + write_canonicalized_headers(&mut buf, &req); + write_canonicalized_resource(&mut buf, &req, virtual_host); + String::from_utf8(buf.to_vec()).unwrap() +} + +fn write_pre_sign_v2_headers(buf: &mut BytesMut, req: &request::Builder) { + buf.write_str(req.method_ref().unwrap().as_str()); + buf.write_char('\n'); + buf.write_str(req.headers_ref().unwrap().get("Content-Md5").unwrap().to_str().unwrap()); + buf.write_char('\n'); + buf.write_str(req.headers_ref().unwrap().get("Content-Type").unwrap().to_str().unwrap()); + buf.write_char('\n'); + buf.write_str(req.headers_ref().unwrap().get("Expires").unwrap().to_str().unwrap()); + buf.write_char('\n'); +} + +fn string_to_sign_v2(req: &request::Builder, virtual_host: bool) -> String { + let mut buf = BytesMut::new(); + write_sign_v2_headers(&mut buf, &req); + write_canonicalized_headers(&mut buf, &req); + write_canonicalized_resource(&mut buf, &req, virtual_host); + String::from_utf8(buf.to_vec()).unwrap() +} + +fn write_sign_v2_headers(buf: &mut BytesMut, req: &request::Builder) { + buf.write_str(req.method_ref().unwrap().as_str()); + buf.write_char('\n'); + buf.write_str(req.headers_ref().unwrap().get("Content-Md5").unwrap().to_str().unwrap()); + buf.write_char('\n'); + buf.write_str(req.headers_ref().unwrap().get("Content-Type").unwrap().to_str().unwrap()); + buf.write_char('\n'); + buf.write_str(req.headers_ref().unwrap().get("Date").unwrap().to_str().unwrap()); + buf.write_char('\n'); +} + +fn write_canonicalized_headers(buf: &mut BytesMut, req: &request::Builder) { + let mut proto_headers = Vec::::new(); + let mut vals = HashMap::>::new(); + for k in req.headers_ref().expect("err").keys() { + let lk = k.as_str().to_lowercase(); + if lk.starts_with("x-amz") { + proto_headers.push(lk.clone()); + let vv = req + .headers_ref() + .expect("err") + .get_all(k) + .iter() + .map(|e| e.to_str().unwrap().to_string()) + .collect(); + vals.insert(lk, vv); + } + } + proto_headers.sort(); + for k in proto_headers { + buf.write_str(&k); + buf.write_char(':'); + for (idx, v) in vals[&k].iter().enumerate() { + if idx > 0 { + buf.write_char(','); + } + buf.write_str(v); + } + buf.write_char('\n'); + } +} + +const INCLUDED_QUERY: &[&str] = &[ + "acl", + "delete", + "lifecycle", + "location", + "logging", + "notification", + "partNumber", + "policy", + "requestPayment", + "response-cache-control", + "response-content-disposition", + "response-content-encoding", + "response-content-language", + "response-content-type", + "response-expires", + "uploadId", + "uploads", + "versionId", + "versioning", + "versions", + "website", +]; + +fn write_canonicalized_resource(buf: &mut BytesMut, req: &request::Builder, virtual_host: bool) { + let request_url = req.uri_ref().unwrap(); + buf.write_str(&encode_url2path(req, virtual_host)); + if request_url.query().unwrap() != "" { + let mut n: i64 = 0; + let result = serde_urlencoded::from_str::>>(req.uri_ref().unwrap().query().unwrap()); + let mut vals = result.unwrap_or_default(); + for resource in INCLUDED_QUERY { + let vv = &vals[*resource]; + if vv.len() > 0 { + n += 1; + match n { + 1 => { + buf.write_char('?'); + } + _ => { + buf.write_char('&'); + buf.write_str(resource); + if vv[0].len() > 0 { + buf.write_char('='); + buf.write_str(&vv[0]); + } + } + } + } + } + } +} diff --git a/ecstore/src/signer/request_signature_v4.rs b/ecstore/src/signer/request_signature_v4.rs new file mode 100644 index 000000000..6bf91586b --- /dev/null +++ b/ecstore/src/signer/request_signature_v4.rs @@ -0,0 +1,716 @@ +use bytes::{Bytes, BytesMut}; +use http::HeaderMap; +use http::Uri; +use http::header::TRAILER; +use http::request::{self, Request}; +use lazy_static::lazy_static; +use std::collections::HashMap; +use std::fmt::Write; +use time::{OffsetDateTime, format_description, macros::datetime, macros::format_description}; +use tracing::{debug, error, info, warn}; + +use super::ordered_qs::OrderedQs; +use super::request_signature_streaming_unsigned_trailer::streaming_unsigned_v4; +use super::utils::stable_sort_by_first; +use super::utils::{get_host_addr, sign_v4_trim_all}; +use crate::client::constants::UNSIGNED_PAYLOAD; +use rustfs_utils::crypto::{hex, hex_sha256, hmac_sha256}; +use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH; + +pub const SIGN_V4_ALGORITHM: &str = "AWS4-HMAC-SHA256"; +pub const SERVICE_TYPE_S3: &str = "s3"; +pub const SERVICE_TYPE_STS: &str = "sts"; + +lazy_static! { + static ref v4_ignored_headers: HashMap = { + let mut m = >::new(); + m.insert("accept-encoding".to_string(), true); + m.insert("authorization".to_string(), true); + m.insert("user-agent".to_string(), true); + m + }; +} + +pub fn get_signing_key(secret: &str, loc: &str, t: OffsetDateTime, service_type: &str) -> [u8; 32] { + let mut s = "AWS4".to_string(); + s.push_str(secret); + let format = format_description!("[year][month][day]"); + let date = hmac_sha256(s.into_bytes(), t.format(&format).unwrap().into_bytes()); + let location = hmac_sha256(date, loc); + let service = hmac_sha256(location, service_type); + let signing_key = hmac_sha256(service, "aws4_request"); + signing_key +} + +pub fn get_signature(signing_key: [u8; 32], string_to_sign: &str) -> String { + hex(hmac_sha256(signing_key, string_to_sign)) +} + +pub fn get_scope(location: &str, t: OffsetDateTime, service_type: &str) -> String { + let format = format_description!("[year][month][day]"); + let mut ans = String::from(""); + ans.push_str(&t.format(&format).unwrap().to_string()); + ans.push('/'); + ans.push_str(location); // TODO: use a `Region` type + ans.push('/'); + ans.push_str(service_type); + ans.push_str("/aws4_request"); + ans +} + +fn get_credential(access_key_id: &str, location: &str, t: OffsetDateTime, service_type: &str) -> String { + let scope = get_scope(location, t, service_type); + let mut s = access_key_id.to_string(); + s.push_str("/"); + s.push_str(&scope); + s +} + +fn get_hashed_payload(req: &request::Builder) -> String { + let headers = req.headers_ref().unwrap(); + let mut hashed_payload = ""; + if let Some(payload) = headers.get("X-Amz-Content-Sha256") { + hashed_payload = payload.to_str().unwrap(); + } + if hashed_payload == "" { + hashed_payload = UNSIGNED_PAYLOAD; + } + hashed_payload.to_string() +} + +fn get_canonical_headers(req: &request::Builder, ignored_headers: &HashMap) -> String { + let mut headers = Vec::::new(); + let mut vals = HashMap::>::new(); + for k in req.headers_ref().expect("err").keys() { + if ignored_headers.get(&k.to_string()).is_some() { + continue; + } + headers.push(k.as_str().to_lowercase()); + let vv = req + .headers_ref() + .expect("err") + .get_all(k) + .iter() + .map(|e| e.to_str().unwrap().to_string()) + .collect(); + vals.insert(k.as_str().to_lowercase(), vv); + } + if !header_exists("host", &headers) { + headers.push("host".to_string()); + } + headers.sort(); + + debug!("get_canonical_headers vals: {:?}", vals); + debug!("get_canonical_headers headers: {:?}", headers); + + let mut buf = BytesMut::new(); + for k in headers { + buf.write_str(&k); + buf.write_char(':'); + let k: &str = &k; + match k { + "host" => { + buf.write_str(&get_host_addr(&req)); + buf.write_char('\n'); + } + _ => { + for (idx, v) in vals[k].iter().enumerate() { + if idx > 0 { + buf.write_char(','); + } + buf.write_str(&sign_v4_trim_all(v)); + } + buf.write_char('\n'); + } + } + } + String::from_utf8(buf.to_vec()).unwrap() +} + +fn header_exists(key: &str, headers: &[String]) -> bool { + for k in headers { + if k == key { + return true; + } + } + false +} + +fn get_signed_headers(req: &request::Builder, ignored_headers: &HashMap) -> String { + let mut headers = Vec::::new(); + let headers_ref = req.headers_ref().expect("err"); + debug!("get_signed_headers headers: {:?}", headers_ref); + for (k, _) in headers_ref { + if ignored_headers.get(&k.to_string()).is_some() { + continue; + } + headers.push(k.as_str().to_lowercase()); + } + if !header_exists("host", &headers) { + headers.push("host".to_string()); + } + headers.sort(); + headers.join(";") +} + +fn get_canonical_request(req: &request::Builder, ignored_headers: &HashMap, hashed_payload: &str) -> String { + let mut canonical_query_string = "".to_string(); + if let Some(q) = req.uri_ref().unwrap().query() { + let mut query = q.split('&').map(|h| h.to_string()).collect::>(); + query.sort(); + canonical_query_string = query.join("&"); + canonical_query_string = canonical_query_string.replace("+", "%20"); + } + + let mut canonical_request = >::new(); + canonical_request.push(req.method_ref().unwrap().to_string()); + canonical_request.push(req.uri_ref().unwrap().path().to_string()); + canonical_request.push(canonical_query_string); + canonical_request.push(get_canonical_headers(&req, ignored_headers)); + canonical_request.push(get_signed_headers(&req, ignored_headers)); + canonical_request.push(hashed_payload.to_string()); + canonical_request.join("\n") +} + +fn get_string_to_sign_v4(t: OffsetDateTime, location: &str, canonical_request: &str, service_type: &str) -> String { + let mut string_to_sign = SIGN_V4_ALGORITHM.to_string(); + string_to_sign.push_str("\n"); + let format = format_description!("[year][month][day]T[hour][minute][second]Z"); + string_to_sign.push_str(&t.format(&format).unwrap()); + string_to_sign.push_str("\n"); + string_to_sign.push_str(&get_scope(location, t, service_type)); + string_to_sign.push_str("\n"); + string_to_sign.push_str(&hex_sha256(canonical_request.as_bytes(), |s| s.to_string())); + string_to_sign +} + +pub fn pre_sign_v4( + req: request::Builder, + access_key_id: &str, + secret_access_key: &str, + session_token: &str, + location: &str, + expires: i64, + t: OffsetDateTime, +) -> request::Builder { + if access_key_id == "" || secret_access_key == "" { + return req; + } + + //let t = OffsetDateTime::now_utc(); + //let date = AmzDate::parse(timestamp).unwrap(); + let t2 = t.replace_time(time::Time::from_hms(0, 0, 0).unwrap()); + + //let credential = get_scope(location, t, SERVICE_TYPE_S3); + let credential = get_credential(access_key_id, location, t, SERVICE_TYPE_S3); + let signed_headers = get_signed_headers(&req, &v4_ignored_headers); + + let mut query = >::new(); + if let Some(q) = req.uri_ref().unwrap().query() { + let result = serde_urlencoded::from_str::>(q); + query = result.unwrap_or_default(); + } + query.push(("X-Amz-Algorithm".to_string(), SIGN_V4_ALGORITHM.to_string())); + let format = format_description!("[year][month][day]T[hour][minute][second]Z"); + query.push(("X-Amz-Date".to_string(), t.format(&format).unwrap().to_string())); + query.push(("X-Amz-Expires".to_string(), format!("{:010}", expires))); + query.push(("X-Amz-SignedHeaders".to_string(), signed_headers)); + query.push(("X-Amz-Credential".to_string(), credential)); + if session_token != "" { + query.push(("X-Amz-Security-Token".to_string(), session_token.to_string())); + } + + let uri = req.uri_ref().unwrap().clone(); + let mut parts = req.uri_ref().unwrap().clone().into_parts(); + parts.path_and_query = Some( + format!("{}?{}", uri.path(), serde_urlencoded::to_string(&query).unwrap()) + .parse() + .unwrap(), + ); + let req = req.uri(Uri::from_parts(parts).unwrap()); + + let canonical_request = get_canonical_request(&req, &v4_ignored_headers, &get_hashed_payload(&req)); + let string_to_sign = get_string_to_sign_v4(t, location, &canonical_request, SERVICE_TYPE_S3); + //println!("canonical_request: \n{}\n", canonical_request); + //println!("string_to_sign: \n{}\n", string_to_sign); + let signing_key = get_signing_key(secret_access_key, location, t, SERVICE_TYPE_S3); + let signature = get_signature(signing_key, &string_to_sign); + + let uri = req.uri_ref().unwrap().clone(); + let mut parts = req.uri_ref().unwrap().clone().into_parts(); + parts.path_and_query = Some( + format!( + "{}?{}&X-Amz-Signature={}", + uri.path(), + serde_urlencoded::to_string(&query).unwrap(), + signature + ) + .parse() + .unwrap(), + ); + let req = req.uri(Uri::from_parts(parts).unwrap()); + + req +} + +fn post_pre_sign_signature_v4(policy_base64: &str, t: OffsetDateTime, secret_access_key: &str, location: &str) -> String { + let signing_key = get_signing_key(secret_access_key, location, t, SERVICE_TYPE_S3); + let signature = get_signature(signing_key, policy_base64); + signature +} + +fn sign_v4_sts(mut req: request::Builder, access_key_id: &str, secret_access_key: &str, location: &str) -> request::Builder { + sign_v4_inner(req, 0, access_key_id, secret_access_key, "", location, SERVICE_TYPE_STS, HeaderMap::new()) +} + +fn sign_v4_inner( + mut req: request::Builder, + content_len: i64, + access_key_id: &str, + secret_access_key: &str, + session_token: &str, + location: &str, + service_type: &str, + trailer: HeaderMap, +) -> request::Builder { + if access_key_id == "" || secret_access_key == "" { + return req; + } + + let t = OffsetDateTime::now_utc(); + let t2 = t.replace_time(time::Time::from_hms(0, 0, 0).unwrap()); + + let mut headers = req.headers_mut().expect("err"); + let format = format_description!("[year][month][day]T[hour][minute][second]Z"); + headers.insert("X-Amz-Date", t.format(&format).unwrap().to_string().parse().unwrap()); + + if session_token != "" { + headers.insert("X-Amz-Security-Token", session_token.parse().unwrap()); + } + + if trailer.len() > 0 { + for (k, _) in &trailer { + headers.append("X-Amz-Trailer", k.as_str().to_lowercase().parse().unwrap()); + } + + headers.insert("Content-Encoding", "aws-chunked".parse().unwrap()); + headers.insert("x-amz-decoded-content-length", format!("{:010}", content_len).parse().unwrap()); + } + + if service_type == SERVICE_TYPE_STS { + headers.remove("X-Amz-Content-Sha256"); + } + + let hashed_payload = get_hashed_payload(&req); + let canonical_request = get_canonical_request(&req, &v4_ignored_headers, &hashed_payload); + let string_to_sign = get_string_to_sign_v4(t, location, &canonical_request, service_type); + let signing_key = get_signing_key(secret_access_key, location, t, service_type); + let credential = get_credential(access_key_id, location, t2, service_type); + let signed_headers = get_signed_headers(&req, &v4_ignored_headers); + let signature = get_signature(signing_key, &string_to_sign); + //debug!("\n\ncanonical_request: \n{}\nstring_to_sign: \n{}\nsignature: \n{}\n\n", &canonical_request, &string_to_sign, &signature); + + let mut headers = req.headers_mut().expect("err"); + + let auth = format!( + "{} Credential={}, SignedHeaders={}, Signature={}", + SIGN_V4_ALGORITHM, credential, signed_headers, signature + ); + headers.insert("Authorization", auth.parse().unwrap()); + + if trailer.len() > 0 { + //req.Trailer = trailer; + for (_, v) in &trailer { + headers.append(http::header::TRAILER, v.clone()); + } + return streaming_unsigned_v4(req, &session_token, content_len, t); + } + req +} + +fn unsigned_trailer(mut req: request::Builder, content_len: i64, trailer: HeaderMap) { + if trailer.len() > 0 { + return; + } + let t = OffsetDateTime::now_utc(); + let t = t.replace_time(time::Time::from_hms(0, 0, 0).unwrap()); + + let mut headers = req.headers_mut().expect("err"); + let format = format_description!("[year][month][day]T[hour][minute][second]Z"); + headers.insert("X-Amz-Date", t.format(&format).unwrap().to_string().parse().unwrap()); + + for (k, _) in &trailer { + headers.append("X-Amz-Trailer", k.as_str().to_lowercase().parse().unwrap()); + } + + headers.insert("Content-Encoding", "aws-chunked".parse().unwrap()); + headers.insert("x-amz-decoded-content-length", format!("{:010}", content_len).parse().unwrap()); + + if trailer.len() > 0 { + for (_, v) in &trailer { + headers.append(http::header::TRAILER, v.clone()); + } + } + streaming_unsigned_v4(req, "", content_len, t); +} + +pub fn sign_v4( + mut req: request::Builder, + content_len: i64, + access_key_id: &str, + secret_access_key: &str, + session_token: &str, + location: &str, +) -> request::Builder { + sign_v4_inner( + req, + content_len, + access_key_id, + secret_access_key, + session_token, + location, + SERVICE_TYPE_S3, + HeaderMap::new(), + ) +} + +pub fn sign_v4_trailer( + req: request::Builder, + access_key_id: &str, + secret_access_key: &str, + session_token: &str, + location: &str, + trailer: HeaderMap, +) -> request::Builder { + sign_v4_inner( + req, + 0, + access_key_id, + secret_access_key, + session_token, + location, + SERVICE_TYPE_S3, + trailer, + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn example_list_objects() { + // let access_key_id = "AKIAIOSFODNN7EXAMPLE"; + let secret_access_key = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"; + let timestamp = "20130524T000000Z"; + let t = datetime!(2013-05-24 0:00 UTC); + // let bucket = "examplebucket"; + let region = "us-east-1"; + let service = "s3"; + let path = "/"; + + let mut req = Request::builder() + .method(http::Method::GET) + .uri("http://examplebucket.s3.amazonaws.com/?"); + let mut headers = req.headers_mut().expect("err"); + headers.insert("host", "examplebucket.s3.amazonaws.com".parse().unwrap()); + headers.insert( + "x-amz-content-sha256", + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + .parse() + .unwrap(), + ); + headers.insert("x-amz-date", timestamp.parse().unwrap()); + + let mut query = >::new(); + query.push(("max-keys".to_string(), "2".to_string())); + query.push(("prefix".to_string(), "J".to_string())); + let uri = req.uri_ref().unwrap().clone(); + let mut parts = req.uri_ref().unwrap().clone().into_parts(); + parts.path_and_query = Some( + format!("{}?{}", uri.path(), serde_urlencoded::to_string(&query).unwrap()) + .parse() + .unwrap(), + ); + let req = req.uri(Uri::from_parts(parts).unwrap()); + + let canonical_request = get_canonical_request(&req, &v4_ignored_headers, &get_hashed_payload(&req)); + assert_eq!( + canonical_request, + concat!( + "GET\n", + "/\n", + "max-keys=2&prefix=J\n", + "host:examplebucket.s3.amazonaws.com\n", + "x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\n", + "x-amz-date:", + "20130524T000000Z", + "\n", + "\n", + "host;x-amz-content-sha256;x-amz-date\n", + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + ) + ); + + let string_to_sign = get_string_to_sign_v4(t, region, &canonical_request, service); + assert_eq!( + string_to_sign, + concat!( + "AWS4-HMAC-SHA256\n", + "20130524T000000Z", + "\n", + "20130524/us-east-1/s3/aws4_request\n", + "df57d21db20da04d7fa30298dd4488ba3a2b47ca3a489c74750e0f1e7df1b9b7", + ) + ); + + let signing_key = get_signing_key(secret_access_key, region, t, service); + let signature = get_signature(signing_key, &string_to_sign); + + assert_eq!(signature, "34b48302e7b5fa45bde8084f4b7868a86f0a534bc59db6670ed5711ef69dc6f7"); + } + + #[test] + fn example_signature() { + // let access_key_id = "rustfsadmin"; + let secret_access_key = "rustfsadmin"; + let timestamp = "20250505T011054Z"; + let t = datetime!(2025-05-05 01:10:54 UTC); + // let bucket = "mblock2"; + let region = "us-east-1"; + let service = "s3"; + let path = "/mblock2/"; + + let mut req = Request::builder() + .method(http::Method::GET) + .uri("http://192.168.1.11:9020/mblock2/?"); + + let mut headers = req.headers_mut().expect("err"); + headers.insert("host", "192.168.1.11:9020".parse().unwrap()); + headers.insert( + "x-amz-content-sha256", + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + .parse() + .unwrap(), + ); + headers.insert("x-amz-date", timestamp.parse().unwrap()); + + let mut query: Vec<(String, String)> = Vec::new(); + let uri = req.uri_ref().unwrap().clone(); + let mut parts = req.uri_ref().unwrap().clone().into_parts(); + parts.path_and_query = Some( + format!("{}?{}", uri.path(), serde_urlencoded::to_string(&query).unwrap()) + .parse() + .unwrap(), + ); + let req = req.uri(Uri::from_parts(parts).unwrap()); + + let canonical_request = get_canonical_request(&req, &v4_ignored_headers, &get_hashed_payload(&req)); + println!("canonical_request: \n{}\n", canonical_request); + assert_eq!( + canonical_request, + concat!( + "GET\n", + "/mblock2/\n", + "\n", + "host:192.168.1.11:9020\n", + "x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\n", + "x-amz-date:", + "20250505T011054Z", + "\n", + "\n", + "host;x-amz-content-sha256;x-amz-date\n", + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + ) + ); + + let string_to_sign = get_string_to_sign_v4(t, region, &canonical_request, service); + println!("string_to_sign: \n{}\n", string_to_sign); + assert_eq!( + string_to_sign, + concat!( + "AWS4-HMAC-SHA256\n", + "20250505T011054Z", + "\n", + "20250505/us-east-1/s3/aws4_request\n", + "c2960d00cc7de7bed3e2e2d1330ec298ded8f78a231c1d32dedac72ebec7f9b0", + ) + ); + + let signing_key = get_signing_key(secret_access_key, region, t, service); + let signature = get_signature(signing_key, &string_to_sign); + println!("signature: \n{}\n", signature); + assert_eq!(signature, "df4116595e27b0dfd1103358947d9199378cc6386c4657abd8c5f0b11ebb4931"); + } + + #[test] + fn example_signature2() { + // let access_key_id = "rustfsadmin"; + let secret_access_key = "rustfsadmin"; + let timestamp = "20250507T051030Z"; + let t = datetime!(2025-05-07 05:10:30 UTC); + // let bucket = "mblock2"; + let region = "us-east-1"; + let service = "s3"; + let path = "/mblock2/"; + + let mut req = Request::builder().method(http::Method::GET).uri("http://192.168.1.11:9020/mblock2/?list-type=2&encoding-type=url&prefix=mypre&delimiter=%2F&fetch-owner=true&max-keys=1"); + + let mut headers = req.headers_mut().expect("err"); + headers.insert("host", "192.168.1.11:9020".parse().unwrap()); + headers.insert( + "x-amz-content-sha256", + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + .parse() + .unwrap(), + ); + headers.insert("x-amz-date", timestamp.parse().unwrap()); + + /*let uri = req.uri_ref().unwrap().clone(); + println!("{:?}", uri); + let mut canonical_query_string = "".to_string(); + if let Some(q) = uri.query() { + let result = serde_urlencoded::from_str::>(q); + let mut query = result.unwrap_or_default(); + query.sort(); + canonical_query_string = query.join("&"); + canonical_query_string.replace("+", "%20"); + } + let mut parts = req.uri_ref().unwrap().clone().into_parts(); + parts.path_and_query = Some(format!("{}?{}", uri.path(), canonical_query_string).parse().unwrap()); + let req = req.uri(Uri::from_parts(parts).unwrap());*/ + println!("{:?}", req.uri_ref().unwrap().query()); + let canonical_request = get_canonical_request(&req, &v4_ignored_headers, &get_hashed_payload(&req)); + println!("canonical_request: \n{}\n", canonical_request); + assert_eq!( + canonical_request, + concat!( + "GET\n", + "/mblock2/\n", + "delimiter=%2F&encoding-type=url&fetch-owner=true&list-type=2&max-keys=1&prefix=mypre\n", + "host:192.168.1.11:9020\n", + "x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\n", + "x-amz-date:", + "20250507T051030Z", + "\n", + "\n", + "host;x-amz-content-sha256;x-amz-date\n", + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + ) + ); + + let string_to_sign = get_string_to_sign_v4(t, region, &canonical_request, service); + println!("string_to_sign: \n{}\n", string_to_sign); + assert_eq!( + string_to_sign, + concat!( + "AWS4-HMAC-SHA256\n", + "20250507T051030Z", + "\n", + "20250507/us-east-1/s3/aws4_request\n", + "e6db9e09e9c873aff0b9ca170998b4753f6a6c36c90bc2dca80613affb47f999", + ) + ); + + let signing_key = get_signing_key(secret_access_key, region, t, service); + let signature = get_signature(signing_key, &string_to_sign); + println!("signature: \n{}\n", signature); + assert_eq!(signature, "760278c9a77d5c245ac83d85917bddc3e3b14343091e8f4ad8edbbf73107d685"); + } + + #[test] + fn example_presigned_url() { + use hyper::Uri; + + let access_key_id = "AKIAIOSFODNN7EXAMPLE"; + let secret_access_key = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"; + let timestamp = "20130524T000000Z"; + let t = datetime!(2013-05-24 0:00 UTC); + // let bucket = "mblock2"; + let region = "us-east-1"; + let service = "s3"; + let path = "/"; + let session_token = ""; + + let mut req = Request::builder() + .method(http::Method::GET) + .uri("http://examplebucket.s3.amazonaws.com/test.txt"); + + let mut headers = req.headers_mut().expect("err"); + headers.insert("host", "examplebucket.s3.amazonaws.com".parse().unwrap()); + + req = pre_sign_v4(req, &access_key_id, &secret_access_key, "", ®ion, 86400, t); + + let mut canonical_request = req.method_ref().unwrap().as_str().to_string(); + canonical_request.push_str("\n"); + canonical_request.push_str(req.uri_ref().unwrap().path()); + canonical_request.push_str("\n"); + canonical_request.push_str(req.uri_ref().unwrap().query().unwrap()); + canonical_request.push_str("\n"); + canonical_request.push_str(&get_canonical_headers(&req, &v4_ignored_headers)); + canonical_request.push_str("\n"); + canonical_request.push_str(&get_signed_headers(&req, &v4_ignored_headers)); + canonical_request.push_str("\n"); + canonical_request.push_str(&get_hashed_payload(&req)); + //println!("canonical_request: \n{}\n", canonical_request); + assert_eq!( + canonical_request, + concat!( + "GET\n", + "/test.txt\n", + "X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Date=20130524T000000Z&X-Amz-Expires=0000086400&X-Amz-SignedHeaders=host&X-Amz-Credential=AKIAIOSFODNN7EXAMPLE%2F20130524%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Signature=aeeed9bbccd4d02ee5c0109b86d86835f995330da4c265957d157751f604d404\n", + "host:examplebucket.s3.amazonaws.com\n", + "\n", + "host\n", + "UNSIGNED-PAYLOAD", + ) + ); + } + + #[test] + fn example_presigned_url2() { + use hyper::Uri; + + let access_key_id = "rustfsadmin"; + let secret_access_key = "rustfsadmin"; + let timestamp = "20130524T000000Z"; + let t = datetime!(2013-05-24 0:00 UTC); + // let bucket = "mblock2"; + let region = "us-east-1"; + let service = "s3"; + let path = "/mblock2/"; + let session_token = ""; + + let mut req = Request::builder().method(http::Method::GET).uri("http://192.168.1.11:9020/mblock2/test.txt?delimiter=%2F&fetch-owner=true&prefix=mypre&encoding-type=url&max-keys=1&list-type=2"); + + let mut headers = req.headers_mut().expect("err"); + headers.insert("host", "192.168.1.11:9020".parse().unwrap()); + + req = pre_sign_v4(req, &access_key_id, &secret_access_key, "", ®ion, 86400, t); + + let mut canonical_request = req.method_ref().unwrap().as_str().to_string(); + canonical_request.push_str("\n"); + canonical_request.push_str(req.uri_ref().unwrap().path()); + canonical_request.push_str("\n"); + canonical_request.push_str(req.uri_ref().unwrap().query().unwrap()); + canonical_request.push_str("\n"); + canonical_request.push_str(&get_canonical_headers(&req, &v4_ignored_headers)); + canonical_request.push_str("\n"); + canonical_request.push_str(&get_signed_headers(&req, &v4_ignored_headers)); + canonical_request.push_str("\n"); + canonical_request.push_str(&get_hashed_payload(&req)); + //println!("canonical_request: \n{}\n", canonical_request); + assert_eq!( + canonical_request, + concat!( + "GET\n", + "/mblock2/test.txt\n", + "delimiter=%2F&fetch-owner=true&prefix=mypre&encoding-type=url&max-keys=1&list-type=2&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Date=20130524T000000Z&X-Amz-Expires=0000086400&X-Amz-SignedHeaders=host&X-Amz-Credential=rustfsadmin%2F20130524%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Signature=e4af975e4a7e2c0449451740c7e9a425123681d2a8830bfb188789ea19618b20\n", + "host:192.168.1.11:9020\n", + "\n", + "host\n", + "UNSIGNED-PAYLOAD", + ) + ); + } +} diff --git a/ecstore/src/signer/utils.rs b/ecstore/src/signer/utils.rs new file mode 100644 index 000000000..61bdf63b8 --- /dev/null +++ b/ecstore/src/signer/utils.rs @@ -0,0 +1,31 @@ +use http::request; + +pub fn get_host_addr(req: &request::Builder) -> String { + let host = req.headers_ref().expect("err").get("host"); + let uri = req.uri_ref().unwrap(); + let req_host; + if let Some(port) = uri.port() { + req_host = format!("{}:{}", uri.host().unwrap(), port); + } else { + req_host = uri.host().unwrap().to_string(); + } + if host.is_some() && req_host != host.unwrap().to_str().unwrap().to_string() { + return host.unwrap().to_str().unwrap().to_string(); + } + /*if req.uri_ref().unwrap().host().is_some() { + return req.uri_ref().unwrap().host().unwrap(); + }*/ + req_host +} + +pub fn sign_v4_trim_all(input: &str) -> String { + let ss = input.split_whitespace().collect::>(); + ss.join(" ") +} + +pub fn stable_sort_by_first(v: &mut [(T, T)]) +where + T: Ord, +{ + v.sort_by(|lhs, rhs| lhs.0.cmp(&rhs.0)); +} diff --git a/ecstore/src/store.rs b/ecstore/src/store.rs index 9a92d72e4..1a85d6e1c 100644 --- a/ecstore/src/store.rs +++ b/ecstore/src/store.rs @@ -1,19 +1,21 @@ #![allow(clippy::map_entry)] +use crate::bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry; use crate::bucket::metadata_sys::{self, set_bucket_metadata}; use crate::bucket::utils::{check_valid_bucket_name, check_valid_bucket_name_strict, is_meta_bucketname}; use crate::config::GLOBAL_StorageClass; use crate::config::storageclass; use crate::disk::endpoint::{Endpoint, EndpointType}; use crate::disk::{DiskAPI, DiskInfo, DiskInfoOptions}; +use crate::error::{Error, Result}; use crate::error::{ StorageError, is_err_bucket_exists, is_err_invalid_upload_id, is_err_object_not_found, is_err_read_quorum, is_err_version_not_found, to_object_err, }; use crate::global::{ DISK_ASSUME_UNKNOWN_SIZE, DISK_FILL_FRACTION, DISK_MIN_INODES, DISK_RESERVE_FRACTION, GLOBAL_BOOT_TIME, - GLOBAL_LOCAL_DISK_MAP, GLOBAL_LOCAL_DISK_SET_DRIVES, get_global_endpoints, is_dist_erasure, is_erasure_sd, - set_global_deployment_id, set_object_layer, + GLOBAL_LOCAL_DISK_MAP, GLOBAL_LOCAL_DISK_SET_DRIVES, GLOBAL_TierConfigMgr, get_global_endpoints, is_dist_erasure, + is_erasure_sd, set_global_deployment_id, set_object_layer, }; use crate::heal::data_usage::{DATA_USAGE_ROOT, DataUsageInfo}; use crate::heal::data_usage_cache::{DataUsageCache, DataUsageCacheInfo}; @@ -26,7 +28,7 @@ use crate::rebalance::RebalanceMeta; use crate::store_api::{ListMultipartsInfo, ListObjectVersionsInfo, MultipartInfo, ObjectIO}; use crate::store_init::{check_disk_fatal_errs, ec_drives_no_config}; use crate::{ - bucket::metadata::BucketMetadata, + bucket::{lifecycle::bucket_lifecycle_ops::TransitionState, metadata::BucketMetadata}, disk::{BUCKET_META_PREFIX, DiskOption, DiskStore, RUSTFS_META_BUCKET, new_disk}, endpoints::EndpointServerPools, rpc::S3PeerSys, @@ -38,10 +40,6 @@ use crate::{ }, store_init, }; -use rustfs_utils::crypto::base64_decode; -use rustfs_utils::path::{SLASH_SEPARATOR, decode_dir_object, encode_dir_object, path_join_buf}; - -use crate::error::{Error, Result}; use common::globals::{GLOBAL_Local_Node_Name, GLOBAL_Rustfs_Host, GLOBAL_Rustfs_Port}; use futures::future::join_all; use glob::Pattern; @@ -49,7 +47,10 @@ use http::HeaderMap; use lazy_static::lazy_static; use madmin::heal_commands::HealResultItem; use rand::Rng as _; +use rustfs_filemeta::FileInfo; use rustfs_filemeta::MetaCacheEntry; +use rustfs_utils::crypto::base64_decode; +use rustfs_utils::path::{SLASH_SEPARATOR, decode_dir_object, encode_dir_object, path_join_buf}; use s3s::dto::{BucketVersioningStatus, ObjectLockConfiguration, ObjectLockEnabled, VersioningConfiguration}; use std::cmp::Ordering; use std::net::SocketAddr; @@ -330,6 +331,14 @@ impl ECStore { } } + init_background_expiry(self.clone()).await; + + TransitionState::init(self.clone()).await; + + if let Err(err) = GLOBAL_TierConfigMgr.write().await.init(self.clone()).await { + info!("TierConfigMgr init error: {}", err); + } + Ok(()) } @@ -1857,7 +1866,6 @@ impl StorageAPI for ECStore { return self.pools[idx].new_multipart_upload(bucket, object, opts).await; } } - let idx = self.get_pool_idx(bucket, object, -1).await?; if opts.data_movement && idx == opts.src_pool_idx { return Err(StorageError::DataMovementOverwriteErr( @@ -1870,6 +1878,49 @@ impl StorageAPI for ECStore { self.pools[idx].new_multipart_upload(bucket, object, opts).await } + #[tracing::instrument(skip(self))] + async fn add_partial(&self, bucket: &str, object: &str, version_id: &str) -> Result<()> { + let object = encode_dir_object(object); + + if self.single_pool() { + self.pools[0].add_partial(bucket, object.as_str(), version_id).await; + } + + let idx = self + .get_pool_idx_existing_with_opts(bucket, object.as_str(), &ObjectOptions::default()) + .await?; + + self.pools[idx].add_partial(bucket, object.as_str(), version_id).await; + Ok(()) + } + #[tracing::instrument(skip(self))] + async fn transition_object(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> { + let object = encode_dir_object(object); + if self.single_pool() { + return self.pools[0].transition_object(bucket, &object, opts).await; + } + + //opts.skip_decommissioned = true; + //opts.no_lock = true; + let idx = self.get_pool_idx_existing_with_opts(bucket, &object, opts).await?; + + self.pools[idx].transition_object(bucket, &object, opts).await + } + + #[tracing::instrument(skip(self))] + async fn restore_transitioned_object(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> { + let object = encode_dir_object(object); + if self.single_pool() { + return self.pools[0].restore_transitioned_object(bucket, &object, opts).await; + } + + //opts.skip_decommissioned = true; + //opts.nolock = true; + let idx = self.get_pool_idx_existing_with_opts(bucket, object.as_str(), opts).await?; + + self.pools[idx].restore_transitioned_object(bucket, &object, opts).await + } + #[tracing::instrument(skip(self))] async fn copy_object_part( &self, @@ -2098,6 +2149,20 @@ impl StorageAPI for ECStore { self.pools[idx].put_object_tags(bucket, object.as_str(), tags, opts).await } + #[tracing::instrument(skip(self))] + async fn delete_object_version(&self, bucket: &str, object: &str, fi: &FileInfo, force_del_marker: bool) -> Result<()> { + check_del_obj_args(bucket, object)?; + + let object = rustfs_utils::path::encode_dir_object(object); + + if self.single_pool() { + return self.pools[0] + .delete_object_version(bucket, object.as_str(), fi, force_del_marker) + .await; + } + Ok(()) + } + #[tracing::instrument(skip(self))] async fn delete_object_tags(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result { let object = encode_dir_object(object); diff --git a/ecstore/src/store_api.rs b/ecstore/src/store_api.rs index 02e073047..d68fca704 100644 --- a/ecstore/src/store_api.rs +++ b/ecstore/src/store_api.rs @@ -4,6 +4,11 @@ use crate::cmd::bucket_replication::{ReplicationStatusType, VersionPurgeStatusTy use crate::error::{Error, Result}; use crate::heal::heal_ops::HealSequence; use crate::store_utils::clean_metadata; +use crate::{ + bucket::lifecycle::bucket_lifecycle_audit::LcAuditEvent, + bucket::lifecycle::lifecycle::ExpirationOptions, + bucket::lifecycle::{bucket_lifecycle_ops::TransitionedObject, lifecycle::TransitionOptions}, +}; use crate::{disk::DiskStore, heal::heal_commands::HealOpts}; use http::{HeaderMap, HeaderValue}; use madmin::heal_commands::HealResultItem; @@ -290,6 +295,10 @@ pub struct ObjectOptions { pub replication_request: bool, pub delete_marker: bool, + pub transition: TransitionOptions, + pub expiration: ExpirationOptions, + pub lifecycle_audit_event: LcAuditEvent, + pub eval_metadata: Option>, } @@ -362,6 +371,7 @@ pub struct ObjectInfo { pub data_blocks: usize, pub version_id: Option, pub delete_marker: bool, + pub transitioned_object: TransitionedObject, pub user_tags: String, pub parts: Vec, pub is_latest: bool, @@ -395,6 +405,7 @@ impl Clone for ObjectInfo { data_blocks: self.data_blocks, version_id: self.version_id, delete_marker: self.delete_marker, + transitioned_object: self.transitioned_object.clone(), user_tags: self.user_tags.clone(), parts: self.parts.clone(), is_latest: self.is_latest, @@ -497,7 +508,18 @@ impl ObjectInfo { // TODO:expires // TODO:ReplicationState - // TODO:TransitionedObject + + let transitioned_object = TransitionedObject { + name: fi.transitioned_objname.clone(), + version_id: if let Some(transition_version_id) = fi.transition_version_id { + transition_version_id.to_string() + } else { + "".to_string() + }, + status: fi.transition_status.clone(), + free_version: fi.tier_free_version(), + tier: fi.transition_tier.clone(), + }; let metadata = { let mut v = fi.metadata.clone(); @@ -540,6 +562,7 @@ impl ObjectInfo { etag, inlined, user_defined: metadata, + transitioned_object, ..Default::default() } } @@ -813,6 +836,7 @@ pub trait StorageAPI: ObjectIO { src_opts: &ObjectOptions, dst_opts: &ObjectOptions, ) -> Result; + async fn delete_object_version(&self, bucket: &str, object: &str, fi: &FileInfo, force_del_marker: bool) -> Result<()>; async fn delete_object(&self, bucket: &str, object: &str, opts: ObjectOptions) -> Result; async fn delete_objects( &self, @@ -887,6 +911,9 @@ pub trait StorageAPI: ObjectIO { async fn put_object_metadata(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result; // DecomTieredObject async fn get_object_tags(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result; + async fn add_partial(&self, bucket: &str, object: &str, version_id: &str) -> Result<()>; + async fn transition_object(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()>; + async fn restore_transitioned_object(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()>; async fn put_object_tags(&self, bucket: &str, object: &str, tags: &str, opts: &ObjectOptions) -> Result; async fn delete_object_tags(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result; diff --git a/ecstore/src/tier/mod.rs b/ecstore/src/tier/mod.rs new file mode 100644 index 000000000..8c54e1690 --- /dev/null +++ b/ecstore/src/tier/mod.rs @@ -0,0 +1,9 @@ +pub mod tier; +pub mod tier_admin; +pub mod tier_config; +pub mod tier_gen; +pub mod tier_handlers; +pub mod warm_backend; +pub mod warm_backend_minio; +pub mod warm_backend_rustfs; +pub mod warm_backend_s3; diff --git a/ecstore/src/tier/tier.rs b/ecstore/src/tier/tier.rs new file mode 100644 index 000000000..e8e54003b --- /dev/null +++ b/ecstore/src/tier/tier.rs @@ -0,0 +1,455 @@ +use bytes::Bytes; +use http::status::StatusCode; +use lazy_static::lazy_static; +use rand::Rng; +use serde::{Deserialize, Serialize}; +use std::{ + collections::{HashMap, hash_map::Entry}, + io::Cursor, + sync::Arc, + time::Duration, +}; +use time::OffsetDateTime; +use tokio::io::BufReader; +use tokio::{select, sync::RwLock, time::interval}; +use tracing::{debug, error, info, warn}; + +use crate::client::admin_handler_utils::AdminError; +use crate::error::{Error, Result, StorageError}; +use crate::new_object_layer_fn; +use crate::tier::{ + tier_admin::TierCreds, + tier_config::{TierConfig, TierType}, + tier_handlers::{ERR_TIER_ALREADY_EXISTS, ERR_TIER_NAME_NOT_UPPERCASE, ERR_TIER_NOT_FOUND}, + warm_backend::{check_warm_backend, new_warm_backend}, +}; +use crate::{ + StorageAPI, + config::com::{CONFIG_PREFIX, read_config}, + disk::RUSTFS_META_BUCKET, + store::ECStore, + store_api::{ObjectOptions, PutObjReader}, +}; +use rustfs_rio::HashReader; +use rustfs_utils::path::{SLASH_SEPARATOR, path_join}; +use s3s::S3ErrorCode; + +use super::{ + tier_handlers::{ERR_TIER_BUCKET_NOT_FOUND, ERR_TIER_CONNECT_ERR, ERR_TIER_INVALID_CREDENTIALS, ERR_TIER_PERM_ERR}, + warm_backend::WarmBackendImpl, +}; + +const TIER_CFG_REFRESH: Duration = Duration::from_secs(15 * 60); + +pub const TIER_CONFIG_FILE: &str = "tier-config.json"; +pub const TIER_CONFIG_FORMAT: u16 = 1; +pub const TIER_CONFIG_V1: u16 = 1; +pub const TIER_CONFIG_VERSION: u16 = 1; + +lazy_static! { + //pub static ref TIER_CONFIG_PATH: PathBuf = path_join(&[PathBuf::from(RUSTFS_CONFIG_PREFIX), PathBuf::from(TIER_CONFIG_FILE)]); +} + +const TIER_CFG_REFRESH_AT_HDR: &str = "X-RustFS-TierCfg-RefreshedAt"; + +pub const ERR_TIER_MISSING_CREDENTIALS: AdminError = AdminError { + code: "XRustFSAdminTierMissingCredentials", + message: "Specified remote credentials are empty", + status_code: StatusCode::FORBIDDEN, +}; + +pub const ERR_TIER_BACKEND_IN_USE: AdminError = AdminError { + code: "XRustFSAdminTierBackendInUse", + message: "Specified remote tier is already in use", + status_code: StatusCode::CONFLICT, +}; + +pub const ERR_TIER_TYPE_UNSUPPORTED: AdminError = AdminError { + code: "XRustFSAdminTierTypeUnsupported", + message: "Specified tier type is unsupported", + status_code: StatusCode::BAD_REQUEST, +}; + +pub const ERR_TIER_BACKEND_NOT_EMPTY: AdminError = AdminError { + code: "XRustFSAdminTierBackendNotEmpty", + message: "Specified remote backend is not empty", + status_code: StatusCode::BAD_REQUEST, +}; + +pub const ERR_TIER_INVALID_CONFIG: AdminError = AdminError { + code: "XRustFSAdminTierInvalidConfig", + message: "Unable to setup remote tier, check tier configuration", + status_code: StatusCode::BAD_REQUEST, +}; + +#[derive(Serialize, Deserialize)] +pub struct TierConfigMgr { + #[serde(skip)] + pub driver_cache: HashMap, + pub tiers: HashMap, + pub last_refreshed_at: OffsetDateTime, +} + +impl TierConfigMgr { + pub fn new() -> Arc> { + Arc::new(RwLock::new(Self { + driver_cache: HashMap::new(), + tiers: HashMap::new(), + last_refreshed_at: OffsetDateTime::now_utc(), + })) + } + + pub fn unmarshal(data: &[u8]) -> std::result::Result { + let cfg: TierConfigMgr = serde_json::from_slice(data)?; + //let mut cfg = TierConfigMgr(m); + //let mut cfg = m; + Ok(cfg) + } + + pub fn marshal(&self) -> std::result::Result { + let data = serde_json::to_vec(&self)?; + + //let mut data = Vec::with_capacity(self.msg_size()+4); + let mut data = Bytes::from(data); + + //LittleEndian::write_u16(&mut data[0..2], TIER_CONFIG_FORMAT); + //LittleEndian::write_u16(&mut data[2..4], TIER_CONFIG_VERSION); + + Ok(data) + } + + pub fn refreshed_at(&self) -> OffsetDateTime { + self.last_refreshed_at + } + + pub fn is_tier_valid(&self, tier_name: &str) -> bool { + let (_, valid) = self.is_tier_name_in_use(tier_name); + valid + } + + pub fn is_tier_name_in_use(&self, tier_name: &str) -> (TierType, bool) { + if let Some(t) = self.tiers.get(tier_name) { + return (t.tier_type.clone(), true); + } + (TierType::Unsupported, false) + } + + pub async fn add(&mut self, tier: TierConfig, force: bool) -> std::result::Result<(), AdminError> { + let tier_name = &tier.name; + if tier_name != tier_name.to_uppercase().as_str() { + return Err(ERR_TIER_NAME_NOT_UPPERCASE); + } + + let (_, b) = self.is_tier_name_in_use(tier_name); + if b { + return Err(ERR_TIER_ALREADY_EXISTS); + } + + let d = new_warm_backend(&tier, true).await?; + + if !force { + let in_use = d.in_use().await; + match in_use { + Ok(b) => { + if b { + return Err(ERR_TIER_BACKEND_IN_USE); + } + } + Err(err) => { + warn!("tier add failed, err: {:?}", err); + if err.to_string().contains("connect") { + return Err(ERR_TIER_CONNECT_ERR); + } else if err.to_string().contains("authorization") { + return Err(ERR_TIER_INVALID_CREDENTIALS); + } else if err.to_string().contains("bucket") { + return Err(ERR_TIER_BUCKET_NOT_FOUND); + } + return Err(ERR_TIER_PERM_ERR); + } + } + } + + self.driver_cache.insert(tier_name.to_string(), d); + self.tiers.insert(tier_name.to_string(), tier); + + Ok(()) + } + + pub async fn remove(&mut self, tier_name: &str, force: bool) -> std::result::Result<(), AdminError> { + let d = self.get_driver(tier_name).await; + if let Err(err) = d { + match err { + ERR_TIER_NOT_FOUND => { + return Ok(()); + } + _ => { + return Err(err); + } + } + } + if !force { + let inuse = d.expect("err").in_use().await; + if let Err(err) = inuse { + return Err(ERR_TIER_PERM_ERR); + } else if inuse.expect("err") { + return Err(ERR_TIER_BACKEND_NOT_EMPTY); + } + } + self.tiers.remove(tier_name); + self.driver_cache.remove(tier_name); + Ok(()) + } + + pub async fn verify(&mut self, tier_name: &str) -> std::result::Result<(), std::io::Error> { + let d = match self.get_driver(tier_name).await { + Ok(d) => d, + Err(err) => { + return Err(std::io::Error::other(err)); + } + }; + if let Err(err) = check_warm_backend(Some(d)).await { + return Err(std::io::Error::other(err)); + } else { + return Ok(()); + } + } + + pub fn empty(&self) -> bool { + self.list_tiers().len() == 0 + } + + pub fn tier_type(&self, tier_name: &str) -> String { + let cfg = self.tiers.get(tier_name); + if cfg.is_none() { + return "internal".to_string(); + } + cfg.expect("err").tier_type.to_string() + } + + pub fn list_tiers(&self) -> Vec { + let mut tier_cfgs = Vec::::new(); + for (_, tier) in self.tiers.iter() { + let tier = tier.clone(); + tier_cfgs.push(tier); + } + tier_cfgs + } + + pub fn get(&self, tier_name: &str) -> Option { + for (tier_name2, tier) in self.tiers.iter() { + if tier_name == tier_name2 { + return Some(tier.clone()); + } + } + None + } + + pub async fn edit(&mut self, tier_name: &str, creds: TierCreds) -> std::result::Result<(), AdminError> { + let (tier_type, exists) = self.is_tier_name_in_use(tier_name); + if !exists { + return Err(ERR_TIER_NOT_FOUND); + } + + let mut cfg = self.tiers[tier_name].clone(); + match tier_type { + TierType::S3 => { + let mut s3 = cfg.s3.as_mut().expect("err"); + if creds.aws_role { + s3.aws_role = true + } + if creds.aws_role_web_identity_token_file != "" && creds.aws_role_arn != "" { + s3.aws_role_arn = creds.aws_role_arn; + s3.aws_role_web_identity_token_file = creds.aws_role_web_identity_token_file; + } + if creds.access_key != "" && creds.secret_key != "" { + s3.access_key = creds.access_key; + s3.secret_key = creds.secret_key; + } + } + TierType::RustFS => { + let mut rustfs = cfg.rustfs.as_mut().expect("err"); + if creds.access_key == "" || creds.secret_key == "" { + return Err(ERR_TIER_MISSING_CREDENTIALS); + } + rustfs.access_key = creds.access_key; + rustfs.secret_key = creds.secret_key; + } + TierType::MinIO => { + let mut minio = cfg.minio.as_mut().expect("err"); + if creds.access_key == "" || creds.secret_key == "" { + return Err(ERR_TIER_MISSING_CREDENTIALS); + } + minio.access_key = creds.access_key; + minio.secret_key = creds.secret_key; + } + _ => (), + } + + let d = new_warm_backend(&cfg, true).await?; + self.tiers.insert(tier_name.to_string(), cfg); + self.driver_cache.insert(tier_name.to_string(), d); + Ok(()) + } + + pub async fn get_driver<'a>(&'a mut self, tier_name: &str) -> std::result::Result<&'a WarmBackendImpl, AdminError> { + Ok(match self.driver_cache.entry(tier_name.to_string()) { + Entry::Occupied(e) => e.into_mut(), + Entry::Vacant(e) => { + let t = self.tiers.get(tier_name); + if t.is_none() { + return Err(ERR_TIER_NOT_FOUND); + } + let d = new_warm_backend(t.expect("err"), false).await?; + e.insert(d) + } + }) + } + + pub async fn reload(&mut self, api: Arc) -> std::result::Result<(), std::io::Error> { + //let Some(api) = new_object_layer_fn() else { return Err(Error::msg("errServerNotInitialized")) }; + let new_config = load_tier_config(api).await; + + match &new_config { + Ok(_c) => {} + Err(err) => { + return Err(std::io::Error::other(err.to_string())); + } + } + self.driver_cache.clear(); + self.tiers.clear(); + let new_config = new_config.expect("err"); + for (tier, cfg) in new_config.tiers { + self.tiers.insert(tier, cfg); + } + self.last_refreshed_at = OffsetDateTime::now_utc(); + Ok(()) + } + + #[tracing::instrument(level = "debug", name = "tier_save", skip(self))] + pub async fn save(&self) -> std::result::Result<(), std::io::Error> { + let Some(api) = new_object_layer_fn() else { + return Err(std::io::Error::other("errServerNotInitialized")); + }; + //let (pr, opts) = GLOBAL_TierConfigMgr.write().config_reader()?; + + self.save_tiering_config(api).await + } + + pub async fn save_tiering_config(&self, api: Arc) -> std::result::Result<(), std::io::Error> { + let data = self.marshal()?; + + let config_file = format!("{}{}{}", CONFIG_PREFIX, SLASH_SEPARATOR, TIER_CONFIG_FILE); + + self.save_config(api, &config_file, data).await + } + + pub async fn save_config( + &self, + api: Arc, + file: &str, + data: Bytes, + ) -> std::result::Result<(), std::io::Error> { + self.save_config_with_opts( + api, + file, + data, + &ObjectOptions { + max_parity: true, + ..Default::default() + }, + ) + .await + } + + pub async fn save_config_with_opts( + &self, + api: Arc, + file: &str, + data: Bytes, + opts: &ObjectOptions, + ) -> std::result::Result<(), std::io::Error> { + debug!("save tier config:{}", file); + let _ = api + .put_object(RUSTFS_META_BUCKET, file, &mut PutObjReader::from_vec(data.to_vec()), opts) + .await?; + Ok(()) + } + + pub async fn refresh_tier_config(&mut self, api: Arc) { + //let r = rand.New(rand.NewSource(time.Now().UnixNano())); + let mut rng = rand::rng(); + let r = rng.random_range(0.0..1.0); + let rand_interval = || Duration::from_secs((r * 60_f64).round() as u64); + + let mut t = interval(TIER_CFG_REFRESH + rand_interval()); + loop { + select! { + _ = t.tick() => { + if let Err(err) = self.reload(api.clone()).await { + info!("{}", err); + } + } + else => () + } + t.reset(); + } + } + + pub async fn init(&mut self, api: Arc) -> Result<()> { + self.reload(api).await?; + //if globalIsDistErasure { + // self.refresh_tier_config(api).await; + //} + Ok(()) + } +} + +async fn new_and_save_tiering_config(api: Arc) -> Result { + let mut cfg = TierConfigMgr { + driver_cache: HashMap::new(), + tiers: HashMap::new(), + last_refreshed_at: OffsetDateTime::now_utc(), + }; + //lookup_configs(&mut cfg, api.clone()).await; + cfg.save_tiering_config(api).await?; + + Ok(cfg) +} + +#[tracing::instrument(level = "debug")] +async fn load_tier_config(api: Arc) -> std::result::Result { + let config_file = format!("{}{}{}", CONFIG_PREFIX, SLASH_SEPARATOR, TIER_CONFIG_FILE); + let data = read_config(api.clone(), config_file.as_str()).await; + if let Err(err) = data { + if is_err_config_not_found(&err) { + warn!("config not found, start to init"); + let cfg = new_and_save_tiering_config(api).await?; + return Ok(cfg); + } else { + error!("read config err {:?}", &err); + return Err(std::io::Error::other(err)); + } + } + + let cfg; + let version = 1; //LittleEndian::read_u16(&data[2..4]); + match version { + TIER_CONFIG_V1/* | TIER_CONFIG_VERSION */ => { + cfg = match TierConfigMgr::unmarshal(&data.unwrap()) { + Ok(cfg) => cfg, + Err(err) => { + return Err(std::io::Error::other(err.to_string())); + } + }; + } + _ => { + return Err(std::io::Error::other(format!("tierConfigInit: unknown version: {}", version))); + } + } + + Ok(cfg) +} + +pub fn is_err_config_not_found(err: &StorageError) -> bool { + matches!(err, StorageError::ObjectNotFound(_, _)) +} diff --git a/ecstore/src/tier/tier_admin.rs b/ecstore/src/tier/tier_admin.rs new file mode 100644 index 000000000..699e4b265 --- /dev/null +++ b/ecstore/src/tier/tier_admin.rs @@ -0,0 +1,26 @@ +use http::status::StatusCode; +use rand::Rng; +use serde::{Deserialize, Serialize}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; +use tracing::warn; + +#[derive(Serialize, Deserialize, Default, Debug, Clone)] +#[serde(default)] +pub struct TierCreds { + #[serde(rename = "accessKey")] + pub access_key: String, + #[serde(rename = "secretKey")] + pub secret_key: String, + + #[serde(rename = "awsRole")] + pub aws_role: bool, + #[serde(rename = "awsRoleWebIdentityTokenFile")] + pub aws_role_web_identity_token_file: String, + #[serde(rename = "awsRoleArn")] + pub aws_role_arn: String, + + //azsp: ServicePrincipalAuth, + + //#[serde(rename = "credsJson")] + pub creds_json: Vec, +} diff --git a/ecstore/src/tier/tier_config.rs b/ecstore/src/tier/tier_config.rs new file mode 100644 index 000000000..b6e2f6316 --- /dev/null +++ b/ecstore/src/tier/tier_config.rs @@ -0,0 +1,316 @@ +use serde::{Deserialize, Serialize}; +use std::fmt::Display; +use tracing::info; + +const C_TierConfigVer: &str = "v1"; + +const ERR_TIER_NAME_EMPTY: &str = "remote tier name empty"; +const ERR_TIER_INVALID_CONFIG: &str = "invalid tier config"; +const ERR_TIER_INVALID_CONFIG_VERSION: &str = "invalid tier config version"; +const ERR_TIER_TYPE_UNSUPPORTED: &str = "unsupported tier type"; + +#[derive(Serialize, Deserialize, Default, Debug, Clone)] +pub enum TierType { + #[default] + Unsupported, + #[serde(rename = "s3")] + S3, + #[serde(rename = "azure")] + Azure, + #[serde(rename = "gcs")] + GCS, + #[serde(rename = "rustfs")] + RustFS, + #[serde(rename = "minio")] + MinIO, +} + +impl Display for TierType { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + TierType::S3 => { + write!(f, "S3") + } + TierType::RustFS => { + write!(f, "RustFS") + } + TierType::MinIO => { + write!(f, "MinIO") + } + _ => { + write!(f, "Unsupported") + } + } + } +} + +impl TierType { + pub fn new(sc_type: &str) -> Self { + match sc_type { + "S3" => TierType::S3, + "RustFS" => TierType::RustFS, + "MinIO" => TierType::MinIO, + _ => TierType::Unsupported, + } + } + + pub fn to_string(&self) -> String { + match self { + TierType::S3 => "s3".to_string(), + TierType::RustFS => "rustfs".to_string(), + TierType::MinIO => "minio".to_string(), + _ => "unsupported".to_string(), + } + } +} + +#[derive(Default, Debug, Serialize, Deserialize)] +#[serde(default)] +pub struct TierConfig { + #[serde(skip)] + pub version: String, + #[serde(rename = "type")] + pub tier_type: TierType, + #[serde(skip)] + pub name: String, + #[serde(rename = "s3", skip_serializing_if = "Option::is_none")] + pub s3: Option, + //TODO: azure: Option, + //TODO: gcs: Option, + #[serde(rename = "rustfs", skip_serializing_if = "Option::is_none")] + pub rustfs: Option, + #[serde(rename = "minio", skip_serializing_if = "Option::is_none")] + pub minio: Option, +} + +impl Clone for TierConfig { + fn clone(&self) -> TierConfig { + let mut s3 = None; + //az TierAzure + //gcs TierGCS + let mut r = None; + let mut m = None; + match self.tier_type { + TierType::S3 => { + let mut s3_ = self.s3.as_ref().expect("err").clone(); + s3_.secret_key = "REDACTED".to_string(); + s3 = Some(s3_); + } + TierType::RustFS => { + let mut r_ = self.rustfs.as_ref().expect("err").clone(); + r_.secret_key = "REDACTED".to_string(); + r = Some(r_); + } + TierType::MinIO => { + let mut m_ = self.minio.as_ref().expect("err").clone(); + m_.secret_key = "REDACTED".to_string(); + m = Some(m_); + } + _ => (), + } + TierConfig { + version: self.version.clone(), + tier_type: self.tier_type.clone(), + name: self.name.clone(), + s3: s3, + //azure: az, + //gcs: gcs, + rustfs: r, + minio: m, + } + } +} + +impl TierConfig { + pub fn unmarshal(data: &[u8]) -> Result { + /*let m: HashMap> = serde_json::from_slice(data)?; + let mut cfg = TierConfig(m); + cfg.set_defaults(); + Ok(cfg)*/ + todo!(); + } + + pub fn marshal(&self) -> Result, std::io::Error> { + let data = serde_json::to_vec(&self)?; + Ok(data) + } + + fn endpoint(&self) -> String { + match self.tier_type { + TierType::S3 => self.s3.as_ref().expect("err").endpoint.clone(), + TierType::RustFS => self.rustfs.as_ref().expect("err").endpoint.clone(), + TierType::MinIO => self.minio.as_ref().expect("err").endpoint.clone(), + _ => { + info!("unexpected tier type {}", self.tier_type); + "".to_string() + } + } + } + + fn bucket(&self) -> String { + match self.tier_type { + TierType::S3 => self.s3.as_ref().expect("err").bucket.clone(), + TierType::RustFS => self.rustfs.as_ref().expect("err").bucket.clone(), + TierType::MinIO => self.minio.as_ref().expect("err").bucket.clone(), + _ => { + info!("unexpected tier type {}", self.tier_type); + "".to_string() + } + } + } + + fn prefix(&self) -> String { + match self.tier_type { + TierType::S3 => self.s3.as_ref().expect("err").prefix.clone(), + TierType::RustFS => self.rustfs.as_ref().expect("err").prefix.clone(), + TierType::MinIO => self.minio.as_ref().expect("err").prefix.clone(), + _ => { + info!("unexpected tier type {}", self.tier_type); + "".to_string() + } + } + } + + fn region(&self) -> String { + match self.tier_type { + TierType::S3 => self.s3.as_ref().expect("err").region.clone(), + TierType::RustFS => self.rustfs.as_ref().expect("err").region.clone(), + TierType::MinIO => self.minio.as_ref().expect("err").region.clone(), + _ => { + info!("unexpected tier type {}", self.tier_type); + "".to_string() + } + } + } +} + +//type S3Options = impl Fn(TierS3) -> Pin>> + Send + Sync + 'static; + +#[derive(Serialize, Deserialize, Default, Debug, Clone)] +#[serde(default)] +pub struct TierS3 { + pub name: String, + pub endpoint: String, + #[serde(rename = "accesskey")] + pub access_key: String, + #[serde(rename = "secretkey")] + pub secret_key: String, + pub bucket: String, + pub prefix: String, + pub region: String, + #[serde(rename = "storageclass")] + pub storage_class: String, + #[serde(skip)] + pub aws_role: bool, + #[serde(skip)] + pub aws_role_web_identity_token_file: String, + #[serde(skip)] + pub aws_role_arn: String, + #[serde(skip)] + pub aws_role_session_name: String, + #[serde(skip)] + pub aws_role_duration_seconds: i32, +} + +impl TierS3 { + fn new(name: &str, access_key: &str, secret_key: &str, bucket: &str, options: Vec) -> Result + where + F: Fn(TierS3) -> Box> + Send + Sync + 'static, + { + if name == "" { + return Err(std::io::Error::other(ERR_TIER_NAME_EMPTY)); + } + let sc = TierS3 { + access_key: access_key.to_string(), + secret_key: secret_key.to_string(), + bucket: bucket.to_string(), + endpoint: "https://s3.amazonaws.com".to_string(), + region: "".to_string(), + storage_class: "".to_string(), + ..Default::default() + }; + + for option in options { + let option = option(sc.clone()); + let option = *option; + option?; + } + + Ok(TierConfig { + version: C_TierConfigVer.to_string(), + tier_type: TierType::S3, + name: name.to_string(), + s3: Some(sc), + ..Default::default() + }) + } +} + +#[derive(Serialize, Deserialize, Default, Debug, Clone)] +#[serde(default)] +pub struct TierRustFS { + pub name: String, + pub endpoint: String, + #[serde(rename = "accesskey")] + pub access_key: String, + #[serde(rename = "secretkey")] + pub secret_key: String, + pub bucket: String, + pub prefix: String, + pub region: String, + #[serde(rename = "storageclass")] + pub storage_class: String, +} + +#[derive(Serialize, Deserialize, Default, Debug, Clone)] +#[serde(default)] +pub struct TierMinIO { + pub name: String, + pub endpoint: String, + #[serde(rename = "accesskey")] + pub access_key: String, + #[serde(rename = "secretkey")] + pub secret_key: String, + pub bucket: String, + pub prefix: String, + pub region: String, +} + +impl TierMinIO { + fn new( + name: &str, + endpoint: &str, + access_key: &str, + secret_key: &str, + bucket: &str, + options: Vec, + ) -> Result + where + F: Fn(TierMinIO) -> Box> + Send + Sync + 'static, + { + if name == "" { + return Err(std::io::Error::other(ERR_TIER_NAME_EMPTY)); + } + let m = TierMinIO { + access_key: access_key.to_string(), + secret_key: secret_key.to_string(), + bucket: bucket.to_string(), + endpoint: endpoint.to_string(), + ..Default::default() + }; + + for option in options { + let option = option(m.clone()); + let option = *option; + option?; + } + + Ok(TierConfig { + version: C_TierConfigVer.to_string(), + tier_type: TierType::MinIO, + name: name.to_string(), + minio: Some(m), + ..Default::default() + }) + } +} diff --git a/ecstore/src/tier/tier_config_gen.rs b/ecstore/src/tier/tier_config_gen.rs new file mode 100644 index 000000000..94f96161c --- /dev/null +++ b/ecstore/src/tier/tier_config_gen.rs @@ -0,0 +1,48 @@ +use std::{fmt::Display, pin::Pin, sync::Arc}; +use tracing::info; +use common::error::{Error, Result}; +use crate::bucket::tier_config::{TierType, TierConfig,}; + +impl TierType { + fn decode_msg(&self/*, dc *msgp.Reader*/) -> Result<()> { + todo!(); + } + + fn encode_msg(&self/*, en *msgp.Writer*/) -> Result<()> { + todo!(); + } + + pub fn marshal_msg(&self, b: &[u8]) -> Result> { + todo!(); + } + + pub fn unmarshal_msg(&self, bts: &[u8]) -> Result> { + todo!(); + } + + pub fn msg_size() -> usize { + todo!(); + } +} + +impl TierConfig { + fn decode_msg(&self, dc *msgp.Reader) -> Result<()> { + todo!(); + } + + pub fn encode_msg(&self, en *msgp.Writer) -> Result<()> { + todo!(); + } + + pub fn marshal_msg(&self, b: &[u8]) -> Result> { + todo!(); + } + + pub fn unmarshal_msg(&self, bts: &[u8]) -> Result> { + todo!(); + } + + fn msg_size(&self) -> usize { + todo!(); + } +} diff --git a/ecstore/src/tier/tier_gen.rs b/ecstore/src/tier/tier_gen.rs new file mode 100644 index 000000000..bbb5020e1 --- /dev/null +++ b/ecstore/src/tier/tier_gen.rs @@ -0,0 +1,23 @@ +use crate::tier::tier::TierConfigMgr; + +impl TierConfigMgr { + fn decode_msg(/*dc *msgp.Reader*/) -> Result<(), std::io::Error> { + todo!(); + } + + fn encode_msg(/*en *msgp.Writer*/) -> Result<(), std::io::Error> { + todo!(); + } + + pub fn marshal_msg(&self, b: &[u8]) -> Result, std::io::Error> { + todo!(); + } + + pub fn unmarshal_msg(buf: &[u8]) -> Result { + todo!(); + } + + pub fn msg_size(&self) -> usize { + 100 + } +} diff --git a/ecstore/src/tier/tier_handlers.rs b/ecstore/src/tier/tier_handlers.rs new file mode 100644 index 000000000..1f4ff510d --- /dev/null +++ b/ecstore/src/tier/tier_handlers.rs @@ -0,0 +1,51 @@ +use crate::client::admin_handler_utils::AdminError; +use http::status::StatusCode; +use tracing::warn; + +pub const ERR_TIER_ALREADY_EXISTS: AdminError = AdminError { + code: "XRustFSAdminTierAlreadyExists", + message: "Specified remote tier already exists", + status_code: StatusCode::CONFLICT, +}; + +pub const ERR_TIER_NOT_FOUND: AdminError = AdminError { + code: "XRustFSAdminTierNotFound", + message: "Specified remote tier was not found", + status_code: StatusCode::NOT_FOUND, +}; + +pub const ERR_TIER_NAME_NOT_UPPERCASE: AdminError = AdminError { + code: "XRustFSAdminTierNameNotUpperCase", + message: "Tier name must be in uppercase", + status_code: StatusCode::BAD_REQUEST, +}; + +pub const ERR_TIER_BUCKET_NOT_FOUND: AdminError = AdminError { + code: "XRustFSAdminTierBucketNotFound", + message: "Remote tier bucket not found", + status_code: StatusCode::BAD_REQUEST, +}; + +pub const ERR_TIER_INVALID_CREDENTIALS: AdminError = AdminError { + code: "XRustFSAdminTierInvalidCredentials", + message: "Invalid remote tier credentials", + status_code: StatusCode::BAD_REQUEST, +}; + +pub const ERR_TIER_RESERVED_NAME: AdminError = AdminError { + code: "XRustFSAdminTierReserved", + message: "Cannot use reserved tier name", + status_code: StatusCode::BAD_REQUEST, +}; + +pub const ERR_TIER_PERM_ERR: AdminError = AdminError { + code: "TierPermErr", + message: "Tier Perm Err", + status_code: StatusCode::OK, +}; + +pub const ERR_TIER_CONNECT_ERR: AdminError = AdminError { + code: "TierConnectErr", + message: "Tier Connect Err", + status_code: StatusCode::OK, +}; diff --git a/ecstore/src/tier/warm_backend.rs b/ecstore/src/tier/warm_backend.rs new file mode 100644 index 000000000..4bee0568e --- /dev/null +++ b/ecstore/src/tier/warm_backend.rs @@ -0,0 +1,105 @@ +use bytes::Bytes; +use std::collections::HashMap; + +use crate::client::{ + admin_handler_utils::AdminError, + transition_api::{ReadCloser, ReaderImpl}, +}; +use crate::error::is_err_bucket_not_found; +use crate::tier::{ + tier::{ERR_TIER_INVALID_CONFIG, ERR_TIER_TYPE_UNSUPPORTED}, + tier_config::{TierConfig, TierType}, + tier_handlers::{ERR_TIER_BUCKET_NOT_FOUND, ERR_TIER_PERM_ERR}, + warm_backend_minio::WarmBackendMinIO, + warm_backend_rustfs::WarmBackendRustFS, + warm_backend_s3::WarmBackendS3, +}; +use tracing::{info, warn}; + +pub type WarmBackendImpl = Box; + +const PROBE_OBJECT: &str = "probeobject"; + +#[derive(Default)] +pub struct WarmBackendGetOpts { + pub start_offset: i64, + pub length: i64, +} + +#[async_trait::async_trait] +pub trait WarmBackend { + async fn put(&self, object: &str, r: ReaderImpl, length: i64) -> Result; + async fn put_with_meta( + &self, + object: &str, + r: ReaderImpl, + length: i64, + meta: HashMap, + ) -> Result; + async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result; + async fn remove(&self, object: &str, rv: &str) -> Result<(), std::io::Error>; + async fn in_use(&self) -> Result; +} + +pub async fn check_warm_backend(w: Option<&WarmBackendImpl>) -> Result<(), AdminError> { + let w = w.expect("err"); + let remote_version_id = w + .put(PROBE_OBJECT, ReaderImpl::Body(Bytes::from("RustFS".as_bytes().to_vec())), 5) + .await; + if let Err(err) = remote_version_id { + return Err(ERR_TIER_PERM_ERR); + } + + let r = w.get(PROBE_OBJECT, "", WarmBackendGetOpts::default()).await; + //xhttp.DrainBody(r); + if let Err(err) = r { + //if is_err_bucket_not_found(&err) { + // return Err(ERR_TIER_BUCKET_NOT_FOUND); + //} + /*else if is_err_signature_does_not_match(err) { + return Err(ERR_TIER_MISSING_CREDENTIALS); + }*/ + //else { + return Err(ERR_TIER_PERM_ERR); + //} + } + if let Err(err) = w.remove(PROBE_OBJECT, &remote_version_id.expect("err")).await { + return Err(ERR_TIER_PERM_ERR); + }; + Ok(()) +} + +pub async fn new_warm_backend(tier: &TierConfig, probe: bool) -> Result { + let mut d: Option = None; + match tier.tier_type { + TierType::S3 => { + let dd = WarmBackendS3::new(tier.s3.as_ref().expect("err"), &tier.name).await; + if let Err(err) = dd { + info!("{}", err); + return Err(ERR_TIER_INVALID_CONFIG); + } + d = Some(Box::new(dd.expect("err"))); + } + TierType::RustFS => { + let dd = WarmBackendRustFS::new(tier.rustfs.as_ref().expect("err"), &tier.name).await; + if let Err(err) = dd { + warn!("{}", err); + return Err(ERR_TIER_INVALID_CONFIG); + } + d = Some(Box::new(dd.expect("err"))); + } + TierType::MinIO => { + let dd = WarmBackendMinIO::new(tier.minio.as_ref().expect("err"), &tier.name).await; + if let Err(err) = dd { + warn!("{}", err); + return Err(ERR_TIER_INVALID_CONFIG); + } + d = Some(Box::new(dd.expect("err"))); + } + _ => { + return Err(ERR_TIER_TYPE_UNSUPPORTED); + } + } + + Ok(d.expect("err")) +} diff --git a/ecstore/src/tier/warm_backend_minio.rs b/ecstore/src/tier/warm_backend_minio.rs new file mode 100644 index 000000000..f97232376 --- /dev/null +++ b/ecstore/src/tier/warm_backend_minio.rs @@ -0,0 +1,139 @@ +use std::collections::HashMap; +use std::sync::{Arc, Mutex}; + +use crate::client::{ + admin_handler_utils::AdminError, + api_put_object::PutObjectOptions, + credentials::{Credentials, SignatureType, Static, Value}, + transition_api::{Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore}, +}; +use crate::tier::{ + tier_config::TierMinIO, + warm_backend::{WarmBackend, WarmBackendGetOpts}, + warm_backend_s3::WarmBackendS3, +}; +use tracing::warn; + +const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5; +const MAX_PARTS_COUNT: i64 = 10000; +const MAX_PART_SIZE: i64 = 1024 * 1024 * 1024 * 5; +const MIN_PART_SIZE: i64 = 1024 * 1024 * 128; + +pub struct WarmBackendMinIO(WarmBackendS3); + +impl WarmBackendMinIO { + pub async fn new(conf: &TierMinIO, tier: &str) -> Result { + if conf.access_key == "" || conf.secret_key == "" { + return Err(std::io::Error::other("both access and secret keys are required")); + } + + if conf.bucket == "" { + return Err(std::io::Error::other("no bucket name was provided")); + } + + let u = match url::Url::parse(&conf.endpoint) { + Ok(u) => u, + Err(e) => { + return Err(std::io::Error::other(e.to_string())); + } + }; + + let creds = Credentials::new(Static(Value { + access_key_id: conf.access_key.clone(), + secret_access_key: conf.secret_key.clone(), + session_token: "".to_string(), + signer_type: SignatureType::SignatureV4, + ..Default::default() + })); + let opts = Options { + creds: creds, + secure: u.scheme() == "https", + //transport: GLOBAL_RemoteTargetTransport, + trailing_headers: true, + ..Default::default() + }; + let scheme = u.scheme(); + let default_port = if scheme == "https" { 443 } else { 80 }; + let client = + TransitionClient::new(&format!("{}:{}", u.host_str().expect("err"), u.port().unwrap_or(default_port)), opts).await?; + //client.set_appinfo(format!("minio-tier-{}", tier), ReleaseTag); + + let client = Arc::new(client); + let core = TransitionCore(Arc::clone(&client)); + Ok(Self(WarmBackendS3 { + client, + core, + bucket: conf.bucket.clone(), + prefix: conf.prefix.strip_suffix("/").unwrap_or(&conf.prefix).to_owned(), + storage_class: "".to_string(), + })) + } +} + +#[async_trait::async_trait] +impl WarmBackend for WarmBackendMinIO { + async fn put_with_meta( + &self, + object: &str, + r: ReaderImpl, + length: i64, + meta: HashMap, + ) -> Result { + let part_size = optimal_part_size(length)?; + let client = self.0.client.clone(); + let res = client + .put_object( + &self.0.bucket, + &self.0.get_dest(object), + r, + length, + &PutObjectOptions { + storage_class: self.0.storage_class.clone(), + part_size: part_size as u64, + disable_content_sha256: true, + user_metadata: meta, + ..Default::default() + }, + ) + .await?; + //self.ToObjectError(err, object) + Ok(res.version_id) + } + + async fn put(&self, object: &str, r: ReaderImpl, length: i64) -> Result { + self.put_with_meta(object, r, length, HashMap::new()).await + } + + async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result { + self.0.get(object, rv, opts).await + } + + async fn remove(&self, object: &str, rv: &str) -> Result<(), std::io::Error> { + self.0.remove(object, rv).await + } + + async fn in_use(&self) -> Result { + self.0.in_use().await + } +} + +fn optimal_part_size(object_size: i64) -> Result { + let mut object_size = object_size; + if object_size == -1 { + object_size = MAX_MULTIPART_PUT_OBJECT_SIZE; + } + + if object_size > MAX_MULTIPART_PUT_OBJECT_SIZE { + return Err(std::io::Error::other("entity too large")); + } + + let configured_part_size = MIN_PART_SIZE; + let mut part_size_flt = object_size as f64 / MAX_PARTS_COUNT as f64; + part_size_flt = (part_size_flt as f64 / configured_part_size as f64).ceil() * configured_part_size as f64; + + let part_size = part_size_flt as i64; + if part_size == 0 { + return Ok(MIN_PART_SIZE); + } + Ok(part_size) +} diff --git a/ecstore/src/tier/warm_backend_rustfs.rs b/ecstore/src/tier/warm_backend_rustfs.rs new file mode 100644 index 000000000..f0e12bb66 --- /dev/null +++ b/ecstore/src/tier/warm_backend_rustfs.rs @@ -0,0 +1,137 @@ +use std::collections::HashMap; +use std::sync::{Arc, Mutex}; +use tracing::warn; + +use crate::client::{ + admin_handler_utils::AdminError, + api_put_object::PutObjectOptions, + credentials::{Credentials, SignatureType, Static, Value}, + transition_api::{Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore}, +}; +use crate::tier::{ + tier_config::TierRustFS, + warm_backend::{WarmBackend, WarmBackendGetOpts}, + warm_backend_s3::WarmBackendS3, +}; + +const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5; +const MAX_PARTS_COUNT: i64 = 10000; +const MAX_PART_SIZE: i64 = 1024 * 1024 * 1024 * 5; +const MIN_PART_SIZE: i64 = 1024 * 1024 * 128; + +pub struct WarmBackendRustFS(WarmBackendS3); + +impl WarmBackendRustFS { + pub async fn new(conf: &TierRustFS, tier: &str) -> Result { + if conf.access_key == "" || conf.secret_key == "" { + return Err(std::io::Error::other("both access and secret keys are required")); + } + + if conf.bucket == "" { + return Err(std::io::Error::other("no bucket name was provided")); + } + + let u = match url::Url::parse(&conf.endpoint) { + Ok(u) => u, + Err(e) => return Err(std::io::Error::other(e)), + }; + + let creds = Credentials::new(Static(Value { + access_key_id: conf.access_key.clone(), + secret_access_key: conf.secret_key.clone(), + session_token: "".to_string(), + signer_type: SignatureType::SignatureV4, + ..Default::default() + })); + let opts = Options { + creds: creds, + secure: u.scheme() == "https", + //transport: GLOBAL_RemoteTargetTransport, + trailing_headers: true, + ..Default::default() + }; + let scheme = u.scheme(); + let default_port = if scheme == "https" { 443 } else { 80 }; + let client = + TransitionClient::new(&format!("{}:{}", u.host_str().expect("err"), u.port().unwrap_or(default_port)), opts).await?; + //client.set_appinfo(format!("rustfs-tier-{}", tier), ReleaseTag); + + let client = Arc::new(client); + let core = TransitionCore(Arc::clone(&client)); + Ok(Self(WarmBackendS3 { + client, + core, + bucket: conf.bucket.clone(), + prefix: conf.prefix.strip_suffix("/").unwrap_or(&conf.prefix).to_owned(), + storage_class: "".to_string(), + })) + } +} + +#[async_trait::async_trait] +impl WarmBackend for WarmBackendRustFS { + async fn put_with_meta( + &self, + object: &str, + r: ReaderImpl, + length: i64, + meta: HashMap, + ) -> Result { + let part_size = optimal_part_size(length)?; + let client = self.0.client.clone(); + let res = client + .put_object( + &self.0.bucket, + &self.0.get_dest(object), + r, + length, + &PutObjectOptions { + storage_class: self.0.storage_class.clone(), + part_size: part_size as u64, + disable_content_sha256: true, + user_metadata: meta, + ..Default::default() + }, + ) + .await?; + //self.ToObjectError(err, object) + Ok(res.version_id) + } + + async fn put(&self, object: &str, r: ReaderImpl, length: i64) -> Result { + self.put_with_meta(object, r, length, HashMap::new()).await + } + + async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result { + self.0.get(object, rv, opts).await + } + + async fn remove(&self, object: &str, rv: &str) -> Result<(), std::io::Error> { + self.0.remove(object, rv).await + } + + async fn in_use(&self) -> Result { + self.0.in_use().await + } +} + +fn optimal_part_size(object_size: i64) -> Result { + let mut object_size = object_size; + if object_size == -1 { + object_size = MAX_MULTIPART_PUT_OBJECT_SIZE; + } + + if object_size > MAX_MULTIPART_PUT_OBJECT_SIZE { + return Err(std::io::Error::other("entity too large")); + } + + let configured_part_size = MIN_PART_SIZE; + let mut part_size_flt = object_size as f64 / MAX_PARTS_COUNT as f64; + part_size_flt = (part_size_flt as f64 / configured_part_size as f64).ceil() * configured_part_size as f64; + + let part_size = part_size_flt as i64; + if part_size == 0 { + return Ok(MIN_PART_SIZE); + } + Ok(part_size) +} diff --git a/ecstore/src/tier/warm_backend_s3.rs b/ecstore/src/tier/warm_backend_s3.rs new file mode 100644 index 000000000..1dda71acd --- /dev/null +++ b/ecstore/src/tier/warm_backend_s3.rs @@ -0,0 +1,175 @@ +use std::collections::HashMap; +use std::sync::Arc; +use url::Url; + +use crate::client::{ + api_get_options::GetObjectOptions, + api_put_object::PutObjectOptions, + api_remove::RemoveObjectOptions, + credentials::{Credentials, SignatureType, Static, Value}, + transition_api::{Options, TransitionClient, TransitionCore}, + transition_api::{ReadCloser, ReaderImpl}, +}; +use crate::error::ErrorResponse; +use crate::error::error_resp_to_object_err; +use crate::tier::{ + tier_config::TierS3, + warm_backend::{WarmBackend, WarmBackendGetOpts}, +}; +use rustfs_utils::path::SLASH_SEPARATOR; + +pub struct WarmBackendS3 { + pub client: Arc, + pub core: TransitionCore, + pub bucket: String, + pub prefix: String, + pub storage_class: String, +} + +impl WarmBackendS3 { + pub async fn new(conf: &TierS3, tier: &str) -> Result { + let u = match Url::parse(&conf.endpoint) { + Ok(u) => u, + Err(err) => { + return Err(std::io::Error::other(err.to_string())); + } + }; + + if conf.aws_role_web_identity_token_file == "" && conf.aws_role_arn != "" + || conf.aws_role_web_identity_token_file != "" && conf.aws_role_arn == "" + { + return Err(std::io::Error::other("both the token file and the role ARN are required")); + } else if conf.access_key == "" && conf.secret_key != "" || conf.access_key != "" && conf.secret_key == "" { + return Err(std::io::Error::other("both the access and secret keys are required")); + } else if conf.aws_role + && (conf.aws_role_web_identity_token_file != "" + || conf.aws_role_arn != "" + || conf.access_key != "" + || conf.secret_key != "") + { + return Err(std::io::Error::other( + "AWS Role cannot be activated with static credentials or the web identity token file", + )); + } else if conf.bucket == "" { + return Err(std::io::Error::other("no bucket name was provided")); + } + + let mut creds: Credentials; + + if conf.access_key != "" && conf.secret_key != "" { + //creds = Credentials::new_static_v4(conf.access_key, conf.secret_key, ""); + creds = Credentials::new(Static(Value { + access_key_id: conf.access_key.clone(), + secret_access_key: conf.secret_key.clone(), + session_token: "".to_string(), + signer_type: SignatureType::SignatureV4, + ..Default::default() + })); + } else { + return Err(std::io::Error::other("insufficient parameters for S3 backend authentication")); + } + let opts = Options { + creds: creds, + secure: u.scheme() == "https", + //transport: GLOBAL_RemoteTargetTransport, + ..Default::default() + }; + let client = TransitionClient::new(&u.host().expect("err").to_string(), opts).await?; + //client.set_appinfo(format!("s3-tier-{}", tier), ReleaseTag); + + let client = Arc::new(client); + let core = TransitionCore(Arc::clone(&client)); + Ok(Self { + client, + core, + bucket: conf.bucket.clone(), + prefix: conf.prefix.clone().trim_matches('/').to_string(), + storage_class: conf.storage_class.clone(), + }) + } + + fn to_object_err(&self, err: ErrorResponse, params: Vec<&str>) -> std::io::Error { + let mut object = ""; + if params.len() >= 1 { + object = params.first().cloned().unwrap_or_default(); + } + + error_resp_to_object_err(err, vec![&self.bucket, &self.get_dest(object)]) + } + + pub fn get_dest(&self, object: &str) -> String { + let mut dest_obj = object.to_string(); + if self.prefix != "" { + dest_obj = format!("{}/{}", &self.prefix, object); + } + return dest_obj; + } +} + +#[async_trait::async_trait] +impl WarmBackend for WarmBackendS3 { + async fn put_with_meta( + &self, + object: &str, + r: ReaderImpl, + length: i64, + meta: HashMap, + ) -> Result { + let client = self.client.clone(); + let res = client + .put_object( + &self.bucket, + &self.get_dest(object), + r, + length, + &PutObjectOptions { + send_content_md5: true, + storage_class: self.storage_class.clone(), + user_metadata: meta, + ..Default::default() + }, + ) + .await?; + Ok(res.version_id) + } + + async fn put(&self, object: &str, r: ReaderImpl, length: i64) -> Result { + self.put_with_meta(object, r, length, HashMap::new()).await + } + + async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result { + let mut gopts = GetObjectOptions::default(); + + if rv != "" { + gopts.version_id = rv.to_string(); + } + if opts.start_offset >= 0 && opts.length > 0 { + if let Err(err) = gopts.set_range(opts.start_offset, opts.start_offset + opts.length - 1) { + return Err(std::io::Error::other(err)); + } + } + let c = TransitionCore(Arc::clone(&self.client)); + let (_, _, r) = c.get_object(&self.bucket, &self.get_dest(object), &gopts).await?; + + Ok(r) + } + + async fn remove(&self, object: &str, rv: &str) -> Result<(), std::io::Error> { + let mut ropts = RemoveObjectOptions::default(); + if rv != "" { + ropts.version_id = rv.to_string(); + } + let client = self.client.clone(); + let err = client.remove_object(&self.bucket, &self.get_dest(object), ropts).await; + Err(std::io::Error::other(err.expect("err"))) + } + + async fn in_use(&self) -> Result { + let result = self + .core + .list_objects_v2(&self.bucket, &self.prefix, "", "", SLASH_SEPARATOR, 1) + .await?; + + Ok(result.common_prefixes.len() > 0 || result.contents.len() > 0) + } +} diff --git a/reader/Cargo.toml b/reader/Cargo.toml new file mode 100644 index 000000000..6ad7c7acb --- /dev/null +++ b/reader/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "reader" +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +version.workspace = true + +[lints] +workspace = true + +[dependencies] +tracing.workspace = true +s3s.workspace = true +thiserror.workspace = true +bytes.workspace = true +pin-project-lite.workspace = true +hex-simd = "0.8.0" +base64-simd = "0.8.0" +md-5.workspace = true +sha2 = { version = "0.11.0-pre.4" } +futures.workspace = true +async-trait.workspace = true +common.workspace = true + +[dev-dependencies] +tokio = { workspace = true, features = ["rt-multi-thread", "macros"] } diff --git a/reader/src/error.rs b/reader/src/error.rs new file mode 100644 index 000000000..9c5017ee3 --- /dev/null +++ b/reader/src/error.rs @@ -0,0 +1,12 @@ +#[derive(Debug, thiserror::Error, PartialEq, Eq)] +pub enum ReaderError { + #[error("stream input error {0}")] + StreamInput(String), + // + #[error("etag: expected ETag {0} does not match computed ETag {1}")] + VerifyError(String, String), + #[error("Bad checksum: Want {0} does not match calculated {1}")] + ChecksumMismatch(String, String), + #[error("Bad sha256: Expected {0} does not match calculated {1}")] + SHA256Mismatch(String, String), +} diff --git a/reader/src/hasher.rs b/reader/src/hasher.rs new file mode 100644 index 000000000..5f86a52fb --- /dev/null +++ b/reader/src/hasher.rs @@ -0,0 +1,184 @@ +use md5::{Digest as Md5Digest, Md5}; +use sha2::{ + Digest, Sha256 as sha_sha256, + digest::{Reset, Update}, +}; +pub trait Hasher { + fn write(&mut self, bytes: &[u8]); + fn reset(&mut self); + fn sum(&mut self) -> String; + fn size(&self) -> usize; + fn block_size(&self) -> usize; +} + +#[derive(Default)] +pub enum HashType { + #[default] + Undefined, + Uuid(Uuid), + Md5(MD5), + Sha256(Sha256), +} + +impl Hasher for HashType { + fn write(&mut self, bytes: &[u8]) { + match self { + HashType::Md5(md5) => md5.write(bytes), + HashType::Sha256(sha256) => sha256.write(bytes), + HashType::Uuid(uuid) => uuid.write(bytes), + HashType::Undefined => (), + } + } + + fn reset(&mut self) { + match self { + HashType::Md5(md5) => md5.reset(), + HashType::Sha256(sha256) => sha256.reset(), + HashType::Uuid(uuid) => uuid.reset(), + HashType::Undefined => (), + } + } + + fn sum(&mut self) -> String { + match self { + HashType::Md5(md5) => md5.sum(), + HashType::Sha256(sha256) => sha256.sum(), + HashType::Uuid(uuid) => uuid.sum(), + HashType::Undefined => "".to_owned(), + } + } + + fn size(&self) -> usize { + match self { + HashType::Md5(md5) => md5.size(), + HashType::Sha256(sha256) => sha256.size(), + HashType::Uuid(uuid) => uuid.size(), + HashType::Undefined => 0, + } + } + + fn block_size(&self) -> usize { + match self { + HashType::Md5(md5) => md5.block_size(), + HashType::Sha256(sha256) => sha256.block_size(), + HashType::Uuid(uuid) => uuid.block_size(), + HashType::Undefined => 64, + } + } +} + +#[derive(Debug)] +pub struct Sha256 { + hasher: sha_sha256, +} + +impl Sha256 { + pub fn new() -> Self { + Self { + hasher: sha_sha256::new(), + } + } +} +impl Default for Sha256 { + fn default() -> Self { + Self::new() + } +} + +impl Hasher for Sha256 { + fn write(&mut self, bytes: &[u8]) { + Update::update(&mut self.hasher, bytes); + } + + fn reset(&mut self) { + Reset::reset(&mut self.hasher); + } + + fn sum(&mut self) -> String { + hex_simd::encode_to_string(self.hasher.clone().finalize(), hex_simd::AsciiCase::Lower) + } + + fn size(&self) -> usize { + 32 + } + + fn block_size(&self) -> usize { + 64 + } +} + +#[derive(Debug)] +pub struct MD5 { + hasher: Md5, +} + +impl MD5 { + pub fn new() -> Self { + Self { hasher: Md5::new() } + } +} +impl Default for MD5 { + fn default() -> Self { + Self::new() + } +} + +impl Hasher for MD5 { + fn write(&mut self, bytes: &[u8]) { + self.hasher.update(bytes); + } + + fn reset(&mut self) {} + + fn sum(&mut self) -> String { + hex_simd::encode_to_string(self.hasher.clone().finalize(), hex_simd::AsciiCase::Lower) + } + + fn size(&self) -> usize { + 32 + } + + fn block_size(&self) -> usize { + 64 + } +} + +pub struct Uuid { + id: String, +} + +impl Uuid { + pub fn new(id: String) -> Self { + Self { id } + } +} + +impl Hasher for Uuid { + fn write(&mut self, _bytes: &[u8]) {} + + fn reset(&mut self) {} + + fn sum(&mut self) -> String { + self.id.clone() + } + + fn size(&self) -> usize { + self.id.len() + } + + fn block_size(&self) -> usize { + 64 + } +} + +pub fn sum_sha256_hex(data: &[u8]) -> String { + let mut hash = Sha256::new(); + hash.write(data); + base64_simd::URL_SAFE_NO_PAD.encode_to_string(hash.sum()) +} + +pub fn sum_md5_base64(data: &[u8]) -> String { + let mut hash = MD5::new(); + hash.write(data); + base64_simd::URL_SAFE_NO_PAD.encode_to_string(hash.sum()) +} diff --git a/reader/src/lib.rs b/reader/src/lib.rs new file mode 100644 index 000000000..433caaa20 --- /dev/null +++ b/reader/src/lib.rs @@ -0,0 +1,7 @@ +pub mod error; +pub mod hasher; +pub mod reader; + +pub fn hex(data: impl AsRef<[u8]>) -> String { + hex_simd::encode_to_string(data, hex_simd::AsciiCase::Lower) +} diff --git a/reader/src/reader.rs b/reader/src/reader.rs new file mode 100644 index 000000000..e68a64aed --- /dev/null +++ b/reader/src/reader.rs @@ -0,0 +1,547 @@ +use bytes::Bytes; +use s3s::StdError; +use std::any::Any; +use std::io::Read; +use std::{collections::VecDeque, io::Cursor}; + +use std::pin::Pin; +use std::task::Poll; + +use crate::{ + error::ReaderError, + hasher::{HashType, Uuid}, +}; + +// use futures::stream::Stream; +use super::hasher::{Hasher, MD5, Sha256}; +use futures::Stream; +use std::io::{Error, Result}; + +pin_project_lite::pin_project! { + #[derive(Default)] + pub struct EtagReader { + #[pin] + inner: S, + md5: HashType, + checksum:Option, + bytes_read:usize, + } +} + +impl EtagReader { + pub fn new(inner: S, etag: Option, force_md5: Option) -> Self { + let md5 = { + if let Some(m) = force_md5 { + HashType::Uuid(Uuid::new(m)) + } else { + HashType::Md5(MD5::new()) + } + }; + Self { + inner, + md5, + checksum: etag, + bytes_read: 0, + } + } + + pub fn etag(&mut self) -> String { + self.md5.sum() + } +} + +impl Stream for EtagReader +where + S: Stream>, +{ + type Item = std::result::Result; + + fn poll_next(self: Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> Poll> { + let this = self.project(); + let poll = this.inner.poll_next(cx); + + if let Poll::Ready(ref res) = poll { + match res { + Some(Ok(bytes)) => { + *this.bytes_read += bytes.len(); + this.md5.write(bytes); + } + Some(Err(err)) => { + return Poll::Ready(Some(Err(Box::new(ReaderError::StreamInput(err.to_string()))))); + } + None => { + if let Some(etag) = this.checksum { + let got = this.md5.sum(); + if got.as_str() != etag.as_str() { + return Poll::Ready(Some(Err(Box::new(ReaderError::VerifyError(etag.to_owned(), got))))); + } + } + } + } + } + + poll + } +} + +pin_project_lite::pin_project! { + #[derive(Default)] + pub struct HashReader { + #[pin] + inner: S, + sha256: Option, + md5: Option, + md5_hex:Option, + sha256_hex:Option, + size:usize, + actual_size: usize, + bytes_read:usize, + } +} + +impl HashReader { + pub fn new(inner: S, size: usize, md5_hex: Option, sha256_hex: Option, actual_size: usize) -> Self { + let md5 = { if md5_hex.is_some() { Some(MD5::new()) } else { None } }; + let sha256 = { if sha256_hex.is_some() { Some(Sha256::new()) } else { None } }; + Self { + inner, + size, + actual_size, + md5_hex, + sha256_hex, + bytes_read: 0, + md5, + sha256, + } + } +} + +impl Stream for HashReader +where + S: Stream>, +{ + type Item = std::result::Result; + + fn poll_next(self: Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> Poll> { + let this = self.project(); + let poll = this.inner.poll_next(cx); + + if let Poll::Ready(ref res) = poll { + match res { + Some(Ok(bytes)) => { + *this.bytes_read += bytes.len(); + if let Some(sha) = this.sha256 { + sha.write(bytes); + } + + if let Some(md5) = this.md5 { + md5.write(bytes); + } + } + Some(Err(err)) => { + return Poll::Ready(Some(Err(Box::new(ReaderError::StreamInput(err.to_string()))))); + } + None => { + if let Some(hash) = this.sha256 { + if let Some(hex) = this.sha256_hex { + let got = hash.sum(); + let src = hex.as_str(); + if src != got.as_str() { + println!("sha256 err src:{},got:{}", src, got); + return Poll::Ready(Some(Err(Box::new(ReaderError::SHA256Mismatch(src.to_string(), got))))); + } + } + } + + if let Some(hash) = this.md5 { + if let Some(hex) = this.md5_hex { + let got = hash.sum(); + let src = hex.as_str(); + if src != got.as_str() { + // TODO: ERR + println!("md5 err src:{},got:{}", src, got); + return Poll::Ready(Some(Err(Box::new(ReaderError::ChecksumMismatch(src.to_string(), got))))); + } + } + } + } + } + } + + // println!("poll {:?}", poll); + + poll + } +} + +#[async_trait::async_trait] +pub trait Reader { + async fn read_at(&mut self, offset: usize, buf: &mut [u8]) -> Result; + async fn seek(&mut self, offset: usize) -> Result<()>; + async fn read_exact(&mut self, buf: &mut [u8]) -> Result; + async fn read_all(&mut self) -> Result> { + let mut data = Vec::new(); + + Ok(data) + } + fn as_any(&self) -> &dyn Any; +} + +#[derive(Debug)] +pub struct BufferReader { + pub inner: Cursor>, + pos: usize, +} + +impl BufferReader { + pub fn new(inner: Vec) -> Self { + Self { + inner: Cursor::new(inner), + pos: 0, + } + } +} + +#[async_trait::async_trait] +impl Reader for BufferReader { + #[tracing::instrument(level = "debug", skip(self, buf))] + async fn read_at(&mut self, offset: usize, buf: &mut [u8]) -> Result { + self.seek(offset).await?; + self.read_exact(buf).await + } + #[tracing::instrument(level = "debug", skip(self))] + async fn seek(&mut self, offset: usize) -> Result<()> { + if self.pos != offset { + self.inner.set_position(offset as u64); + } + + Ok(()) + } + #[tracing::instrument(level = "debug", skip(self))] + async fn read_exact(&mut self, buf: &mut [u8]) -> Result { + let bytes_read = self.inner.read_exact(buf)?; + self.pos += buf.len(); + //Ok(bytes_read) + Ok(0) + } + + async fn read_all(&mut self) -> Result> { + let mut data = Vec::new(); + self.inner.read_to_end(&mut data)?; + + Ok(data) + } + + fn as_any(&self) -> &dyn Any { + self + } +} + +pin_project_lite::pin_project! { + pub struct ChunkedStream { + #[pin] + inner: S, + chuck_size: usize, + streams: VecDeque, + remaining:Vec, + } +} + +impl ChunkedStream { + pub fn new(inner: S, chuck_size: usize) -> Self { + Self { + inner, + chuck_size, + streams: VecDeque::new(), + remaining: Vec::new(), + } + } +} + +impl Stream for ChunkedStream +where + S: Stream> + Send + Sync, + // E: std::error::Error + Send + Sync, +{ + type Item = std::result::Result; + + fn poll_next(self: Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> Poll> { + let (items, op_items) = self.inner.size_hint(); + let this = self.project(); + + if let Some(b) = this.streams.pop_front() { + return Poll::Ready(Some(Ok(b))); + } + + let poll = this.inner.poll_next(cx); + + match poll { + Poll::Ready(res_op) => match res_op { + Some(res) => match res { + Ok(bytes) => { + let chuck_size = *this.chuck_size; + let mut bytes = bytes; + + // println!("get len {}", bytes.len()); + // 如果有剩余 + if !this.remaining.is_empty() { + let need_size = chuck_size - this.remaining.len(); + // 传入的数据大小需要补齐的大小,使用传入数据补齐 + if bytes.len() >= need_size { + let add_bytes = bytes.split_to(need_size); + this.remaining.extend_from_slice(&add_bytes); + this.streams.push_back(Bytes::from(this.remaining.clone())); + this.remaining.clear(); + } else { + // 不够,直接追加 + let need_size = bytes.len(); + let add_bytes = bytes.split_to(need_size); + this.remaining.extend_from_slice(&add_bytes); + } + } + + loop { + if bytes.len() < chuck_size { + break; + } + let chuck = bytes.split_to(chuck_size); + this.streams.push_back(chuck); + } + + if !bytes.is_empty() { + this.remaining.extend_from_slice(&bytes); + } + + if let Some(b) = this.streams.pop_front() { + return Poll::Ready(Some(Ok(b))); + } + + if items > 0 || op_items.is_some() { + return Poll::Pending; + } + + if !this.remaining.is_empty() { + let b = this.remaining.clone(); + this.remaining.clear(); + return Poll::Ready(Some(Ok(Bytes::from(b)))); + } + Poll::Ready(None) + } + Err(err) => Poll::Ready(Some(Err(err))), + }, + None => { + // println!("get empty"); + if let Some(b) = this.streams.pop_front() { + return Poll::Ready(Some(Ok(b))); + } + if !this.remaining.is_empty() { + let b = this.remaining.clone(); + this.remaining.clear(); + return Poll::Ready(Some(Ok(Bytes::from(b)))); + } + Poll::Ready(None) + } + }, + Poll::Pending => { + // println!("get Pending"); + Poll::Pending + } + } + + // if let Poll::Ready(Some(res)) = poll { + // warn!("poll res ..."); + // match res { + // Ok(bytes) => { + // let chuck_size = *this.chuck_size; + // let mut bytes = bytes; + // if this.remaining.len() > 0 { + // let need_size = chuck_size - this.remaining.len(); + // let add_bytes = bytes.split_to(need_size); + // this.remaining.extend_from_slice(&add_bytes); + // warn!("poll push_back remaining ...1"); + // this.streams.push_back(Bytes::from(this.remaining.clone())); + // this.remaining.clear(); + // } + + // loop { + // if bytes.len() < chuck_size { + // break; + // } + // let chuck = bytes.split_to(chuck_size); + // warn!("poll push_back ...1"); + // this.streams.push_back(chuck); + // } + + // warn!("poll remaining extend_from_slice...1"); + // this.remaining.extend_from_slice(&bytes); + // } + // Err(err) => return Poll::Ready(Some(Err(err))), + // } + // } + + // if let Some(b) = this.streams.pop_front() { + // warn!("poll pop_front ..."); + // return Poll::Ready(Some(Ok(b))); + // } + + // if this.remaining.len() > 0 { + // let b = this.remaining.clone(); + // this.remaining.clear(); + + // warn!("poll remaining ...1"); + // return Poll::Ready(Some(Ok(Bytes::from(b)))); + // } + // Poll::Pending + } + + fn size_hint(&self) -> (usize, Option) { + let mut items = self.streams.len(); + if !self.remaining.is_empty() { + items += 1; + } + (items, Some(items)) + } +} + +#[cfg(test)] +mod test { + + use super::*; + use futures::StreamExt; + + #[tokio::test] + async fn test_etag_reader() { + let data1 = vec![1u8; 60]; // 65536 + let data2 = vec![0u8; 32]; // 65536 + let chunk1 = Bytes::from(data1); + let chunk2 = Bytes::from(data2); + + let chunk_results: Vec> = vec![Ok(chunk1), Ok(chunk2)]; + + let mut stream = futures::stream::iter(chunk_results); + + let mut hash_reader = EtagReader::new(&mut stream, None, None); + + // let chunk_size = 8; + + // let mut chunked_stream = ChunkStream::new(&mut hash_reader, chunk_size); + + loop { + match hash_reader.next().await { + Some(res) => match res { + Ok(bytes) => { + println!("bytes: {}, {:?}", bytes.len(), bytes); + } + Err(err) => { + println!("err:{:?}", err); + break; + } + }, + None => { + println!("next none"); + break; + } + } + } + + println!("etag:{}", hash_reader.etag()); + + // 9a7dfa2fcd7b69c89a30cfd3a9be11ab58cb6172628bd7e967fad1e187456d45 + // println!("md5: {:?}", hash_reader.hex()); + } + + #[tokio::test] + async fn test_hash_reader() { + let data1 = vec![1u8; 60]; // 65536 + let data2 = vec![0u8; 32]; // 65536 + let size = data1.len() + data2.len(); + let chunk1 = Bytes::from(data1); + let chunk2 = Bytes::from(data2); + + let chunk_results: Vec> = vec![Ok(chunk1), Ok(chunk2)]; + + let mut stream = futures::stream::iter(chunk_results); + + let mut hash_reader = HashReader::new( + &mut stream, + size, + Some("d94c485610a7a00a574df55e45d3cc0c".to_string()), + Some("9a7dfa2fcd7b69c89a30cfd3a9be11ab58cb6172628bd7e967fad1e187456d45".to_string()), + 0, + ); + + // let chunk_size = 8; + + // let mut chunked_stream = ChunkStream::new(&mut hash_reader, chunk_size); + + loop { + match hash_reader.next().await { + Some(res) => match res { + Ok(bytes) => { + println!("bytes: {}, {:?}", bytes.len(), bytes); + } + Err(err) => { + println!("err:{:?}", err); + break; + } + }, + None => { + println!("next none"); + break; + } + } + } + + // BUG: borrow of moved value: `md5_stream` + + // 9a7dfa2fcd7b69c89a30cfd3a9be11ab58cb6172628bd7e967fad1e187456d45 + // println!("md5: {:?}", hash_reader.hex()); + } + + #[tokio::test] + async fn test_chunked_stream() { + let data1 = vec![1u8; 60]; // 65536 + let data2 = vec![0u8; 33]; // 65536 + let data3 = vec![4u8; 5]; // 65536 + let chunk1 = Bytes::from(data1); + let chunk2 = Bytes::from(data2); + let chunk3 = Bytes::from(data3); + + let chunk_results: Vec> = vec![Ok(chunk1), Ok(chunk2), Ok(chunk3)]; + + let mut stream = futures::stream::iter(chunk_results); + // let mut hash_reader = HashReader::new( + // &mut stream, + // size, + // Some("d94c485610a7a00a574df55e45d3cc0c".to_string()), + // Some("9a7dfa2fcd7b69c89a30cfd3a9be11ab58cb6172628bd7e967fad1e187456d45".to_string()), + // 0, + // ); + + let chunk_size = 8; + + let mut etag_reader = EtagReader::new(&mut stream, None, None); + + let mut chunked_stream = ChunkedStream::new(&mut etag_reader, chunk_size); + + loop { + match chunked_stream.next().await { + Some(res) => match res { + Ok(bytes) => { + println!("bytes: {}, {:?}", bytes.len(), bytes); + } + Err(err) => { + println!("err:{:?}", err); + break; + } + }, + None => { + println!("next none"); + break; + } + } + } + + println!("etag:{}", etag_reader.etag()); + } +} diff --git a/rustfs/src/admin/handlers.rs b/rustfs/src/admin/handlers.rs index f59b32e5d..75dcfa383 100644 --- a/rustfs/src/admin/handlers.rs +++ b/rustfs/src/admin/handlers.rs @@ -63,6 +63,7 @@ pub mod pools; pub mod rebalance; pub mod service_account; pub mod sts; +pub mod tier; pub mod trace; pub mod user; use urlencoding::decode; diff --git a/rustfs/src/admin/handlers/tier.rs b/rustfs/src/admin/handlers/tier.rs new file mode 100644 index 000000000..a11a92e71 --- /dev/null +++ b/rustfs/src/admin/handlers/tier.rs @@ -0,0 +1,385 @@ +use std::str::from_utf8; + +use http::{HeaderMap, StatusCode}; +use iam::get_global_action_cred; +use matchit::Params; +use s3s::{Body, S3Error, S3ErrorCode, S3Request, S3Response, S3Result, header::CONTENT_TYPE, s3_error}; +use serde::Deserialize; +use serde_urlencoded::from_bytes; +use tracing::{debug, info, warn}; + +use crate::{ + admin::{router::Operation, utils::has_space_be}, + auth::{check_key_valid, get_session_token}, +}; +use crypto::{decrypt_data, encrypt_data}; +use ecstore::{ + client::admin_handler_utils::AdminError, + config::storageclass, + global::GLOBAL_TierConfigMgr, + tier::{ + tier::{ERR_TIER_BACKEND_IN_USE, ERR_TIER_BACKEND_NOT_EMPTY, ERR_TIER_MISSING_CREDENTIALS}, + tier_admin::TierCreds, + tier_config::{TierConfig, TierType}, + tier_handlers::{ + ERR_TIER_ALREADY_EXISTS, ERR_TIER_CONNECT_ERR, ERR_TIER_INVALID_CREDENTIALS, ERR_TIER_NAME_NOT_UPPERCASE, + ERR_TIER_NOT_FOUND, + }, + }, +}; + +#[derive(Debug, Deserialize, Default)] +pub struct AddTierQuery { + #[serde(rename = "accessKey")] + pub access_key: Option, + pub status: Option, + pub tier: Option, + pub force: String, +} + +pub struct AddTier {} +#[async_trait::async_trait] +impl Operation for AddTier { + #[tracing::instrument(level = "debug", skip(self))] + async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { + let query = { + if let Some(query) = req.uri.query() { + let input: AddTierQuery = + from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "get body failed1"))?; + input + } else { + AddTierQuery::default() + } + }; + + let Some(input_cred) = req.credentials else { + return Err(s3_error!(InvalidRequest, "get cred failed")); + }; + + let (cred, _owner) = + check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?; + + let mut input = req.input; + let body = match input.store_all_unlimited().await { + Ok(b) => b, + Err(e) => { + warn!("get body failed, e: {:?}", e); + return Err(s3_error!(InvalidRequest, "get body failed")); + } + }; + + let mut args: TierConfig = serde_json::from_slice(&body) + .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("unmarshal body err {}", e)))?; + + match args.tier_type { + TierType::S3 => { + args.name = args.s3.clone().unwrap().name; + } + TierType::RustFS => { + args.name = args.rustfs.clone().unwrap().name; + } + TierType::MinIO => { + args.name = args.minio.clone().unwrap().name; + } + _ => (), + } + debug!("add tier args {:?}", args); + + let mut force: bool = false; + let force_str = query.force; + if force_str != "" { + force = force_str.parse().unwrap(); + } + match args.name.as_str() { + storageclass::STANDARD | storageclass::RRS => { + warn!("tier reserved name, args.name: {}", args.name); + return Err(s3_error!(InvalidRequest, "Cannot use reserved tier name")); + } + &_ => (), + } + + let mut tier_config_mgr = GLOBAL_TierConfigMgr.write().await; + //tier_config_mgr.reload(api); + match tier_config_mgr.add(args, force).await { + Err(ERR_TIER_ALREADY_EXISTS) => { + return Err(S3Error::with_message( + S3ErrorCode::Custom("TierNameAlreadyExist".into()), + "tier name already exists!", + )); + } + Err(ERR_TIER_NAME_NOT_UPPERCASE) => { + return Err(S3Error::with_message( + S3ErrorCode::Custom("TierNameNotUppercase".into()), + "tier name not uppercase!", + )); + } + Err(ERR_TIER_BACKEND_IN_USE) => { + return Err(S3Error::with_message( + S3ErrorCode::Custom("TierNameBackendInUse!".into()), + "tier name backend in use!", + )); + } + Err(ERR_TIER_CONNECT_ERR) => { + return Err(S3Error::with_message( + S3ErrorCode::Custom("TierConnectError".into()), + "tier connect error!", + )); + } + Err(ERR_TIER_INVALID_CREDENTIALS) => { + return Err(S3Error::with_message( + S3ErrorCode::Custom(ERR_TIER_INVALID_CREDENTIALS.code.into()), + ERR_TIER_INVALID_CREDENTIALS.message, + )); + } + Err(e) => { + warn!("tier_config_mgr add failed, e: {:?}", e); + return Err(S3Error::with_message(S3ErrorCode::Custom("TierAddFailed".into()), "tier add failed")); + } + Ok(_) => (), + } + if let Err(e) = tier_config_mgr.save().await { + warn!("tier_config_mgr save failed, e: {:?}", e); + return Err(S3Error::with_message(S3ErrorCode::Custom("TierAddFailed".into()), "tier save failed")); + } + //globalNotificationSys.LoadTransitionTierConfig(ctx); + + let mut header = HeaderMap::new(); + header.insert(CONTENT_TYPE, "application/json".parse().unwrap()); + + Ok(S3Response::with_headers((StatusCode::OK, Body::empty()), header)) + } +} + +pub struct EditTier {} +#[async_trait::async_trait] +impl Operation for EditTier { + async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { + let query = { + if let Some(query) = req.uri.query() { + let input: AddTierQuery = + from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "get body failed1"))?; + input + } else { + AddTierQuery::default() + } + }; + + let Some(input_cred) = req.credentials else { + return Err(s3_error!(InvalidRequest, "get cred failed")); + }; + + let (cred, _owner) = + check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?; + + //{"accesskey":"gggggg","secretkey":"jjjjjjj"} + //{"detailedMessage":"failed to perform PUT: The Access Key Id you provided does not exist in our records.","message":"an error occurred, please try again"} + //200 OK + let mut input = req.input; + let body = match input.store_all_unlimited().await { + Ok(b) => b, + Err(e) => { + warn!("get body failed, e: {:?}", e); + return Err(s3_error!(InvalidRequest, "get body failed")); + } + }; + + let creds: TierCreds = serde_json::from_slice(&body) + .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("unmarshal body err {}", e)))?; + + debug!("edit tier args {:?}", creds); + + let tier_name = params.get("tiername").map(|s| s.to_string()).unwrap_or_default(); + + let mut tier_config_mgr = GLOBAL_TierConfigMgr.write().await; + //tier_config_mgr.reload(api); + match tier_config_mgr.edit(&tier_name, creds).await { + Err(ERR_TIER_NOT_FOUND) => { + return Err(S3Error::with_message(S3ErrorCode::Custom("TierNotFound".into()), "tier not found!")); + } + Err(ERR_TIER_MISSING_CREDENTIALS) => { + return Err(S3Error::with_message( + S3ErrorCode::Custom("TierMissingCredentials".into()), + "tier missing credentials!", + )); + } + Err(e) => { + warn!("tier_config_mgr edit failed, e: {:?}", e); + return Err(S3Error::with_message(S3ErrorCode::Custom("TierEditFailed".into()), "tier edit failed")); + } + Ok(_) => (), + } + if let Err(e) = tier_config_mgr.save().await { + warn!("tier_config_mgr save failed, e: {:?}", e); + return Err(S3Error::with_message(S3ErrorCode::Custom("TierEditFailed".into()), "tier save failed")); + } + //globalNotificationSys.LoadTransitionTierConfig(ctx); + + let mut header = HeaderMap::new(); + header.insert(CONTENT_TYPE, "application/json".parse().unwrap()); + + Ok(S3Response::with_headers((StatusCode::OK, Body::empty()), header)) + } +} + +#[derive(Debug, Deserialize, Default)] +pub struct BucketQuery { + #[serde(rename = "bucket")] + pub bucket: String, +} +pub struct ListTiers {} +#[async_trait::async_trait] +impl Operation for ListTiers { + async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { + let query = { + if let Some(query) = req.uri.query() { + let input: BucketQuery = + from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "get body failed"))?; + input + } else { + BucketQuery::default() + } + }; + + //{"items":[{"minio":{"accesskey":"minioadmin","bucket":"mblock2","endpoint":"http://192.168.1.11:9020","name":"COLDTIER","objects":"0","prefix":"mypre/","secretkey":"REDACTED","usage":"0 B","versions":"0"},"status":true,"type":"minio"}]} + let mut tier_config_mgr = GLOBAL_TierConfigMgr.read().await; + let tiers = tier_config_mgr.list_tiers(); + + let data = serde_json::to_vec(&tiers) + .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("marshal tiers err {}", e)))?; + + let mut header = HeaderMap::new(); + header.insert(CONTENT_TYPE, "application/json".parse().unwrap()); + + Ok(S3Response::with_headers((StatusCode::OK, Body::from(data)), header)) + } +} + +pub struct RemoveTier {} +#[async_trait::async_trait] +impl Operation for RemoveTier { + async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { + warn!("handle RemoveTier"); + + let query = { + if let Some(query) = req.uri.query() { + let input: AddTierQuery = + from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "get body failed"))?; + input + } else { + AddTierQuery::default() + } + }; + + let Some(input_cred) = req.credentials else { + return Err(s3_error!(InvalidRequest, "get cred failed")); + }; + + let (cred, _owner) = + check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?; + + let sys_cred = get_global_action_cred() + .ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "get_global_action_cred failed"))?; + + let mut force: bool = false; + let force_str = query.force; + if force_str != "" { + force = force_str.parse().unwrap(); + } + + let tier_name = params.get("tiername").map(|s| s.to_string()).unwrap_or_default(); + + let mut tier_config_mgr = GLOBAL_TierConfigMgr.write().await; + //tier_config_mgr.reload(api); + match tier_config_mgr.remove(&tier_name, force).await { + Err(ERR_TIER_NOT_FOUND) => { + return Err(S3Error::with_message(S3ErrorCode::Custom("TierNotFound".into()), "tier not found.")); + } + Err(ERR_TIER_BACKEND_NOT_EMPTY) => { + return Err(S3Error::with_message(S3ErrorCode::Custom("TierNameBackendInUse".into()), "tier is used.")); + } + Err(e) => { + warn!("tier_config_mgr remove failed, e: {:?}", e); + return Err(S3Error::with_message( + S3ErrorCode::Custom("TierRemoveFailed".into()), + "tier remove failed", + )); + } + Ok(_) => (), + } + if let Err(e) = tier_config_mgr.save().await { + warn!("tier_config_mgr save failed, e: {:?}", e); + return Err(S3Error::with_message(S3ErrorCode::Custom("TierRemoveFailed".into()), "tier save failed")); + } + //globalNotificationSys.LoadTransitionTierConfig(ctx); + + let mut header = HeaderMap::new(); + header.insert(CONTENT_TYPE, "application/json".parse().unwrap()); + + Ok(S3Response::with_headers((StatusCode::OK, Body::empty()), header)) + } +} + +pub struct VerifyTier {} +#[async_trait::async_trait] +impl Operation for VerifyTier { + async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { + warn!("handle RemoveTier"); + + let query = { + if let Some(query) = req.uri.query() { + let input: AddTierQuery = + from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "get body failed"))?; + input + } else { + AddTierQuery::default() + } + }; + + let Some(input_cred) = req.credentials else { + return Err(s3_error!(InvalidRequest, "get cred failed")); + }; + + let (cred, _owner) = + check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?; + + let sys_cred = get_global_action_cred() + .ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "get_global_action_cred failed"))?; + + let mut tier_config_mgr = GLOBAL_TierConfigMgr.write().await; + tier_config_mgr.verify(&query.tier.unwrap()); + + let mut header = HeaderMap::new(); + header.insert(CONTENT_TYPE, "application/json".parse().unwrap()); + + Ok(S3Response::with_headers((StatusCode::OK, Body::empty()), header)) + } +} + +pub struct GetTierInfo {} +#[async_trait::async_trait] +impl Operation for GetTierInfo { + async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { + warn!("handle GetTierInfo"); + + let query = { + if let Some(query) = req.uri.query() { + let input: AddTierQuery = + from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "get body failed"))?; + input + } else { + AddTierQuery::default() + } + }; + + let mut tier_config_mgr = GLOBAL_TierConfigMgr.read().await; + let info = tier_config_mgr.get(&query.tier.unwrap()); + + let data = serde_json::to_vec(&info) + .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("marshal tier err {}", e)))?; + + let mut header = HeaderMap::new(); + header.insert(CONTENT_TYPE, "application/json".parse().unwrap()); + + Ok(S3Response::with_headers((StatusCode::OK, Body::from(data)), header)) + } +} diff --git a/rustfs/src/admin/mod.rs b/rustfs/src/admin/mod.rs index 4a9089370..85df4f296 100644 --- a/rustfs/src/admin/mod.rs +++ b/rustfs/src/admin/mod.rs @@ -7,7 +7,7 @@ pub mod utils; use handlers::{ group, policys, pools, rebalance, service_account::{AddServiceAccount, DeleteServiceAccount, InfoServiceAccount, ListServiceAccount, UpdateServiceAccount}, - sts, user, + sts, tier, user, }; use handlers::{GetReplicationMetricsHandler, ListRemoteTargetHandler, RemoveRemoteTargetHandler, SetRemoteTargetHandler}; @@ -311,5 +311,38 @@ fn register_user_route(r: &mut S3Router) -> std::io::Result<()> AdminOperation(&policys::SetPolicyForUserOrGroup {}), )?; + // ? + r.insert( + Method::GET, + format!("{}{}", ADMIN_PREFIX, "/v3/tier").as_str(), + AdminOperation(&tier::ListTiers {}), + )?; + // ? + r.insert( + Method::GET, + format!("{}{}", ADMIN_PREFIX, "/v3/tier-stats").as_str(), + AdminOperation(&tier::GetTierInfo {}), + )?; + // ?force=xxx + r.insert( + Method::DELETE, + format!("{}{}", ADMIN_PREFIX, "/v3/tier/{tiername}").as_str(), + AdminOperation(&tier::RemoveTier {}), + )?; + // ?force=xxx + // body: AddOrUpdateTierReq + r.insert( + Method::PUT, + format!("{}{}", ADMIN_PREFIX, "/v3/tier").as_str(), + AdminOperation(&tier::AddTier {}), + )?; + // ? + // body: AddOrUpdateTierReq + r.insert( + Method::POST, + format!("{}{}", ADMIN_PREFIX, "/v3/tier/{tiername}").as_str(), + AdminOperation(&tier::EditTier {}), + )?; + Ok(()) } diff --git a/rustfs/src/storage/ecfs.rs b/rustfs/src/storage/ecfs.rs index 45e29e6ff..38241cfc9 100644 --- a/rustfs/src/storage/ecfs.rs +++ b/rustfs/src/storage/ecfs.rs @@ -18,6 +18,7 @@ use chrono::Utc; use datafusion::arrow::csv::WriterBuilder as CsvWriterBuilder; use datafusion::arrow::json::WriterBuilder as JsonWriterBuilder; use datafusion::arrow::json::writer::JsonArray; +use ecstore::bucket::error::BucketMetadataError; use ecstore::bucket::metadata::BUCKET_LIFECYCLE_CONFIG; use ecstore::bucket::metadata::BUCKET_NOTIFICATION_CONFIG; use ecstore::bucket::metadata::BUCKET_POLICY_CONFIG; @@ -54,7 +55,13 @@ use ecstore::store_api::ObjectToDelete; use ecstore::store_api::PutObjReader; use ecstore::store_api::StorageAPI; // use ecstore::store_api::RESERVED_METADATA_PREFIX; -use futures::StreamExt; +use ecstore::bucket::lifecycle::bucket_lifecycle_ops::validate_transition_tier; +use ecstore::bucket::utils::serialize; +use ecstore::cmd::bucket_replication::ReplicationStatusType; +use ecstore::cmd::bucket_replication::ReplicationType; +use ecstore::store_api::RESERVED_METADATA_PREFIX_LOWER; +use futures::pin_mut; +use futures::{Stream, StreamExt}; use http::HeaderMap; use lazy_static::lazy_static; use policy::auth; @@ -99,6 +106,11 @@ use tracing::info; use tracing::warn; use uuid::Uuid; +use ecstore::bucket::{ + lifecycle::{bucket_lifecycle_ops::ERR_INVALID_STORAGECLASS, lifecycle::Lifecycle}, + object_lock::objectlock_sys::BucketObjectLockSys, +}; + macro_rules! try_ { ($result:expr) => { match $result { @@ -1710,12 +1722,35 @@ impl S3 for FS { .. } = req.input; - // warn!("lifecycle_configuration {:?}", &lifecycle_configuration); + let rcfg = metadata_sys::get_object_lock_config(&bucket).await; + if rcfg.is_err() { + return Err(S3Error::with_message( + S3ErrorCode::Custom("BucketLockIsNotExist".into()), + "bucket lock is not exist.", + )); + } + let rcfg = rcfg.expect("get_lifecycle_config err!").0; - // TODO: objcetLock + //info!("lifecycle_configuration: {:?}", &lifecycle_configuration); let Some(input_cfg) = lifecycle_configuration else { return Err(s3_error!(InvalidArgument)) }; + if let Err(err) = input_cfg.validate(&rcfg).await { + //return Err(S3Error::with_message(S3ErrorCode::Custom("BucketLockValidateFailed".into()), "bucket lock validate failed.")); + return Err(S3Error::with_message( + S3ErrorCode::Custom("ValidateFailed".into()), + format!("{}", err.to_string()), + )); + } + + if let Err(err) = validate_transition_tier(&input_cfg).await { + //warn!("lifecycle_configuration add failed, err: {:?}", err); + return Err(S3Error::with_message( + S3ErrorCode::Custom("CustomError".into()), + format!("{}", err.to_string()), + )); + } + let data = try_!(serialize(&input_cfg)); metadata_sys::update(&bucket, BUCKET_LIFECYCLE_CONFIG, data) .await @@ -2156,6 +2191,30 @@ impl S3 for FS { })) } + async fn get_object_attributes( + &self, + req: S3Request, + ) -> S3Result> { + let GetObjectAttributesInput { bucket, key, .. } = req.input; + + let Some(store) = new_object_layer_fn() else { + return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); + }; + + if let Err(e) = store + .get_object_reader(&bucket, &key, None, HeaderMap::new(), &ObjectOptions::default()) + .await + { + return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("{}", e))); + } + + Ok(S3Response::new(GetObjectAttributesOutput { + delete_marker: None, + object_parts: None, + ..Default::default() + })) + } + async fn put_object_acl(&self, req: S3Request) -> S3Result> { let PutObjectAclInput { bucket, @@ -2399,7 +2458,7 @@ impl S3 for FS { let retain_until_date = object_info .user_defined .get("x-amz-object-lock-retain-until-date") - .and_then(|v| OffsetDateTime::parse(v.as_str(), &Rfc3339).ok()) + .and_then(|v| OffsetDateTime::parse(v.as_str(), &Rfc3339).ok()) .map(Timestamp::from); Ok(S3Response::new(GetObjectRetentionOutput {