From b9b7d86ae40a249299bfcf2d1e73bec6f5968d0e Mon Sep 17 00:00:00 2001 From: weisd Date: Wed, 18 Mar 2026 21:05:09 +0800 Subject: [PATCH] feat: improve legacy metadata and admin compatibility (#2202) --- .gitignore | 1 + Cargo.lock | 179 ++- Cargo.toml | 7 +- Dockerfile | 7 +- Dockerfile.decommission-local | 65 ++ Dockerfile.source | 12 +- crates/credentials/Cargo.toml | 2 +- crates/credentials/src/credentials.rs | 37 + crates/credentials/src/lib.rs | 1 + crates/credentials/src/serde_datetime.rs | 68 ++ crates/crypto/src/encdec.rs | 3 + crates/crypto/src/encdec/stream_io.rs | 219 ++++ crates/crypto/src/encdec/tests.rs | 66 +- crates/crypto/src/lib.rs | 3 + crates/ecstore/Cargo.toml | 2 + crates/ecstore/benches/erasure_benchmark.rs | 119 +- crates/ecstore/src/bitrot.rs | 8 +- .../ecstore/src/bucket/bucket_target_sys.rs | 53 +- .../bucket/lifecycle/bucket_lifecycle_ops.rs | 38 +- .../ecstore/src/bucket/lifecycle/lifecycle.rs | 140 ++- crates/ecstore/src/bucket/metadata.rs | 274 ++++- crates/ecstore/src/bucket/metadata_sys.rs | 14 +- crates/ecstore/src/bucket/metadata_test.rs | 261 +++++ crates/ecstore/src/bucket/migration.rs | 420 +++++++ crates/ecstore/src/bucket/mod.rs | 4 + crates/ecstore/src/bucket/msgp_decode.rs | 189 ++++ crates/ecstore/src/bucket/quota/mod.rs | 74 +- .../bucket/replication/replication_pool.rs | 43 +- .../replication/replication_resyncer.rs | 484 ++++++-- crates/ecstore/src/bucket/utils.rs | 4 +- crates/ecstore/src/config/com.rs | 339 +++++- crates/ecstore/src/config/mod.rs | 5 + crates/ecstore/src/disk/local.rs | 10 +- crates/ecstore/src/disk/mod.rs | 1 + crates/ecstore/src/erasure_coding/bitrot.rs | 2 +- crates/ecstore/src/erasure_coding/erasure.rs | 391 +++++-- crates/ecstore/src/erasure_coding/mod.rs | 2 +- crates/ecstore/src/pools.rs | 760 ++++++++++++- crates/ecstore/src/set_disk.rs | 172 ++- crates/ecstore/src/set_disk/heal.rs | 16 +- crates/ecstore/src/set_disk/read.rs | 17 +- crates/ecstore/src/store.rs | 2 +- crates/ecstore/src/store/init.rs | 3 +- crates/ecstore/src/store/object.rs | 111 +- crates/ecstore/src/store_api.rs | 6 +- crates/ecstore/src/store_api/types.rs | 39 +- crates/ecstore/src/store_init.rs | 64 +- crates/ecstore/src/store_list_objects.rs | 2 +- crates/ecstore/src/store_utils.rs | 6 +- crates/ecstore/src/tier/tier.rs | 876 +++++++++++++- crates/ecstore/src/tier/tier_config.rs | 16 +- .../ecstore/tests/legacy_bitrot_read_test.rs | 238 ++++ crates/filemeta/examples/dump_versions.rs | 24 + crates/filemeta/src/fileinfo.rs | 39 +- crates/filemeta/src/filemeta.rs | 280 +++-- crates/filemeta/src/filemeta/codec.rs | 24 +- crates/filemeta/src/filemeta/inline_data.rs | 12 +- crates/filemeta/src/filemeta/msgp_decode.rs | 212 ++++ crates/filemeta/src/filemeta/version.rs | 1008 +++++++++++++---- crates/filemeta/src/filemeta_inline.rs | 52 + crates/filemeta/src/replication.rs | 4 +- crates/filemeta/src/test_data.rs | 6 + crates/heal/tests/heal_integration_test.rs | 70 +- crates/iam/src/manager.rs | 71 +- crates/iam/src/store.rs | 54 + crates/iam/src/store/object.rs | 126 ++- crates/iam/src/sys.rs | 5 +- crates/keystone/Cargo.toml | 2 + crates/keystone/src/config.rs | 96 +- crates/kms/Cargo.toml | 2 + crates/kms/src/config.rs | 66 +- crates/madmin/src/group.rs | 53 +- crates/madmin/src/user.rs | 396 +++++-- crates/notify/src/stream.rs | 6 +- crates/obs/src/telemetry/local.rs | 40 +- crates/obs/src/telemetry/otel.rs | 91 +- crates/policy/Cargo.toml | 2 +- crates/policy/src/auth/mod.rs | 18 + crates/policy/src/lib.rs | 1 + crates/policy/src/policy/doc.rs | 58 + crates/policy/src/policy/policy.rs | 12 +- crates/policy/src/serde_datetime.rs | 121 ++ crates/rio/src/compress_index.rs | 4 + crates/rio/src/http_reader.rs | 8 +- crates/rio/src/lib.rs | 2 +- crates/scanner/src/data_usage_define.rs | 11 - crates/scanner/src/scanner_io.rs | 16 +- .../tests/lifecycle_integration_test.rs | 4 +- crates/trusted-proxies/src/cloud/detector.rs | 59 +- crates/utils/Cargo.toml | 4 +- crates/utils/src/envs.rs | 320 +++++- crates/utils/src/hash.rs | 112 +- crates/utils/src/http/header_compat.rs | 123 ++ crates/utils/src/http/headers.rs | 32 - crates/utils/src/http/metadata_compat.rs | 181 +++ crates/utils/src/http/mod.rs | 4 + crates/utils/src/obj/metadata.rs | 5 +- docker-compose.decommission.yml | 49 + entrypoint.sh | 3 +- rustfs/Cargo.toml | 2 + rustfs/src/admin/console.rs | 43 +- rustfs/src/admin/console_test.rs | 1 - rustfs/src/admin/handlers/policies.rs | 582 +++++++++- rustfs/src/admin/handlers/pools.rs | 7 +- rustfs/src/admin/handlers/quota.rs | 169 ++- rustfs/src/admin/handlers/service_account.rs | 886 +++++++++++++-- rustfs/src/admin/handlers/user.rs | 122 +- rustfs/src/admin/route_registration_test.rs | 54 +- rustfs/src/admin/router.rs | 45 +- rustfs/src/admin/utils.rs | 84 ++ rustfs/src/app/bucket_usecase.rs | 197 ++-- rustfs/src/app/multipart_usecase.rs | 270 +++-- rustfs/src/app/object_usecase.rs | 61 +- rustfs/src/config/config_test.rs | 74 +- rustfs/src/config/mod.rs | 155 ++- rustfs/src/main.rs | 74 +- rustfs/src/server/http.rs | 3 +- rustfs/src/server/layer.rs | 111 +- rustfs/src/server/prefix.rs | 7 + rustfs/src/server/readiness.rs | 2 + rustfs/src/storage/ecfs.rs | 21 +- rustfs/src/storage/ecfs_extend.rs | 12 +- rustfs/src/storage/ecfs_test.rs | 27 +- rustfs/src/storage/options.rs | 134 ++- rustfs/src/storage/s3_api/bucket.rs | 13 +- rustfs/src/storage/s3_api/multipart.rs | 5 +- rustfs/src/storage/sse.rs | 5 +- scripts/dev_rustfs.env | 4 +- scripts/s3-tests/run.sh | 6 +- scripts/test/decommission_docker.md | 49 + scripts/test/decommission_docker.sh | 173 +++ scripts/test/decommission_validation.md | 57 + scripts/test/decommission_validation.sh | 434 +++++++ 133 files changed, 11707 insertions(+), 1945 deletions(-) create mode 100644 Dockerfile.decommission-local create mode 100644 crates/credentials/src/serde_datetime.rs create mode 100644 crates/crypto/src/encdec/stream_io.rs create mode 100644 crates/ecstore/src/bucket/metadata_test.rs create mode 100644 crates/ecstore/src/bucket/migration.rs create mode 100644 crates/ecstore/src/bucket/msgp_decode.rs create mode 100644 crates/ecstore/tests/legacy_bitrot_read_test.rs create mode 100644 crates/filemeta/examples/dump_versions.rs create mode 100644 crates/filemeta/src/filemeta/msgp_decode.rs create mode 100644 crates/policy/src/serde_datetime.rs create mode 100644 crates/utils/src/http/header_compat.rs create mode 100644 crates/utils/src/http/metadata_compat.rs create mode 100644 docker-compose.decommission.yml create mode 100644 scripts/test/decommission_docker.md create mode 100755 scripts/test/decommission_docker.sh create mode 100644 scripts/test/decommission_validation.md create mode 100755 scripts/test/decommission_validation.sh diff --git a/.gitignore b/.gitignore index dd77b6ee4..162b0a581 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,7 @@ vendor cli/rustfs-gui/embedded-rustfs/rustfs *.log deploy/certs/* +deploy/data/* *jsonl .env .rustfs.sys diff --git a/Cargo.lock b/Cargo.lock index 045893824..f817ce108 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -69,6 +69,17 @@ dependencies = [ "subtle", ] +[[package]] +name = "ahash" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891477e0c6a8957309ee5c45a6368af3ae14bb510732d2684ffa19af310920f9" +dependencies = [ + "getrandom 0.2.17", + "once_cell", + "version_check", +] + [[package]] name = "ahash" version = "0.8.12" @@ -285,7 +296,7 @@ version = "57.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c8955af33b25f3b175ee10af580577280b4bd01f7e823d94c7cdef7cf8c9aef" dependencies = [ - "ahash", + "ahash 0.8.12", "arrow-buffer", "arrow-data", "arrow-schema", @@ -442,7 +453,7 @@ version = "57.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68bf3e3efbd1278f770d67e5dc410257300b161b93baedb3aae836144edcaf4b" dependencies = [ - "ahash", + "ahash 0.8.12", "arrow-array", "arrow-buffer", "arrow-data", @@ -731,7 +742,7 @@ dependencies = [ "http 0.2.12", "http 1.4.0", "http-body 1.0.1", - "lru", + "lru 0.16.3", "percent-encoding", "regex-lite", "sha2 0.10.9", @@ -2192,7 +2203,7 @@ dependencies = [ "hashbrown 0.14.5", "lock_api", "once_cell", - "parking_lot_core", + "parking_lot_core 0.9.12", ] [[package]] @@ -2244,7 +2255,7 @@ dependencies = [ "liblzma", "log", "object_store", - "parking_lot", + "parking_lot 0.12.5", "parquet", "rand 0.9.2", "regex", @@ -2277,7 +2288,7 @@ dependencies = [ "itertools 0.14.0", "log", "object_store", - "parking_lot", + "parking_lot 0.12.5", "tokio", ] @@ -2310,7 +2321,7 @@ version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ea2df29b9592a5d55b8238eaf67d2f21963d5a08cd1a8b7670134405206caabd" dependencies = [ - "ahash", + "ahash 0.8.12", "arrow", "arrow-ipc", "chrono", @@ -2468,7 +2479,7 @@ dependencies = [ "itertools 0.14.0", "log", "object_store", - "parking_lot", + "parking_lot 0.12.5", "parquet", "tokio", ] @@ -2494,7 +2505,7 @@ dependencies = [ "futures", "log", "object_store", - "parking_lot", + "parking_lot 0.12.5", "rand 0.9.2", "tempfile", "url", @@ -2573,7 +2584,7 @@ version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "325a00081898945d48d6194d9ca26120e523c993be3bb7c084061a5a2a72e787" dependencies = [ - "ahash", + "ahash 0.8.12", "arrow", "datafusion-common", "datafusion-doc", @@ -2594,7 +2605,7 @@ version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "809bbcb1e0dbec5d0ce30d493d135aea7564f1ba4550395f7f94321223df2dae" dependencies = [ - "ahash", + "ahash 0.8.12", "arrow", "datafusion-common", "datafusion-expr-common", @@ -2636,7 +2647,7 @@ dependencies = [ "datafusion-common", "datafusion-expr", "datafusion-physical-plan", - "parking_lot", + "parking_lot 0.12.5", "paste", ] @@ -2705,7 +2716,7 @@ version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d3a86264bb9163e7360b6622e789bc7fcbb43672e78a8493f0bc369a41a57c6" dependencies = [ - "ahash", + "ahash 0.8.12", "arrow", "datafusion-common", "datafusion-expr", @@ -2716,7 +2727,7 @@ dependencies = [ "hashbrown 0.16.1", "indexmap 2.13.0", "itertools 0.14.0", - "parking_lot", + "parking_lot 0.12.5", "paste", "petgraph", "recursive", @@ -2744,7 +2755,7 @@ version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2ae769ea5d688b4e74e9be5cad6f9d9f295b540825355868a3ab942380dd97ce" dependencies = [ - "ahash", + "ahash 0.8.12", "arrow", "chrono", "datafusion-common", @@ -2752,7 +2763,7 @@ dependencies = [ "hashbrown 0.16.1", "indexmap 2.13.0", "itertools 0.14.0", - "parking_lot", + "parking_lot 0.12.5", ] [[package]] @@ -2780,7 +2791,7 @@ version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "79949cbb109c2a45c527bfe0d956b9f2916807c05d4d2e66f3fd0af827ac2b61" dependencies = [ - "ahash", + "ahash 0.8.12", "arrow", "arrow-ord", "arrow-schema", @@ -2800,7 +2811,7 @@ dependencies = [ "indexmap 2.13.0", "itertools 0.14.0", "log", - "parking_lot", + "parking_lot 0.12.5", "pin-project-lite", "tokio", ] @@ -2833,7 +2844,7 @@ dependencies = [ "datafusion-execution", "datafusion-expr", "datafusion-physical-plan", - "parking_lot", + "parking_lot 0.12.5", ] [[package]] @@ -2873,9 +2884,9 @@ dependencies = [ "http-body-util", "libc", "log", - "lru", + "lru 0.16.3", "mime_guess", - "parking_lot", + "parking_lot 0.12.5", "percent-encoding", "pin-project-lite", "reflink-copy", @@ -4032,6 +4043,9 @@ name = "hashbrown" version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +dependencies = [ + "ahash 0.7.8", +] [[package]] name = "hashbrown" @@ -4501,7 +4515,7 @@ version = "0.11.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "232929e1d75fe899576a3d5c7416ad0d88dbfbb3c3d6aa00873a7408a50ddb88" dependencies = [ - "ahash", + "ahash 0.8.12", "indexmap 2.13.0", "is-terminal", "itoa", @@ -4519,7 +4533,7 @@ version = "0.12.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "90807d610575744524d9bdc69f3885d96f0e6c3354565b0828354a7ff2a262b8" dependencies = [ - "ahash", + "ahash 0.8.12", "clap", "crossbeam-channel", "crossbeam-utils", @@ -4553,6 +4567,15 @@ dependencies = [ "hybrid-array", ] +[[package]] +name = "instant" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" +dependencies = [ + "cfg-if", +] + [[package]] name = "integer-encoding" version = "3.0.4" @@ -5040,6 +5063,15 @@ version = "0.4.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +[[package]] +name = "lru" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999beba7b6e8345721bd280141ed958096a2e4abdf74f67ff4ce49b4b54e47a" +dependencies = [ + "hashbrown 0.12.3", +] + [[package]] name = "lru" version = "0.16.3" @@ -5202,7 +5234,7 @@ version = "0.24.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d5312e9ba3771cfa961b585728215e3d972c950a3eed9252aa093d6301277e8" dependencies = [ - "ahash", + "ahash 0.8.12", "portable-atomic", ] @@ -5271,7 +5303,7 @@ dependencies = [ "equivalent", "event-listener", "futures-util", - "parking_lot", + "parking_lot 0.12.5", "portable-atomic", "smallvec", "tagptr", @@ -5306,7 +5338,7 @@ dependencies = [ "libc", "log", "neli-proc-macros", - "parking_lot", + "parking_lot 0.12.5", ] [[package]] @@ -5629,7 +5661,7 @@ dependencies = [ "http 1.4.0", "humantime", "itertools 0.14.0", - "parking_lot", + "parking_lot 0.12.5", "percent-encoding", "thiserror 2.0.18", "tokio", @@ -5882,6 +5914,17 @@ version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" +[[package]] +name = "parking_lot" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" +dependencies = [ + "instant", + "lock_api", + "parking_lot_core 0.8.6", +] + [[package]] name = "parking_lot" version = "0.12.5" @@ -5889,7 +5932,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" dependencies = [ "lock_api", - "parking_lot_core", + "parking_lot_core 0.9.12", +] + +[[package]] +name = "parking_lot_core" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc" +dependencies = [ + "cfg-if", + "instant", + "libc", + "redox_syscall 0.2.16", + "smallvec", + "winapi", ] [[package]] @@ -5911,7 +5968,7 @@ version = "57.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ee96b29972a257b855ff2341b37e61af5f12d6af1158b6dcdb5b31ea07bb3cb" dependencies = [ - "ahash", + "ahash 0.8.12", "arrow-array", "arrow-buffer", "arrow-cast", @@ -6403,7 +6460,7 @@ dependencies = [ "fnv", "lazy_static", "memchr", - "parking_lot", + "parking_lot 0.12.5", "thiserror 2.0.18", ] @@ -6763,7 +6820,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "36ea961700fd7260e7fa3701c8287d901b2172c51f9c1421fa0f21d7f7e184b7" dependencies = [ "clocksource", - "parking_lot", + "parking_lot 0.12.5", "thiserror 1.0.69", ] @@ -6827,6 +6884,15 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "redox_syscall" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +dependencies = [ + "bitflags 1.3.2", +] + [[package]] name = "redox_syscall" version = "0.5.18" @@ -6856,6 +6922,21 @@ dependencies = [ "thiserror 2.0.18", ] +[[package]] +name = "reed-solomon-erasure" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7263373d500d4d4f505d43a2a662d475a894aa94503a1ee28e9188b5f3960d4f" +dependencies = [ + "cc", + "libc", + "libm", + "lru 0.7.8", + "parking_lot 0.11.2", + "smallvec", + "spin 0.9.8", +] + [[package]] name = "reed-solomon-simd" version = "3.1.0" @@ -7292,6 +7373,7 @@ dependencies = [ "rustfs-common", "rustfs-config", "rustfs-credentials", + "rustfs-crypto", "rustfs-ecstore", "rustfs-filemeta", "rustfs-heal", @@ -7326,6 +7408,7 @@ dependencies = [ "starshard", "subtle", "sysinfo", + "temp-env", "tempfile", "thiserror 2.0.18", "tikv-jemalloc-ctl", @@ -7488,12 +7571,13 @@ dependencies = [ "md-5 0.11.0-rc.5", "moka", "num_cpus", - "parking_lot", + "parking_lot 0.12.5", "path-absolutize", "pin-project-lite", "quick-xml 0.39.2", "rand 0.10.0", "ratelimit", + "reed-solomon-erasure", "reed-solomon-simd", "regex", "reqwest 0.13.2", @@ -7516,6 +7600,7 @@ dependencies = [ "rustls", "s3s", "serde", + "serde_bytes", "serde_json", "serde_urlencoded", "serial_test", @@ -7633,8 +7718,10 @@ dependencies = [ "reqwest 0.13.2", "rustfs-credentials", "rustfs-policy", + "rustfs-utils", "serde", "serde_json", + "temp-env", "thiserror 2.0.18", "time", "tokio", @@ -7656,9 +7743,11 @@ dependencies = [ "moka", "rand 0.10.0", "reqwest 0.13.2", + "rustfs-utils", "serde", "serde_json", "sha2 0.11.0-rc.5", + "temp-env", "tempfile", "thiserror 2.0.18", "tokio", @@ -7676,7 +7765,7 @@ dependencies = [ "async-trait", "crossbeam-queue", "futures", - "parking_lot", + "parking_lot 0.12.5", "rustfs-utils", "serde", "serde_json", @@ -7941,7 +8030,7 @@ dependencies = [ "futures-core", "http 1.4.0", "object_store", - "parking_lot", + "parking_lot 0.12.5", "pin-project-lite", "rustfs-common", "rustfs-ecstore", @@ -7964,7 +8053,7 @@ dependencies = [ "datafusion", "derive_builder 0.20.2", "futures", - "parking_lot", + "parking_lot 0.12.5", "rustfs-s3select-api", "s3s", "snafu 0.9.0", @@ -8066,6 +8155,7 @@ name = "rustfs-utils" version = "0.0.5" dependencies = [ "base64-simd", + "blake2 0.11.0-rc.5", "blake3", "brotli", "bytes", @@ -8097,6 +8187,7 @@ dependencies = [ "siphasher", "snap", "sysinfo", + "temp-env", "tempfile", "thiserror 2.0.18", "tokio", @@ -8306,7 +8397,7 @@ checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" [[package]] name = "s3s" version = "0.14.0-dev" -source = "git+https://github.com/s3s-project/s3s?rev=c2dc7b16535659904d4efff52c558fc039be1ef3#c2dc7b16535659904d4efff52c558fc039be1ef3" +source = "git+https://github.com/rustfs/s3s?rev=d9556e3c0036bd3f2b330966009cbaa5aebf19a3#d9556e3c0036bd3f2b330966009cbaa5aebf19a3" dependencies = [ "arc-swap", "arrayvec", @@ -8518,6 +8609,16 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_bytes" +version = "0.11.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8" +dependencies = [ + "serde", + "serde_core", +] + [[package]] name = "serde_core" version = "1.0.228" @@ -8645,7 +8746,7 @@ dependencies = [ "futures-util", "log", "once_cell", - "parking_lot", + "parking_lot 0.12.5", "scc", "serial_test_derive", ] @@ -9241,7 +9342,7 @@ version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96374855068f47402c3121c6eed88d29cb1de8f3ab27090e273e420bdabcf050" dependencies = [ - "parking_lot", + "parking_lot 0.12.5", ] [[package]] @@ -9468,7 +9569,7 @@ dependencies = [ "bytes", "libc", "mio", - "parking_lot", + "parking_lot 0.12.5", "pin-project-lite", "signal-hook-registry", "socket2", diff --git a/Cargo.toml b/Cargo.toml index df7126054..623d41505 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -148,6 +148,7 @@ rmcp = { version = "1.2.0" } rmp = { version = "0.8.15" } rmp-serde = { version = "1.3.1" } serde = { version = "1.0.228", features = ["derive"] } +serde_bytes = "0.11" serde_json = { version = "1.0.149", features = ["raw_value"] } serde_urlencoded = "0.7.1" schemars = "1.2.1" @@ -155,6 +156,7 @@ schemars = "1.2.1" # Cryptography and Security aes-gcm = { version = "0.11.0-rc.3", features = ["rand_core"] } argon2 = { version = "0.6.0-rc.7" } +blake2 = "0.11.0-rc.5" blake3 = { version = "1.8.3", features = ["rayon", "mmap"] } chacha20poly1305 = { version = "0.11.0-rc.3" } crc-fast = "1.9.0" @@ -236,13 +238,14 @@ pretty_assertions = "1.4.1" rand = { version = "0.10.0", features = ["serde"] } ratelimit = "0.10.0" rayon = "1.11.0" -reed-solomon-simd = { version = "3.1.0" } +reed-solomon-erasure = { version = "6.0", default-features = false, features = ["std", "simd-accel"] } +reed-solomon-simd = "3.1.0" regex = { version = "1.12.3" } rumqttc = { version = "0.25.1" } rustix = { version = "1.1.4", features = ["fs"] } rust-embed = { version = "8.11.0" } rustc-hash = { version = "2.1.1" } -s3s = { git = "https://github.com/s3s-project/s3s", rev = "c2dc7b16535659904d4efff52c558fc039be1ef3", features = ["minio"] } +s3s = { git = "https://github.com/rustfs/s3s", rev = "d9556e3c0036bd3f2b330966009cbaa5aebf19a3", features = ["minio"] } serial_test = "3.4.0" shadow-rs = { version = "1.7.1", default-features = false } siphasher = "1.0.2" diff --git a/Dockerfile b/Dockerfile index 6cb8eabb8..cecd2bc29 100644 --- a/Dockerfile +++ b/Dockerfile @@ -86,12 +86,7 @@ RUN addgroup -g 10001 -S rustfs && \ chown -R rustfs:rustfs /data /logs && \ chmod 0750 /data /logs -ENV RUSTFS_ADDRESS=":9000" \ - RUSTFS_CONSOLE_ADDRESS=":9001" \ - RUSTFS_ACCESS_KEY="rustfsadmin" \ - RUSTFS_SECRET_KEY="rustfsadmin" \ - RUSTFS_CONSOLE_ENABLE="true" \ - RUSTFS_CORS_ALLOWED_ORIGINS="*" \ +ENV RUSTFS_CORS_ALLOWED_ORIGINS="*" \ RUSTFS_CONSOLE_CORS_ALLOWED_ORIGINS="*" \ RUSTFS_VOLUMES="/data" \ RUSTFS_OBS_LOGGER_LEVEL=warn \ diff --git a/Dockerfile.decommission-local b/Dockerfile.decommission-local new file mode 100644 index 000000000..6a3bcbd96 --- /dev/null +++ b/Dockerfile.decommission-local @@ -0,0 +1,65 @@ +# Copyright 2024 RustFS Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM rust:1.91-trixie + +RUN set -eux; \ + export DEBIAN_FRONTEND=noninteractive; \ + apt-get update; \ + apt-get install -y --no-install-recommends \ + build-essential \ + ca-certificates \ + curl \ + git \ + pkg-config \ + libssl-dev \ + lld \ + protobuf-compiler \ + flatbuffers-compiler; \ + rm -rf /var/lib/apt/lists/* + +WORKDIR /usr/src/rustfs + +COPY . . + +RUN ./scripts/static.sh +RUN cargo run --bin gproto +RUN cargo build --release --locked --bin rustfs + +RUN set -eux; \ + groupadd -g 10001 rustfs; \ + useradd -u 10001 -g rustfs -M -s /usr/sbin/nologin rustfs + +WORKDIR /app + +RUN set -eux; \ + mkdir -p /data /logs; \ + chown -R rustfs:rustfs /data /logs /app; \ + chmod 0750 /data /logs + +COPY entrypoint.sh /entrypoint.sh +RUN install -m 0755 /usr/src/rustfs/target/release/rustfs /usr/bin/rustfs && chmod +x /entrypoint.sh + +ENV RUSTFS_VOLUMES="/data" \ + RUST_LOG="warn" \ + RUSTFS_OBS_LOG_DIRECTORY="/logs" \ + RUSTFS_USERNAME="rustfs" \ + RUSTFS_GROUPNAME="rustfs" \ + RUSTFS_UID="10001" \ + RUSTFS_GID="10001" + +EXPOSE 9000 9001 + +ENTRYPOINT ["/entrypoint.sh"] +CMD ["/usr/bin/rustfs"] diff --git a/Dockerfile.source b/Dockerfile.source index 764f85130..d2d3e6f37 100644 --- a/Dockerfile.source +++ b/Dockerfile.source @@ -157,11 +157,7 @@ WORKDIR /app ENV CARGO_INCREMENTAL=1 # Ensure we have the same default env vars available -ENV RUSTFS_ADDRESS=":9000" \ - RUSTFS_ACCESS_KEY="rustfsadmin" \ - RUSTFS_SECRET_KEY="rustfsadmin" \ - RUSTFS_CONSOLE_ENABLE="true" \ - RUSTFS_VOLUMES="/data" \ +ENV RUSTFS_VOLUMES="/data" \ RUST_LOG="warn" \ RUSTFS_OBS_LOG_DIRECTORY="/logs" \ RUSTFS_USERNAME="rustfs" \ @@ -222,11 +218,7 @@ COPY entrypoint.sh /entrypoint.sh RUN chmod +x /usr/bin/rustfs /entrypoint.sh # Default environment (override in docker run/compose as needed) -ENV RUSTFS_ADDRESS=":9000" \ - RUSTFS_ACCESS_KEY="rustfsadmin" \ - RUSTFS_SECRET_KEY="rustfsadmin" \ - RUSTFS_CONSOLE_ENABLE="true" \ - RUSTFS_VOLUMES="/data" \ +ENV RUSTFS_VOLUMES="/data" \ RUST_LOG="warn" \ RUSTFS_USERNAME="rustfs" \ RUSTFS_GROUPNAME="rustfs" \ diff --git a/crates/credentials/Cargo.toml b/crates/credentials/Cargo.toml index 051644e7f..25e38048b 100644 --- a/crates/credentials/Cargo.toml +++ b/crates/credentials/Cargo.toml @@ -15,7 +15,7 @@ base64-simd = { workspace = true } rand = { workspace = true } serde = { workspace = true } serde_json.workspace = true -time = { workspace = true, features = ["serde-human-readable"] } +time = { workspace = true, features = ["serde", "parsing", "formatting", "macros"] } [lints] workspace = true diff --git a/crates/credentials/src/credentials.rs b/crates/credentials/src/credentials.rs index 09c5919fe..0b97ff326 100644 --- a/crates/credentials/src/credentials.rs +++ b/crates/credentials/src/credentials.rs @@ -273,15 +273,25 @@ impl<'a> fmt::Display for Masked<'a> { /// #[derive(Serialize, Deserialize, Clone, Default)] pub struct Credentials { + #[serde(rename = "accessKey", alias = "access_key", default)] pub access_key: String, + #[serde(rename = "secretKey", alias = "secret_key", default)] pub secret_key: String, + #[serde(rename = "sessionToken", alias = "session_token", default)] pub session_token: String, + #[serde(default, with = "crate::serde_datetime::option")] pub expiration: Option, + #[serde(default)] pub status: String, + #[serde(rename = "parentUser", alias = "parent_user", default)] pub parent_user: String, + #[serde(default)] pub groups: Option>, + #[serde(default)] pub claims: Option>, + #[serde(default)] pub name: Option, + #[serde(default)] pub description: Option, } @@ -491,4 +501,31 @@ mod tests { assert_eq!(format!("{:?}", Masked(Some("中文"))), "中***|2"); assert_eq!(format!("{:?}", Masked(Some("中文测试"))), "中***试|4"); } + + #[test] + fn test_credentials_expiration_serialize_as_rfc3339() { + use time::OffsetDateTime; + + let c = Credentials { + access_key: "ak".to_string(), + secret_key: "sk12345678".to_string(), + expiration: Some(OffsetDateTime::now_utc()), + ..Default::default() + }; + + let json = serde_json::to_string(&c).expect("serialize"); + assert!( + json.contains('T') && (json.contains('Z') || json.contains("+00:00")), + "Credentials expiration should be RFC3339; got: {}", + json + ); + } + + #[test] + fn test_credentials_deserialize_minio_style_rfc3339_expiration() { + let minio_style = r#"{"accessKey":"ak","secretKey":"sk12345678","expiration":"2025-03-07T12:00:00Z"}"#; + let c: Credentials = serde_json::from_str(minio_style).expect("deserialize"); + assert_eq!(c.access_key, "ak"); + assert!(c.expiration.is_some()); + } } diff --git a/crates/credentials/src/lib.rs b/crates/credentials/src/lib.rs index 876d61b86..24dd6ba27 100644 --- a/crates/credentials/src/lib.rs +++ b/crates/credentials/src/lib.rs @@ -14,6 +14,7 @@ mod constants; mod credentials; +mod serde_datetime; pub use constants::*; pub use credentials::*; diff --git a/crates/credentials/src/serde_datetime.rs b/crates/credentials/src/serde_datetime.rs new file mode 100644 index 000000000..8dae6e471 --- /dev/null +++ b/crates/credentials/src/serde_datetime.rs @@ -0,0 +1,68 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Serde helpers for expiration timestamp: serialize as RFC3339 (MinIO-compatible), +//! deserialize from RFC3339 or legacy RustFS human-readable format. + +use time::OffsetDateTime; +use time::format_description; +use time::format_description::well_known::Rfc3339; + +static LEGACY_FORMAT: std::sync::OnceLock = std::sync::OnceLock::new(); + +fn legacy_format() -> &'static time::format_description::OwnedFormatItem { + LEGACY_FORMAT.get_or_init(|| { + format_description::parse_owned::<2>( + "[year]-[month]-[day] [hour]:[minute]:[second].[subsecond] [offset_hour sign:mandatory]:[offset_minute]:[offset_second]", + ) + .expect("legacy format description is valid") + }); + LEGACY_FORMAT.get().expect("initialized above") +} + +fn parse_rfc3339_or_legacy(s: &str) -> Result { + OffsetDateTime::parse(s, &Rfc3339).or_else(|_| OffsetDateTime::parse(s, legacy_format()).map_err(Into::into)) +} + +/// Option: serialize as RFC3339; deserialize from RFC3339 or legacy. +pub mod option { + use serde::{Deserialize, Deserializer, Serializer}; + use time::OffsetDateTime; + + use super::{Rfc3339, parse_rfc3339_or_legacy}; + + pub fn serialize(opt: &Option, serializer: S) -> Result + where + S: Serializer, + { + match opt { + Some(dt) => { + let s = dt.format(&Rfc3339).map_err(serde::ser::Error::custom)?; + serializer.serialize_some(&s) + } + None => serializer.serialize_none(), + } + } + + pub fn deserialize<'de, D>(deserializer: D) -> Result, D::Error> + where + D: Deserializer<'de>, + { + let opt: Option<&str> = Option::deserialize(deserializer)?; + match opt { + None => Ok(None), + Some(s) => parse_rfc3339_or_legacy(s).map(Some).map_err(serde::de::Error::custom), + } + } +} diff --git a/crates/crypto/src/encdec.rs b/crates/crypto/src/encdec.rs index 134eb3079..86bb5a476 100644 --- a/crates/crypto/src/encdec.rs +++ b/crates/crypto/src/encdec.rs @@ -21,5 +21,8 @@ pub(crate) mod id; pub(crate) mod decrypt; pub(crate) mod encrypt; +#[cfg(any(test, feature = "crypto"))] +pub(crate) mod stream_io; + #[cfg(test)] mod tests; diff --git a/crates/crypto/src/encdec/stream_io.rs b/crates/crypto/src/encdec/stream_io.rs new file mode 100644 index 000000000..06d435b1a --- /dev/null +++ b/crates/crypto/src/encdec/stream_io.rs @@ -0,0 +1,219 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! sio-go compatible stream encryption for IAM config. +//! Header: salt(32) + alg_id(1) + nonce_prefix(8) = 41 bytes. +//! Body: DARE-style fragmented AEAD (bufSize=16384, per-fragment nonce). + +#![allow(deprecated)] // AeadInPlace deprecated in favor of AeadInOut; keep for aead 0.6 compatibility + +use crate::encdec::id::ID; +use crate::error::Error; +use aes_gcm::{ + Aes256Gcm, + aead::{AeadCore, AeadInPlace, KeyInit as _, array::Array}, +}; +use chacha20poly1305::ChaCha20Poly1305; + +const STREAM_IO_HEADER_LEN: usize = 41; +const SIO_BUF_SIZE: usize = 16384; +const SIO_NONCE_PREFIX_LEN: usize = 8; +const AES_GCM_OVERHEAD: usize = 16; +const CHACHA_OVERHEAD: usize = 16; + +/// Decrypt data in stream_io (sio-go) format. +pub fn decrypt_stream_io(password: &[u8], data: &[u8]) -> Result, Error> { + if data.len() < STREAM_IO_HEADER_LEN { + return Err(Error::ErrUnexpectedHeader); + } + let salt = &data[0..32]; + let id = ID::try_from(data[32])?; + let nonce_prefix = &data[33..41]; + let body = &data[STREAM_IO_HEADER_LEN..]; + + let key = id.get_key(password, salt)?; + + match id { + ID::Argon2idChaCHa20Poly1305 => decrypt_stream( + ChaCha20Poly1305::new_from_slice(&key).map_err(|e| Error::ErrInvalidInput(e.to_string()))?, + nonce_prefix, + body, + CHACHA_OVERHEAD, + ), + _ => decrypt_stream( + Aes256Gcm::new_from_slice(&key).map_err(|e| Error::ErrInvalidInput(e.to_string()))?, + nonce_prefix, + body, + AES_GCM_OVERHEAD, + ), + } +} + +fn decrypt_stream(aead: A, nonce_prefix: &[u8], body: &[u8], overhead: usize) -> Result, Error> +where + A: AeadInPlace, +{ + let ciphertext_len = SIO_BUF_SIZE + overhead; + let ad = build_associated_data(&aead, nonce_prefix)?; + let mut plain = Vec::with_capacity(body.len()); + + let mut seq_num: u32 = 1; + let mut pos = 0; + + while pos < body.len() { + let remaining = body.len() - pos; + let frag_len = remaining.min(ciphertext_len); + let is_last = (pos + frag_len) == body.len(); + + if frag_len < overhead { + return Err(Error::ErrDecryptFailed(aes_gcm::aead::Error)); + } + + let mut nonce = [0u8; 12]; + nonce[0..SIO_NONCE_PREFIX_LEN].copy_from_slice(nonce_prefix); + nonce[8..12].copy_from_slice(&seq_num.to_le_bytes()); + + let mut ad_mut = ad.clone(); + ad_mut[0] = if is_last { 0x80 } else { 0x00 }; + + let fragment = &body[pos..pos + frag_len]; + let tag_len = overhead; + let (ct, tag) = fragment.split_at(frag_len - tag_len); + + let mut buffer = ct.to_vec(); + let nonce_arr = Array::::NonceSize>::try_from(&nonce[..]) + .map_err(|_| Error::ErrDecryptFailed(aes_gcm::aead::Error))?; + let tag_arr = + Array::::TagSize>::try_from(tag).map_err(|_| Error::ErrDecryptFailed(aes_gcm::aead::Error))?; + aead.decrypt_in_place_detached(&nonce_arr, &ad_mut, &mut buffer, &tag_arr) + .map_err(|_| Error::ErrDecryptFailed(aes_gcm::aead::Error))?; + plain.extend_from_slice(&buffer); + + pos += frag_len; + seq_num += 1; + + if is_last { + break; + } + } + + Ok(plain) +} + +fn build_associated_data(aead: &A, nonce_prefix: &[u8]) -> Result, Error> +where + A: AeadInPlace, +{ + let mut nonce = [0u8; 12]; + nonce[0..SIO_NONCE_PREFIX_LEN].copy_from_slice(nonce_prefix); + nonce[8..12].copy_from_slice(&0u32.to_le_bytes()); + + let nonce_arr = Array::::NonceSize>::try_from(&nonce[..]) + .map_err(|_| Error::ErrEncryptFailed(aes_gcm::aead::Error))?; + let mut empty: [u8; 0] = []; + let tag = aead + .encrypt_in_place_detached(&nonce_arr, &[] as &[u8], &mut empty) + .map_err(Error::ErrEncryptFailed)?; + + let mut ad = vec![0u8; 1 + tag.len()]; + ad[0] = 0x00; + ad[1..].copy_from_slice(tag.as_slice()); + Ok(ad) +} + +/// Encrypt data in stream_io (sio-go) format. +pub fn encrypt_stream_io(password: &[u8], data: &[u8]) -> Result, Error> { + let salt: [u8; 32] = rand::random(); + + #[cfg(feature = "fips")] + let id = ID::Pbkdf2AESGCM; + + #[cfg(not(feature = "fips"))] + let id = if crate::encdec::encrypt::native_aes() { + ID::Argon2idAESGCM + } else { + ID::Argon2idChaCHa20Poly1305 + }; + + let key = id.get_key(password, &salt)?; + let nonce_prefix: [u8; SIO_NONCE_PREFIX_LEN] = rand::random(); + + let mut out = Vec::with_capacity(STREAM_IO_HEADER_LEN + data.len() + 32); + out.extend_from_slice(&salt); + out.push(id as u8); + out.extend_from_slice(&nonce_prefix); + + match id { + ID::Argon2idChaCHa20Poly1305 => encrypt_stream( + ChaCha20Poly1305::new_from_slice(&key).map_err(|e| Error::ErrInvalidInput(e.to_string()))?, + &nonce_prefix, + data, + &mut out, + CHACHA_OVERHEAD, + )?, + _ => encrypt_stream( + Aes256Gcm::new_from_slice(&key).map_err(|e| Error::ErrInvalidInput(e.to_string()))?, + &nonce_prefix, + data, + &mut out, + AES_GCM_OVERHEAD, + )?, + } + + Ok(out) +} + +fn encrypt_stream( + aead: A, + nonce_prefix: &[u8; SIO_NONCE_PREFIX_LEN], + data: &[u8], + out: &mut Vec, + _overhead: usize, +) -> Result<(), Error> +where + A: AeadInPlace, +{ + let ad = build_associated_data(&aead, nonce_prefix)?; + let mut seq_num: u32 = 1; + let mut pos = 0; + + while pos < data.len() { + let remaining = data.len() - pos; + let is_last = remaining <= SIO_BUF_SIZE; + + let chunk_len = if is_last { remaining } else { SIO_BUF_SIZE }; + let chunk = &data[pos..pos + chunk_len]; + + let mut nonce = [0u8; 12]; + nonce[0..SIO_NONCE_PREFIX_LEN].copy_from_slice(nonce_prefix); + nonce[8..12].copy_from_slice(&seq_num.to_le_bytes()); + + let mut ad_mut = ad.clone(); + ad_mut[0] = if is_last { 0x80 } else { 0x00 }; + + let mut buffer = chunk.to_vec(); + let nonce_arr = Array::::NonceSize>::try_from(&nonce[..]) + .map_err(|_| Error::ErrEncryptFailed(aes_gcm::aead::Error))?; + let tag = aead + .encrypt_in_place_detached(&nonce_arr, &ad_mut, &mut buffer) + .map_err(Error::ErrEncryptFailed)?; + out.extend_from_slice(&buffer); + out.extend_from_slice(tag.as_slice()); + + pos += chunk_len; + seq_num += 1; + } + + Ok(()) +} diff --git a/crates/crypto/src/encdec/tests.rs b/crates/crypto/src/encdec/tests.rs index 675c1e4fa..5eb850044 100644 --- a/crates/crypto/src/encdec/tests.rs +++ b/crates/crypto/src/encdec/tests.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::{decrypt_data, encrypt_data}; +use crate::{decrypt_data, decrypt_stream_io, encrypt_data, encrypt_stream_io}; const PASSWORD: &[u8] = "test_password".as_bytes(); const LONG_PASSWORD: &[u8] = "very_long_password_with_many_characters_for_testing_purposes_123456789".as_bytes(); @@ -317,3 +317,67 @@ fn test_concurrent_encryption_safety() -> Result<(), crate::Error> { Ok(()) } + +#[test] +fn test_stream_io_roundtrip() -> Result<(), crate::Error> { + let password = b"access:secret"; + let data = br#"{"Version":1,"policy":"readonly"}"#; + let encrypted = encrypt_stream_io(password, data)?; + let decrypted = decrypt_stream_io(password, &encrypted)?; + assert_eq!(data, decrypted.as_slice()); + Ok(()) +} + +#[test] +fn test_stream_io_large_data_roundtrip() -> Result<(), crate::Error> { + let password = b"access:secret"; + let data = vec![0xAB; 32 * 1024]; // > SIO_BUF_SIZE to test fragmentation + let encrypted = encrypt_stream_io(password, &data)?; + let decrypted = decrypt_stream_io(password, &encrypted)?; + assert_eq!(data, decrypted); + Ok(()) +} + +#[test] +fn test_stream_io_wrong_password_fails() { + let password = b"access:secret"; + let data = br#"{"Version":1}"#; + let encrypted = encrypt_stream_io(password, data).expect("encrypt should succeed"); + let result = decrypt_stream_io(b"wrong:password", &encrypted); + assert!(result.is_err(), "decrypt with wrong password should fail"); +} + +#[test] +fn test_stream_io_empty_data() -> Result<(), crate::Error> { + let password = b"access:secret"; + let data: &[u8] = &[]; + let encrypted = encrypt_stream_io(password, data)?; + let decrypted = decrypt_stream_io(password, &encrypted)?; + assert!(decrypted.is_empty()); + Ok(()) +} + +#[test] +fn test_stream_io_header_format() -> Result<(), crate::Error> { + let password = b"access:secret"; + let data = b"test"; + let encrypted = encrypt_stream_io(password, data)?; + // stream_io header: salt(32) + alg_id(1) + nonce_prefix(8) = 41 bytes + const STREAM_IO_HEADER_LEN: usize = 41; + assert!(encrypted.len() >= STREAM_IO_HEADER_LEN, "encrypted should have at least 41-byte header"); + assert!( + encrypted[32] == 0x00 || encrypted[32] == 0x01 || encrypted[32] == 0x02, + "alg_id should be 0x00, 0x01, or 0x02" + ); + Ok(()) +} + +#[test] +fn test_stream_io_truncated_data_fails() { + let password = b"access:secret"; + let data = b"test"; + let encrypted = encrypt_stream_io(password, data).expect("encrypt should succeed"); + let truncated = &encrypted[..40]; // less than 41-byte header + let result = decrypt_stream_io(password, truncated); + assert!(result.is_err(), "truncated data should fail decrypt"); +} diff --git a/crates/crypto/src/lib.rs b/crates/crypto/src/lib.rs index 8969f9cae..8bed5bfd8 100644 --- a/crates/crypto/src/lib.rs +++ b/crates/crypto/src/lib.rs @@ -19,6 +19,9 @@ mod jwt; pub use encdec::decrypt::decrypt_data; pub use encdec::encrypt::encrypt_data; + +#[cfg(feature = "crypto")] +pub use encdec::stream_io::{decrypt_stream_io, encrypt_stream_io}; pub use error::Error; pub use jwt::decode::decode as jwt_decode; pub use jwt::encode::encode as jwt_encode; diff --git a/crates/ecstore/Cargo.toml b/crates/ecstore/Cargo.toml index 4bfe3c130..71b862780 100644 --- a/crates/ecstore/Cargo.toml +++ b/crates/ecstore/Cargo.toml @@ -66,6 +66,7 @@ http-body = { workspace = true } http-body-util.workspace = true url.workspace = true uuid = { workspace = true, features = ["v4", "fast-rng", "serde"] } +reed-solomon-erasure = { workspace = true } reed-solomon-simd = { workspace = true } lazy_static.workspace = true rustfs-lock.workspace = true @@ -73,6 +74,7 @@ regex = { workspace = true } path-absolutize = { workspace = true } rmp.workspace = true rmp-serde.workspace = true +serde_bytes.workspace = true tokio-util = { workspace = true, features = ["io", "compat"] } base64 = { workspace = true } hmac = { workspace = true } diff --git a/crates/ecstore/benches/erasure_benchmark.rs b/crates/ecstore/benches/erasure_benchmark.rs index bdbe622b0..b57099848 100644 --- a/crates/ecstore/benches/erasure_benchmark.rs +++ b/crates/ecstore/benches/erasure_benchmark.rs @@ -118,41 +118,38 @@ fn bench_encode_performance(c: &mut Criterion) { }); group.finish(); - // Test direct SIMD implementation for large shards (>= 512 bytes) + // Test direct reed-solomon-erasure implementation for large shards (>= 512 bytes) let shard_size = calc_shard_size(config.data_size, config.data_shards); - if shard_size >= 512 { - let mut simd_group = c.benchmark_group("encode_simd_direct"); - simd_group.throughput(Throughput::Bytes(config.data_size as u64)); - simd_group.sample_size(10); - simd_group.measurement_time(Duration::from_secs(5)); + if shard_size >= 512 && config.parity_shards > 0 { + use reed_solomon_erasure::galois_8::ReedSolomon; - simd_group.bench_with_input(BenchmarkId::new("simd_direct", &config.name), &(&data, &config), |b, (data, config)| { - b.iter(|| { - // Direct SIMD implementation - let per_shard_size = calc_shard_size(data.len(), config.data_shards); - match reed_solomon_simd::ReedSolomonEncoder::new(config.data_shards, config.parity_shards, per_shard_size) { - Ok(mut encoder) => { - // Create properly sized buffer and fill with data - let mut buffer = vec![0u8; per_shard_size * config.data_shards]; + let mut rse_group = c.benchmark_group("encode_rse_direct"); + rse_group.throughput(Throughput::Bytes(config.data_size as u64)); + rse_group.sample_size(10); + rse_group.measurement_time(Duration::from_secs(5)); + + if let Ok(rs) = ReedSolomon::new(config.data_shards, config.parity_shards) { + let total_shards = config.data_shards + config.parity_shards; + let per_shard_size = calc_shard_size(config.data_size, config.data_shards); + let need_total = per_shard_size * total_shards; + + rse_group.bench_with_input( + BenchmarkId::new("rse_direct", &config.name), + &(&data, need_total, per_shard_size), + |b, (data, need_total, per_shard_size)| { + b.iter(|| { + let mut buffer = vec![0u8; *need_total]; let copy_len = data.len().min(buffer.len()); buffer[..copy_len].copy_from_slice(&data[..copy_len]); - // Add data shards with correct shard size - for chunk in buffer.chunks_exact(per_shard_size) { - encoder.add_original_shard(black_box(chunk)).unwrap(); - } - - let result = encoder.encode().unwrap(); - black_box(result); - } - Err(_) => { - // SIMD doesn't support this configuration, skip - black_box(()); - } - } - }); - }); - simd_group.finish(); + let mut slices: Vec<&mut [u8]> = buffer.chunks_exact_mut(*per_shard_size).collect(); + rs.encode(&mut slices).unwrap(); + black_box(buffer); + }); + }, + ); + } + rse_group.finish(); } } } @@ -203,47 +200,33 @@ fn bench_decode_performance(c: &mut Criterion) { ); group.finish(); - // Test direct SIMD decoding for large shards + // Test direct reed-solomon-erasure decoding for large shards let shard_size = calc_shard_size(config.data_size, config.data_shards); - if shard_size >= 512 { - let mut simd_group = c.benchmark_group("decode_simd_direct"); - simd_group.throughput(Throughput::Bytes(config.data_size as u64)); - simd_group.sample_size(10); - simd_group.measurement_time(Duration::from_secs(5)); + if shard_size >= 512 && config.parity_shards > 0 { + use reed_solomon_erasure::galois_8::ReedSolomon; - simd_group.bench_with_input( - BenchmarkId::new("simd_direct", &config.name), - &(&encoded_shards, &config), - |b, (shards, config)| { - b.iter(|| { - let per_shard_size = calc_shard_size(config.data_size, config.data_shards); - match reed_solomon_simd::ReedSolomonDecoder::new(config.data_shards, config.parity_shards, per_shard_size) - { - Ok(mut decoder) => { - // Add available shards (except lost ones) - for (i, shard) in shards.iter().enumerate() { - if i != config.data_shards - 1 && i != config.data_shards { - if i < config.data_shards { - decoder.add_original_shard(i, black_box(shard)).unwrap(); - } else { - let recovery_idx = i - config.data_shards; - decoder.add_recovery_shard(recovery_idx, black_box(shard)).unwrap(); - } - } - } + if let Ok(rs) = ReedSolomon::new(config.data_shards, config.parity_shards) { + let mut rse_group = c.benchmark_group("decode_rse_direct"); + rse_group.throughput(Throughput::Bytes(config.data_size as u64)); + rse_group.sample_size(10); + rse_group.measurement_time(Duration::from_secs(5)); - let result = decoder.decode().unwrap(); - black_box(result); - } - Err(_) => { - // SIMD doesn't support this configuration, skip - black_box(()); - } - } - }); - }, - ); - simd_group.finish(); + rse_group.bench_with_input( + BenchmarkId::new("rse_direct", &config.name), + &(&encoded_shards, &config), + |b, (shards, config)| { + b.iter(|| { + let mut shards_opt: Vec>> = shards.iter().map(|s| Some(s.to_vec())).collect(); + shards_opt[config.data_shards - 1] = None; + shards_opt[config.data_shards] = None; + + rs.reconstruct_data(&mut shards_opt).unwrap(); + black_box(shards_opt); + }); + }, + ); + rse_group.finish(); + } } } } diff --git a/crates/ecstore/src/bitrot.rs b/crates/ecstore/src/bitrot.rs index d575c82d8..fd91316c3 100644 --- a/crates/ecstore/src/bitrot.rs +++ b/crates/ecstore/src/bitrot.rs @@ -119,7 +119,7 @@ mod tests { async fn test_create_bitrot_reader_with_inline_data() { let test_data = b"hello world test data"; let shard_size = 16; - let checksum_algo = HashAlgorithm::HighwayHash256; + let checksum_algo = HashAlgorithm::HighwayHash256S; let result = create_bitrot_reader(Some(test_data), None, "test-bucket", "test-path", 0, 0, shard_size, checksum_algo, false).await; @@ -131,7 +131,7 @@ mod tests { #[tokio::test] async fn test_create_bitrot_reader_without_data_or_disk() { let shard_size = 16; - let checksum_algo = HashAlgorithm::HighwayHash256; + let checksum_algo = HashAlgorithm::HighwayHash256S; let result = create_bitrot_reader(None, None, "test-bucket", "test-path", 0, 1024, shard_size, checksum_algo, false).await; @@ -151,7 +151,7 @@ mod tests { "test-path", 1024, // length 1024, // shard_size - HashAlgorithm::HighwayHash256, + HashAlgorithm::HighwayHash256S, ) .await; @@ -183,7 +183,7 @@ mod tests { "test-path", 1024, // length 1024, // shard_size - HashAlgorithm::HighwayHash256, + HashAlgorithm::HighwayHash256S, ) .await; diff --git a/crates/ecstore/src/bucket/bucket_target_sys.rs b/crates/ecstore/src/bucket/bucket_target_sys.rs index e840b1f1a..a29208556 100644 --- a/crates/ecstore/src/bucket/bucket_target_sys.rs +++ b/crates/ecstore/src/bucket/bucket_target_sys.rs @@ -43,11 +43,13 @@ use rustfs_config::{DEFAULT_TRUST_LEAF_CERT_AS_CA, ENV_TRUST_LEAF_CERT_AS_CA, RU use rustfs_filemeta::{ReplicationStatusType, ReplicationType}; use rustfs_utils::http::{ AMZ_BUCKET_REPLICATION_STATUS, AMZ_OBJECT_LOCK_BYPASS_GOVERNANCE, AMZ_OBJECT_LOCK_LEGAL_HOLD, AMZ_OBJECT_LOCK_MODE, - AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, AMZ_STORAGE_CLASS, AMZ_WEBSITE_REDIRECT_LOCATION, RUSTFS_BUCKET_REPLICATION_CHECK, - RUSTFS_BUCKET_REPLICATION_DELETE_MARKER, RUSTFS_BUCKET_REPLICATION_REQUEST, RUSTFS_BUCKET_SOURCE_ETAG, - RUSTFS_BUCKET_SOURCE_MTIME, RUSTFS_BUCKET_SOURCE_VERSION_ID, RUSTFS_FORCE_DELETE, is_amz_header, is_minio_header, + AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, AMZ_STORAGE_CLASS, AMZ_WEBSITE_REDIRECT_LOCATION, is_amz_header, is_minio_header, is_rustfs_header, is_standard_header, is_storageclass_header, }; +use rustfs_utils::http::{ + SUFFIX_FORCE_DELETE, SUFFIX_SOURCE_DELETEMARKER, SUFFIX_SOURCE_ETAG, SUFFIX_SOURCE_MTIME, SUFFIX_SOURCE_REPLICATION_CHECK, + SUFFIX_SOURCE_REPLICATION_REQUEST, SUFFIX_SOURCE_VERSION_ID, insert_header, +}; use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::error::Error; @@ -68,8 +70,6 @@ use uuid::Uuid; const DEFAULT_HEALTH_CHECK_DURATION: Duration = Duration::from_secs(5); const DEFAULT_HEALTH_CHECK_RELOAD_DURATION: Duration = Duration::from_secs(30 * 60); -const REPLICATION_REQUEST_TRUE: HeaderValue = HeaderValue::from_static("true"); - pub static GLOBAL_BUCKET_TARGET_SYS: OnceLock = OnceLock::new(); #[derive(Debug, Clone)] @@ -1081,23 +1081,21 @@ impl PutObjectOptions { } if !self.internal.source_version_id.is_empty() { - header.insert( - RUSTFS_BUCKET_SOURCE_VERSION_ID, - HeaderValue::from_str(&self.internal.source_version_id).expect("err"), - ); + insert_header(&mut header, SUFFIX_SOURCE_VERSION_ID, &self.internal.source_version_id); } if self.internal.source_etag.is_empty() { - header.insert(RUSTFS_BUCKET_SOURCE_ETAG, HeaderValue::from_str(&self.internal.source_etag).expect("err")); + insert_header(&mut header, SUFFIX_SOURCE_ETAG, &self.internal.source_etag); } if self.internal.source_mtime.unix_timestamp() != 0 { - header.insert( - RUSTFS_BUCKET_SOURCE_MTIME, - HeaderValue::from_str(&self.internal.source_mtime.format(&Rfc3339).unwrap_or_default()).expect("err"), + insert_header( + &mut header, + SUFFIX_SOURCE_MTIME, + self.internal.source_mtime.format(&Rfc3339).unwrap_or_default(), ); } if self.internal.replication_request { - header.insert(RUSTFS_BUCKET_REPLICATION_REQUEST, REPLICATION_REQUEST_TRUE); + insert_header(&mut header, SUFFIX_SOURCE_REPLICATION_REQUEST, "true"); } header @@ -1266,10 +1264,8 @@ impl TargetClient { let builder = self.client.put_object(); let version_id = opts.internal.source_version_id.clone(); - if !version_id.is_empty() - && let Ok(header_value) = HeaderValue::from_str(&version_id) - { - headers.insert(RUSTFS_BUCKET_SOURCE_VERSION_ID, header_value); + if !version_id.is_empty() { + insert_header(&mut headers, SUFFIX_SOURCE_VERSION_ID, &version_id); } match builder @@ -1303,13 +1299,11 @@ impl TargetClient { ) -> Result { let mut headers = HeaderMap::new(); let version_id = opts.internal.source_version_id.clone(); - if !version_id.is_empty() - && let Ok(header_value) = HeaderValue::from_str(&version_id) - { - headers.insert(RUSTFS_BUCKET_SOURCE_VERSION_ID, header_value); + if !version_id.is_empty() { + insert_header(&mut headers, SUFFIX_SOURCE_VERSION_ID, &version_id); } if opts.internal.replication_request { - headers.insert(RUSTFS_BUCKET_REPLICATION_REQUEST, REPLICATION_REQUEST_TRUE); + insert_header(&mut headers, SUFFIX_SOURCE_REPLICATION_REQUEST, "true"); } match self @@ -1418,21 +1412,18 @@ impl TargetClient { ) -> Result<(), S3ClientError> { let mut headers = HeaderMap::new(); if opts.force_delete { - headers.insert(RUSTFS_FORCE_DELETE, "true".parse().unwrap()); + insert_header(&mut headers, SUFFIX_FORCE_DELETE, "true"); } if opts.governance_bypass { headers.insert(AMZ_OBJECT_LOCK_BYPASS_GOVERNANCE, "true".parse().unwrap()); } if opts.replication_delete_marker { - headers.insert(RUSTFS_BUCKET_REPLICATION_DELETE_MARKER, "true".parse().unwrap()); + insert_header(&mut headers, SUFFIX_SOURCE_DELETEMARKER, "true"); } if let Some(t) = opts.replication_mtime { - headers.insert( - RUSTFS_BUCKET_SOURCE_MTIME, - t.format(&Rfc3339).unwrap_or_default().as_str().parse().unwrap(), - ); + insert_header(&mut headers, SUFFIX_SOURCE_MTIME, t.format(&Rfc3339).unwrap_or_default()); } if !opts.replication_status.is_empty() { @@ -1440,10 +1431,10 @@ impl TargetClient { } if opts.replication_request { - headers.insert(RUSTFS_BUCKET_REPLICATION_REQUEST, "true".parse().unwrap()); + insert_header(&mut headers, SUFFIX_SOURCE_REPLICATION_REQUEST, "true"); } if opts.replication_validity_check { - headers.insert(RUSTFS_BUCKET_REPLICATION_CHECK, "true".parse().unwrap()); + insert_header(&mut headers, SUFFIX_SOURCE_REPLICATION_CHECK, "true"); } match self diff --git a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs index 0dfb7d42f..d70742a07 100644 --- a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs +++ b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs @@ -45,8 +45,7 @@ use rustfs_common::heal_channel::rep_has_active_rules; use rustfs_common::metrics::{IlmAction, Metrics}; use rustfs_filemeta::{NULL_VERSION_ID, RestoreStatusOps, is_restored_object_on_disk}; use rustfs_s3_common::EventName; -use rustfs_utils::path::encode_dir_object; -use rustfs_utils::string::strings_has_prefix_fold; +use rustfs_utils::{get_env_i64, get_env_usize, path::encode_dir_object, string::strings_has_prefix_fold}; use s3s::Body; use s3s::dto::{ BucketLifecycleConfiguration, DefaultRetention, ReplicationConfiguration, RestoreRequest, RestoreRequestType, RestoreStatus, @@ -97,8 +96,14 @@ impl LifecycleSys { } pub async fn get(&self, bucket: &str) -> Option { - let lc = get_lifecycle_config(bucket).await.expect("get_lifecycle_config err!").0; - Some(lc) + match get_lifecycle_config(bucket).await { + Ok((lc, _)) => Some(lc), + Err(err) if err == Error::ConfigNotFound => None, + Err(err) => { + warn!(bucket, error = ?err, "failed to load lifecycle config"); + None + } + } } pub fn trace(_oi: &ObjectInfo) -> TraceFn { @@ -471,10 +476,7 @@ impl TransitionState { } pub async fn init(api: Arc) { - let max_workers = env::var("RUSTFS_MAX_TRANSITION_WORKERS") - .ok() - .and_then(|s| s.parse::().ok()) - .unwrap_or_else(|| std::cmp::min(num_cpus::get() as i64, 16)); + let max_workers = get_env_i64("RUSTFS_MAX_TRANSITION_WORKERS", std::cmp::min(num_cpus::get() as i64, 16)); let mut n = max_workers; let tw = 8; //globalILMConfig.getTransitionWorkers(); if tw > 0 { @@ -569,17 +571,11 @@ impl TransitionState { pub async fn update_workers_inner(api: Arc, n: i64) { let mut n = n; if n == 0 { - let max_workers = env::var("RUSTFS_MAX_TRANSITION_WORKERS") - .ok() - .and_then(|s| s.parse::().ok()) - .unwrap_or_else(|| std::cmp::min(num_cpus::get() as i64, 16)); + let max_workers = get_env_i64("RUSTFS_MAX_TRANSITION_WORKERS", std::cmp::min(num_cpus::get() as i64, 16)); n = max_workers; } // Allow environment override of maximum workers - let absolute_max = env::var("RUSTFS_ABSOLUTE_MAX_WORKERS") - .ok() - .and_then(|s| s.parse::().ok()) - .unwrap_or(32); + let absolute_max = get_env_i64("RUSTFS_ABSOLUTE_MAX_WORKERS", 32); n = std::cmp::min(n, absolute_max); let mut num_workers = GLOBAL_TransitionState.num_workers.load(Ordering::SeqCst); @@ -603,10 +599,7 @@ impl TransitionState { } pub async fn init_background_expiry(api: Arc) { - let mut workers = env::var("RUSTFS_MAX_EXPIRY_WORKERS") - .ok() - .and_then(|s| s.parse::().ok()) - .unwrap_or_else(|| std::cmp::min(num_cpus::get(), 16)); + let mut workers = get_env_usize("RUSTFS_MAX_EXPIRY_WORKERS", std::cmp::min(num_cpus::get(), 16)); //globalILMConfig.getExpirationWorkers() if let Ok(env_expiration_workers) = env::var("_RUSTFS_ILM_EXPIRATION_WORKERS") { if let Ok(num_expirations) = env_expiration_workers.parse::() { @@ -615,10 +608,7 @@ pub async fn init_background_expiry(api: Arc) { } if workers == 0 { - workers = env::var("RUSTFS_DEFAULT_EXPIRY_WORKERS") - .ok() - .and_then(|s| s.parse::().ok()) - .unwrap_or(8); + workers = get_env_usize("RUSTFS_DEFAULT_EXPIRY_WORKERS", 8); } //let expiry_state = GLOBAL_ExpiryStSate.write().await; diff --git a/crates/ecstore/src/bucket/lifecycle/lifecycle.rs b/crates/ecstore/src/bucket/lifecycle/lifecycle.rs index b27789b9f..0cbe7f6e7 100644 --- a/crates/ecstore/src/bucket/lifecycle/lifecycle.rs +++ b/crates/ecstore/src/bucket/lifecycle/lifecycle.rs @@ -49,6 +49,8 @@ const ERR_LIFECYCLE_INVALID_EXPIRATION_DAYS: &str = "Lifecycle expiration days m const ERR_LIFECYCLE_INVALID_EXPIRATION_DATE_NOT_MIDNIGHT: &str = "Expiration.Date must be at midnight UTC"; const ERR_LIFECYCLE_INVALID_RULE_ID_TOO_LONG: &str = "Rule ID must be at most 255 characters"; const ERR_LIFECYCLE_INVALID_RULE_STATUS: &str = "Rule status must be either Enabled or Disabled"; +const ERR_LIFECYCLE_DEL_MARKER_WITH_TAGS: &str = "Rule with DelMarkerExpiration cannot have tags based filtering"; +const ERR_LIFECYCLE_RULE_MUST_HAVE_ACTION: &str = "Rule must have at least one of Expiration, Transition, NoncurrentVersionExpiration, NoncurrentVersionTransition, or DelMarkerExpiration"; pub use rustfs_common::metrics::IlmAction; @@ -117,19 +119,43 @@ impl RuleValidate for LifecycleRule { }*/ fn validate(&self) -> Result<(), std::io::Error> { - /*self.validate_id()?; - self.validate_status()?; - self.validate_expiration()?; - self.validate_noncurrent_expiration()?; - self.validate_prefix_and_filter()?; - self.validate_transition()?; - self.validate_noncurrent_transition()?; - if (!self.Filter.Tag.IsEmpty() || len(self.Filter.And.Tags) != 0) && !self.delmarker_expiration.Empty() { - return errInvalidRuleDelMarkerExpiration + // Rule with DelMarkerExpiration cannot have tags based filtering + let has_tag_filter = self + .filter + .as_ref() + .map_or(false, |f| f.tag.is_some() || f.and.as_ref().and_then(|a| a.tags.as_ref()).is_some()); + if has_tag_filter && self.del_marker_expiration.is_some() { + return Err(std::io::Error::other(ERR_LIFECYCLE_DEL_MARKER_WITH_TAGS)); + } + // Rule must have at least one action + let has_expiration = self.expiration.is_some(); + let has_transition = self.transitions.as_ref().map_or(false, |t| !t.is_empty()); + let has_noncurrent_expiration = self + .noncurrent_version_expiration + .as_ref() + .and_then(|e| e.noncurrent_days) + .map_or(false, |d| d != 0); + let has_noncurrent_transition = self + .noncurrent_version_transitions + .as_ref() + .and_then(|t| t.first()) + .and_then(|t| t.storage_class.as_ref()) + .is_some(); + let has_abort_incomplete_multipart_upload = self.abort_incomplete_multipart_upload.is_some(); + let has_del_marker_expiration = self + .del_marker_expiration + .as_ref() + .and_then(|d| d.days) + .map_or(false, |d| d > 0); + if !has_expiration + && !has_transition + && !has_noncurrent_expiration + && !has_noncurrent_transition + && !has_abort_incomplete_multipart_upload + && !has_del_marker_expiration + { + return Err(std::io::Error::other(ERR_LIFECYCLE_RULE_MUST_HAVE_ACTION)); } - if !self.expiration.set && !self.transition.set && !self.noncurrent_version_expiration.set && !self.noncurrent_version_transitions.unwrap()[0].set && self.delmarker_expiration.Empty() { - return errXMLNotWellFormed - }*/ Ok(()) } } @@ -456,6 +482,27 @@ impl Lifecycle for BucketLifecycleConfiguration { } } } + // DelMarkerExpiration: expire delete marker after N days from mod_time + if obj.delete_marker { + if let Some(ref dme) = rule.del_marker_expiration { + if let Some(days) = dme.days { + if days > 0 { + let due = expected_expiry_time(mod_time, days); + if now.unix_timestamp() >= due.unix_timestamp() { + events.push(Event { + action: IlmAction::DelMarkerDeleteAllVersionsAction, + rule_id: rule.id.clone().unwrap_or_default(), + due: Some(due), + noncurrent_days: 0, + newer_noncurrent_versions: 0, + storage_class: "".into(), + }); + } + continue; + } + } + } + } } if !obj.is_latest { @@ -866,6 +913,7 @@ mod tests { #[serial] async fn validate_rejects_non_positive_expiration_days() { let lc = BucketLifecycleConfiguration { + expiry_updated_at: None, rules: vec![LifecycleRule { status: ExpirationStatus::from_static(ExpirationStatus::ENABLED), expiration: Some(LifecycleExpiration { @@ -873,6 +921,7 @@ mod tests { ..Default::default() }), abort_incomplete_multipart_upload: None, + del_marker_expiration: None, filter: None, id: None, noncurrent_version_expiration: None, @@ -894,6 +943,7 @@ mod tests { #[serial] async fn validate_accepts_positive_expiration_days() { let lc = BucketLifecycleConfiguration { + expiry_updated_at: None, rules: vec![LifecycleRule { status: ExpirationStatus::from_static(ExpirationStatus::ENABLED), expiration: Some(LifecycleExpiration { @@ -901,6 +951,7 @@ mod tests { ..Default::default() }), abort_incomplete_multipart_upload: None, + del_marker_expiration: None, filter: None, id: None, noncurrent_version_expiration: None, @@ -915,10 +966,37 @@ mod tests { .expect("expected validation to pass"); } + #[tokio::test] + #[serial] + async fn validate_accepts_abort_incomplete_multipart_upload_only_rule() { + let lc = BucketLifecycleConfiguration { + expiry_updated_at: None, + rules: vec![LifecycleRule { + status: ExpirationStatus::from_static(ExpirationStatus::ENABLED), + expiration: None, + abort_incomplete_multipart_upload: Some(s3s::dto::AbortIncompleteMultipartUpload { + days_after_initiation: Some(2), + }), + del_marker_expiration: None, + filter: None, + id: Some("abort-only".to_string()), + noncurrent_version_expiration: None, + noncurrent_version_transitions: None, + prefix: Some("test/".to_string()), + transitions: None, + }], + }; + + lc.validate(&ObjectLockConfiguration::default()) + .await + .expect("expected validation to pass"); + } + #[tokio::test] #[serial] async fn validate_rejects_non_midnight_expiration_date() { let lc = BucketLifecycleConfiguration { + expiry_updated_at: None, rules: vec![LifecycleRule { status: ExpirationStatus::from_static(ExpirationStatus::ENABLED), expiration: Some(LifecycleExpiration { @@ -926,6 +1004,7 @@ mod tests { ..Default::default() }), abort_incomplete_multipart_upload: None, + del_marker_expiration: None, filter: None, id: None, noncurrent_version_expiration: None, @@ -945,6 +1024,7 @@ mod tests { async fn predict_expiration_selects_closest_expiry_for_put_object() { let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap(); let lc = BucketLifecycleConfiguration { + expiry_updated_at: None, rules: vec![ LifecycleRule { status: ExpirationStatus::from_static(ExpirationStatus::ENABLED), @@ -953,6 +1033,7 @@ mod tests { ..Default::default() }), abort_incomplete_multipart_upload: None, + del_marker_expiration: None, filter: None, id: Some("rule-days".to_string()), noncurrent_version_expiration: None, @@ -967,6 +1048,7 @@ mod tests { ..Default::default() }), abort_incomplete_multipart_upload: None, + del_marker_expiration: None, filter: None, id: Some("rule-date".to_string()), noncurrent_version_expiration: None, @@ -996,6 +1078,7 @@ mod tests { #[serial] async fn validate_accepts_multiple_rules_without_ids() { let lc = BucketLifecycleConfiguration { + expiry_updated_at: None, rules: vec![ LifecycleRule { status: ExpirationStatus::from_static(ExpirationStatus::ENABLED), @@ -1004,6 +1087,7 @@ mod tests { ..Default::default() }), abort_incomplete_multipart_upload: None, + del_marker_expiration: None, filter: None, id: None, noncurrent_version_expiration: None, @@ -1018,6 +1102,7 @@ mod tests { ..Default::default() }), abort_incomplete_multipart_upload: None, + del_marker_expiration: None, filter: None, id: None, noncurrent_version_expiration: None, @@ -1037,6 +1122,7 @@ mod tests { #[serial] async fn validate_rejects_rule_id_too_long() { let lc = BucketLifecycleConfiguration { + expiry_updated_at: None, rules: vec![LifecycleRule { status: ExpirationStatus::from_static(ExpirationStatus::ENABLED), expiration: Some(LifecycleExpiration { @@ -1044,6 +1130,7 @@ mod tests { ..Default::default() }), abort_incomplete_multipart_upload: None, + del_marker_expiration: None, filter: None, id: Some("a".repeat(256)), noncurrent_version_expiration: None, @@ -1062,6 +1149,7 @@ mod tests { #[serial] async fn validate_rejects_duplicate_rule_ids() { let lc = BucketLifecycleConfiguration { + expiry_updated_at: None, rules: vec![ LifecycleRule { status: ExpirationStatus::from_static(ExpirationStatus::ENABLED), @@ -1070,6 +1158,7 @@ mod tests { ..Default::default() }), abort_incomplete_multipart_upload: None, + del_marker_expiration: None, filter: None, id: Some("dup-rule".to_string()), noncurrent_version_expiration: None, @@ -1090,6 +1179,7 @@ mod tests { noncurrent_version_transitions: None, prefix: None, transitions: None, + del_marker_expiration: None, }, ], }; @@ -1103,6 +1193,7 @@ mod tests { async fn eval_inner_expires_latest_object_after_days_due() { let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap(); let lc = BucketLifecycleConfiguration { + expiry_updated_at: None, rules: vec![LifecycleRule { status: ExpirationStatus::from_static(ExpirationStatus::ENABLED), expiration: Some(LifecycleExpiration { @@ -1110,6 +1201,7 @@ mod tests { ..Default::default() }), abort_incomplete_multipart_upload: None, + del_marker_expiration: None, filter: None, id: Some("expire-days".to_string()), noncurrent_version_expiration: None, @@ -1137,6 +1229,7 @@ mod tests { async fn eval_inner_keeps_latest_object_before_days_due() { let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap(); let lc = BucketLifecycleConfiguration { + expiry_updated_at: None, rules: vec![LifecycleRule { status: ExpirationStatus::from_static(ExpirationStatus::ENABLED), expiration: Some(LifecycleExpiration { @@ -1144,6 +1237,7 @@ mod tests { ..Default::default() }), abort_incomplete_multipart_upload: None, + del_marker_expiration: None, filter: None, id: Some("expire-days".to_string()), noncurrent_version_expiration: None, @@ -1169,10 +1263,12 @@ mod tests { async fn eval_inner_transitions_latest_object_after_days_due() { let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap(); let lc = BucketLifecycleConfiguration { + expiry_updated_at: None, rules: vec![LifecycleRule { status: ExpirationStatus::from_static(ExpirationStatus::ENABLED), expiration: None, abort_incomplete_multipart_upload: None, + del_marker_expiration: None, filter: None, id: Some("transition-days".to_string()), noncurrent_version_expiration: None, @@ -1205,10 +1301,12 @@ mod tests { async fn eval_inner_expires_noncurrent_version_after_due() { let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap(); let lc = BucketLifecycleConfiguration { + expiry_updated_at: None, rules: vec![LifecycleRule { status: ExpirationStatus::from_static(ExpirationStatus::ENABLED), expiration: None, abort_incomplete_multipart_upload: None, + del_marker_expiration: None, filter: None, id: Some("noncurrent-expire".to_string()), noncurrent_version_expiration: Some(s3s::dto::NoncurrentVersionExpiration { @@ -1241,10 +1339,12 @@ mod tests { async fn eval_inner_transitions_noncurrent_version_after_due() { let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap(); let lc = BucketLifecycleConfiguration { + expiry_updated_at: None, rules: vec![LifecycleRule { status: ExpirationStatus::from_static(ExpirationStatus::ENABLED), expiration: None, abort_incomplete_multipart_upload: None, + del_marker_expiration: None, filter: None, id: Some("noncurrent-transition".to_string()), noncurrent_version_expiration: None, @@ -1278,10 +1378,12 @@ mod tests { #[serial] async fn noncurrent_versions_expiration_limit_returns_configured_limits() { let lc = Arc::new(BucketLifecycleConfiguration { + expiry_updated_at: None, rules: vec![LifecycleRule { status: ExpirationStatus::from_static(ExpirationStatus::ENABLED), expiration: None, abort_incomplete_multipart_upload: None, + del_marker_expiration: None, filter: None, id: Some("noncurrent-limit".to_string()), noncurrent_version_expiration: Some(s3s::dto::NoncurrentVersionExpiration { @@ -1313,6 +1415,7 @@ mod tests { #[serial] async fn validate_rejects_invalid_status_case_sensitive() { let lc = BucketLifecycleConfiguration { + expiry_updated_at: None, rules: vec![LifecycleRule { status: ExpirationStatus::from_static("enabled"), expiration: Some(LifecycleExpiration { @@ -1320,6 +1423,7 @@ mod tests { ..Default::default() }), abort_incomplete_multipart_upload: None, + del_marker_expiration: None, filter: None, id: None, noncurrent_version_expiration: None, @@ -1340,6 +1444,7 @@ mod tests { let mut filter = LifecycleRuleFilter::default(); filter.prefix = Some("prefix".to_string()); let lc = BucketLifecycleConfiguration { + expiry_updated_at: None, rules: vec![LifecycleRule { status: ExpirationStatus::from_static(ExpirationStatus::ENABLED), expiration: Some(LifecycleExpiration { @@ -1353,6 +1458,7 @@ mod tests { noncurrent_version_transitions: None, prefix: None, transitions: None, + del_marker_expiration: None, }], }; @@ -1385,6 +1491,7 @@ mod tests { filter.and = Some(and); let lc = BucketLifecycleConfiguration { + expiry_updated_at: None, rules: vec![LifecycleRule { status: ExpirationStatus::from_static(ExpirationStatus::ENABLED), expiration: Some(LifecycleExpiration { @@ -1398,6 +1505,7 @@ mod tests { noncurrent_version_transitions: None, prefix: None, transitions: None, + del_marker_expiration: None, }], }; @@ -1425,6 +1533,7 @@ mod tests { async fn expired_object_delete_marker_requires_single_version() { let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap(); let lc = BucketLifecycleConfiguration { + expiry_updated_at: None, rules: vec![LifecycleRule { status: ExpirationStatus::from_static(ExpirationStatus::ENABLED), expiration: Some(LifecycleExpiration { @@ -1439,6 +1548,7 @@ mod tests { noncurrent_version_transitions: None, prefix: None, transitions: None, + del_marker_expiration: None, }], }; @@ -1462,6 +1572,7 @@ mod tests { async fn expired_object_delete_marker_deletes_only_delete_marker_after_due() { let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap(); let lc = BucketLifecycleConfiguration { + expiry_updated_at: None, rules: vec![LifecycleRule { status: ExpirationStatus::from_static(ExpirationStatus::ENABLED), expiration: Some(LifecycleExpiration { @@ -1476,6 +1587,7 @@ mod tests { noncurrent_version_transitions: None, prefix: None, transitions: None, + del_marker_expiration: None, }], }; @@ -1500,6 +1612,7 @@ mod tests { async fn expired_object_delete_marker_without_date_or_days_deletes_immediately() { let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap(); let lc = BucketLifecycleConfiguration { + expiry_updated_at: None, rules: vec![LifecycleRule { status: ExpirationStatus::from_static(ExpirationStatus::ENABLED), expiration: Some(LifecycleExpiration { @@ -1513,6 +1626,7 @@ mod tests { noncurrent_version_transitions: None, prefix: None, transitions: None, + del_marker_expiration: None, }], }; @@ -1539,6 +1653,7 @@ mod tests { let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap(); let future_date = base_time + Duration::days(10); let lc = BucketLifecycleConfiguration { + expiry_updated_at: None, rules: vec![LifecycleRule { status: ExpirationStatus::from_static(ExpirationStatus::ENABLED), expiration: Some(LifecycleExpiration { @@ -1553,6 +1668,7 @@ mod tests { noncurrent_version_transitions: None, prefix: None, transitions: None, + del_marker_expiration: None, }], }; diff --git a/crates/ecstore/src/bucket/metadata.rs b/crates/ecstore/src/bucket/metadata.rs index ec4140466..6680f6c15 100644 --- a/crates/ecstore/src/bucket/metadata.rs +++ b/crates/ecstore/src/bucket/metadata.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use super::msgp_decode::{read_msgp_ext8_time, skip_msgp_value, write_msgp_time}; use super::object_lock::ObjectLockApi; use super::versioning::VersioningApi; use super::{quota::BucketQuota, target::BucketTargets}; @@ -22,7 +23,6 @@ use crate::error::{Error, Result}; use crate::new_object_layer_fn; use crate::store::ECStore; use byteorder::{BigEndian, ByteOrder, LittleEndian}; -use rmp_serde::Serializer as rmpSerializer; use rustfs_policy::policy::BucketPolicy; use s3s::dto::{ BucketLifecycleConfiguration, CORSConfiguration, NotificationConfiguration, ObjectLockConfiguration, @@ -30,12 +30,41 @@ use s3s::dto::{ VersioningConfiguration, }; use serde::Serializer; -use serde::{Deserialize, Serialize}; use std::collections::HashMap; +use std::io::{Read, Write}; use std::sync::Arc; use time::OffsetDateTime; use tracing::error; +fn read_msgp_str(rd: &mut R) -> Result { + let len = rmp::decode::read_str_len(rd)? as usize; + let mut buf = vec![0u8; len]; + rd.read_exact(&mut buf)?; + Ok(String::from_utf8(buf)?) +} + +fn read_msgp_time_value(rd: &mut R) -> Result { + let marker = rmp::decode::read_marker(rd).map_err(|e| Error::other(format!("{e:?}")))?; + match marker { + rmp::Marker::Null => Ok(OffsetDateTime::UNIX_EPOCH), + rmp::Marker::Ext8 => read_msgp_ext8_time(rd), + _ => Err(Error::other(format!("expected time ext or nil, got marker: {marker:?}"))), + } +} + +fn read_msgp_bin(rd: &mut R) -> Result> { + let len = rmp::decode::read_bin_len(rd)? as usize; + let mut buf = vec![0u8; len]; + rd.read_exact(&mut buf)?; + Ok(buf) +} + +fn write_bin_field(wr: &mut W, key: &str, val: &[u8]) -> Result<()> { + rmp::encode::write_str(wr, key)?; + rmp::encode::write_bin(wr, val)?; + Ok(()) +} + pub const BUCKET_METADATA_FILE: &str = ".metadata.bin"; pub const BUCKET_METADATA_FORMAT: u16 = 1; pub const BUCKET_METADATA_VERSION: u16 = 1; @@ -54,8 +83,7 @@ pub const BUCKET_CORS_CONFIG: &str = "cors.xml"; pub const BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG: &str = "public-access-block.xml"; pub const BUCKET_ACL_CONFIG: &str = "bucket-acl.json"; -#[derive(Debug, Deserialize, Serialize, Clone)] -#[serde(rename_all = "PascalCase", default)] +#[derive(Debug, Clone)] pub struct BucketMetadata { pub name: String, pub created: OffsetDateTime, @@ -90,36 +118,21 @@ pub struct BucketMetadata { pub public_access_block_config_updated_at: OffsetDateTime, pub bucket_acl_config_updated_at: OffsetDateTime, - #[serde(skip)] pub new_field_updated_at: OffsetDateTime, - #[serde(skip)] pub policy_config: Option, - #[serde(skip)] pub notification_config: Option, - #[serde(skip)] pub lifecycle_config: Option, - #[serde(skip)] pub object_lock_config: Option, - #[serde(skip)] pub versioning_config: Option, - #[serde(skip)] pub sse_config: Option, - #[serde(skip)] pub tagging_config: Option, - #[serde(skip)] pub quota_config: Option, - #[serde(skip)] pub replication_config: Option, - #[serde(skip)] pub bucket_target_config: Option, - #[serde(skip)] pub bucket_target_config_meta: Option>, - #[serde(skip)] pub cors_config: Option, - #[serde(skip)] pub public_access_block_config: Option, - #[serde(skip)] pub bucket_acl_config: Option, } @@ -198,17 +211,137 @@ impl BucketMetadata { self.lock_enabled || (self.versioning_config.as_ref().is_some_and(|v| v.enabled())) } + /// Decode from msgp bytes. Field order follows MinIO BucketMetadata for compatibility. + pub fn decode_from(&mut self, rd: &mut R) -> Result<()> { + let mut fields = rmp::decode::read_map_len(rd)?; + *self = Self::default(); + + while fields > 0 { + fields -= 1; + + let key_len = rmp::decode::read_str_len(rd)?; + let mut key_buf = vec![0u8; key_len as usize]; + rd.read_exact(&mut key_buf)?; + let key = String::from_utf8(key_buf)?; + + match key.as_str() { + "Name" => self.name = read_msgp_str(rd)?, + "Created" => self.created = read_msgp_time_value(rd)?, + "LockEnabled" => self.lock_enabled = rmp::decode::read_bool(rd)?, + "PolicyConfigJSON" => self.policy_config_json = read_msgp_bin(rd)?, + "NotificationConfigXML" => self.notification_config_xml = read_msgp_bin(rd)?, + "LifecycleConfigXML" => self.lifecycle_config_xml = read_msgp_bin(rd)?, + "ObjectLockConfigXML" => self.object_lock_config_xml = read_msgp_bin(rd)?, + "VersioningConfigXML" => self.versioning_config_xml = read_msgp_bin(rd)?, + "EncryptionConfigXML" => self.encryption_config_xml = read_msgp_bin(rd)?, + "TaggingConfigXML" => self.tagging_config_xml = read_msgp_bin(rd)?, + "QuotaConfigJSON" => self.quota_config_json = read_msgp_bin(rd)?, + "ReplicationConfigXML" => self.replication_config_xml = read_msgp_bin(rd)?, + "BucketTargetsConfigJSON" => self.bucket_targets_config_json = read_msgp_bin(rd)?, + "BucketTargetsConfigMetaJSON" => self.bucket_targets_config_meta_json = read_msgp_bin(rd)?, + "PolicyConfigUpdatedAt" => self.policy_config_updated_at = read_msgp_time_value(rd)?, + "ObjectLockConfigUpdatedAt" => self.object_lock_config_updated_at = read_msgp_time_value(rd)?, + "EncryptionConfigUpdatedAt" => self.encryption_config_updated_at = read_msgp_time_value(rd)?, + "TaggingConfigUpdatedAt" => self.tagging_config_updated_at = read_msgp_time_value(rd)?, + "QuotaConfigUpdatedAt" => self.quota_config_updated_at = read_msgp_time_value(rd)?, + "ReplicationConfigUpdatedAt" => self.replication_config_updated_at = read_msgp_time_value(rd)?, + "VersioningConfigUpdatedAt" => self.versioning_config_updated_at = read_msgp_time_value(rd)?, + "LifecycleConfigUpdatedAt" => self.lifecycle_config_updated_at = read_msgp_time_value(rd)?, + "NotificationConfigUpdatedAt" => self.notification_config_updated_at = read_msgp_time_value(rd)?, + "BucketTargetsConfigUpdatedAt" => self.bucket_targets_config_updated_at = read_msgp_time_value(rd)?, + "BucketTargetsConfigMetaUpdatedAt" => self.bucket_targets_config_meta_updated_at = read_msgp_time_value(rd)?, + "CorsConfigXML" => self.cors_config_xml = read_msgp_bin(rd)?, + "PublicAccessBlockConfigXML" => self.public_access_block_config_xml = read_msgp_bin(rd)?, + "BucketAclConfigJSON" => self.bucket_acl_config_json = read_msgp_bin(rd)?, + "CorsConfigUpdatedAt" => self.cors_config_updated_at = read_msgp_time_value(rd)?, + "PublicAccessBlockConfigUpdatedAt" => self.public_access_block_config_updated_at = read_msgp_time_value(rd)?, + "BucketAclConfigUpdatedAt" => self.bucket_acl_config_updated_at = read_msgp_time_value(rd)?, + other => { + tracing::debug!(field = %other, "BucketMetadata decode_from: skipping unknown field"); + skip_msgp_value(rd)?; + } + } + } + + Ok(()) + } + + /// Encode to msgp bytes. Field order follows MinIO BucketMetadata for compatibility. + pub fn encode_to(&self, wr: &mut W) -> Result<()> { + // Map size: MinIO fields (25) + RustFS extensions (6) + let map_len: u32 = 31; + rmp::encode::write_map_len(wr, map_len)?; + + // MinIO field order (same as Go struct) + rmp::encode::write_str(wr, "Name")?; + rmp::encode::write_str(wr, &self.name)?; + + rmp::encode::write_str(wr, "Created")?; + write_msgp_time(wr, self.created)?; + + rmp::encode::write_str(wr, "LockEnabled")?; + rmp::encode::write_bool(wr, self.lock_enabled)?; + + write_bin_field(wr, "PolicyConfigJSON", &self.policy_config_json)?; + write_bin_field(wr, "NotificationConfigXML", &self.notification_config_xml)?; + write_bin_field(wr, "LifecycleConfigXML", &self.lifecycle_config_xml)?; + write_bin_field(wr, "ObjectLockConfigXML", &self.object_lock_config_xml)?; + write_bin_field(wr, "VersioningConfigXML", &self.versioning_config_xml)?; + write_bin_field(wr, "EncryptionConfigXML", &self.encryption_config_xml)?; + write_bin_field(wr, "TaggingConfigXML", &self.tagging_config_xml)?; + write_bin_field(wr, "QuotaConfigJSON", &self.quota_config_json)?; + write_bin_field(wr, "ReplicationConfigXML", &self.replication_config_xml)?; + write_bin_field(wr, "BucketTargetsConfigJSON", &self.bucket_targets_config_json)?; + write_bin_field(wr, "BucketTargetsConfigMetaJSON", &self.bucket_targets_config_meta_json)?; + + rmp::encode::write_str(wr, "PolicyConfigUpdatedAt")?; + write_msgp_time(wr, self.policy_config_updated_at)?; + rmp::encode::write_str(wr, "ObjectLockConfigUpdatedAt")?; + write_msgp_time(wr, self.object_lock_config_updated_at)?; + rmp::encode::write_str(wr, "EncryptionConfigUpdatedAt")?; + write_msgp_time(wr, self.encryption_config_updated_at)?; + rmp::encode::write_str(wr, "TaggingConfigUpdatedAt")?; + write_msgp_time(wr, self.tagging_config_updated_at)?; + rmp::encode::write_str(wr, "QuotaConfigUpdatedAt")?; + write_msgp_time(wr, self.quota_config_updated_at)?; + rmp::encode::write_str(wr, "ReplicationConfigUpdatedAt")?; + write_msgp_time(wr, self.replication_config_updated_at)?; + rmp::encode::write_str(wr, "VersioningConfigUpdatedAt")?; + write_msgp_time(wr, self.versioning_config_updated_at)?; + rmp::encode::write_str(wr, "LifecycleConfigUpdatedAt")?; + write_msgp_time(wr, self.lifecycle_config_updated_at)?; + rmp::encode::write_str(wr, "NotificationConfigUpdatedAt")?; + write_msgp_time(wr, self.notification_config_updated_at)?; + rmp::encode::write_str(wr, "BucketTargetsConfigUpdatedAt")?; + write_msgp_time(wr, self.bucket_targets_config_updated_at)?; + rmp::encode::write_str(wr, "BucketTargetsConfigMetaUpdatedAt")?; + write_msgp_time(wr, self.bucket_targets_config_meta_updated_at)?; + + // RustFS extensions + write_bin_field(wr, "CorsConfigXML", &self.cors_config_xml)?; + write_bin_field(wr, "PublicAccessBlockConfigXML", &self.public_access_block_config_xml)?; + write_bin_field(wr, "BucketAclConfigJSON", &self.bucket_acl_config_json)?; + rmp::encode::write_str(wr, "CorsConfigUpdatedAt")?; + write_msgp_time(wr, self.cors_config_updated_at)?; + rmp::encode::write_str(wr, "PublicAccessBlockConfigUpdatedAt")?; + write_msgp_time(wr, self.public_access_block_config_updated_at)?; + rmp::encode::write_str(wr, "BucketAclConfigUpdatedAt")?; + write_msgp_time(wr, self.bucket_acl_config_updated_at)?; + + Ok(()) + } + pub fn marshal_msg(&self) -> Result> { let mut buf = Vec::new(); - - self.serialize(&mut rmpSerializer::new(&mut buf).with_struct_map())?; - + self.encode_to(&mut buf)?; Ok(buf) } pub fn unmarshal(buf: &[u8]) -> Result { - let t: BucketMetadata = rmp_serde::from_slice(buf)?; - Ok(t) + let mut bm = Self::default(); + let mut cur = std::io::Cursor::new(buf); + bm.decode_from(&mut cur)?; + Ok(bm) } pub fn check_header(buf: &[u8]) -> Result<()> { @@ -382,50 +515,80 @@ impl BucketMetadata { } fn parse_all_configs(&mut self, _api: Arc) -> Result<()> { - self.parse_policy_config()?; - if !self.notification_config_xml.is_empty() { - self.notification_config = Some(deserialize::(&self.notification_config_xml)?); + if let Err(e) = self.parse_policy_config() { + tracing::warn!(bucket = %self.name, config = "policy", error = %e, "parse_all_configs: failed to parse"); } - if !self.lifecycle_config_xml.is_empty() { - self.lifecycle_config = Some(deserialize::(&self.lifecycle_config_xml)?); + if !self.notification_config_xml.is_empty() + && let Err(e) = deserialize::(&self.notification_config_xml) + .map(|c| self.notification_config = Some(c)) + { + tracing::warn!(bucket = %self.name, config = "notification", error = %e, "parse_all_configs: failed to parse"); } - - if !self.object_lock_config_xml.is_empty() { - self.object_lock_config = Some(deserialize::(&self.object_lock_config_xml)?); + if !self.lifecycle_config_xml.is_empty() + && let Err(e) = + deserialize::(&self.lifecycle_config_xml).map(|c| self.lifecycle_config = Some(c)) + { + tracing::warn!(bucket = %self.name, config = "lifecycle", error = %e, "parse_all_configs: failed to parse"); } - if !self.versioning_config_xml.is_empty() { - self.versioning_config = Some(deserialize::(&self.versioning_config_xml)?); + if !self.object_lock_config_xml.is_empty() + && let Err(e) = + deserialize::(&self.object_lock_config_xml).map(|c| self.object_lock_config = Some(c)) + { + tracing::warn!(bucket = %self.name, config = "object_lock", error = %e, "parse_all_configs: failed to parse"); } - if !self.encryption_config_xml.is_empty() { - self.sse_config = Some(deserialize::(&self.encryption_config_xml)?); + if !self.versioning_config_xml.is_empty() + && let Err(e) = + deserialize::(&self.versioning_config_xml).map(|c| self.versioning_config = Some(c)) + { + tracing::warn!(bucket = %self.name, config = "versioning", error = %e, "parse_all_configs: failed to parse"); } - if !self.tagging_config_xml.is_empty() { - self.tagging_config = Some(deserialize::(&self.tagging_config_xml)?); + if !self.encryption_config_xml.is_empty() + && let Err(e) = + deserialize::(&self.encryption_config_xml).map(|c| self.sse_config = Some(c)) + { + tracing::warn!(bucket = %self.name, config = "encryption", error = %e, "parse_all_configs: failed to parse"); } - if !self.quota_config_json.is_empty() { - self.quota_config = Some(serde_json::from_slice(&self.quota_config_json)?); + if !self.tagging_config_xml.is_empty() + && let Err(e) = deserialize::(&self.tagging_config_xml).map(|c| self.tagging_config = Some(c)) + { + tracing::warn!(bucket = %self.name, config = "tagging", error = %e, "parse_all_configs: failed to parse"); } - if !self.replication_config_xml.is_empty() { - self.replication_config = Some(deserialize::(&self.replication_config_xml)?); + if !self.quota_config_json.is_empty() + && let Err(e) = serde_json::from_slice(&self.quota_config_json).map(|c| self.quota_config = Some(c)) + { + tracing::warn!(bucket = %self.name, config = "quota", error = %e, "parse_all_configs: failed to parse"); + } + if !self.replication_config_xml.is_empty() + && let Err(e) = + deserialize::(&self.replication_config_xml).map(|c| self.replication_config = Some(c)) + { + tracing::warn!(bucket = %self.name, config = "replication", error = %e, "parse_all_configs: failed to parse"); } - //let temp = self.bucket_targets_config_json.clone(); if !self.bucket_targets_config_json.is_empty() { - let bucket_targets: BucketTargets = serde_json::from_slice(&self.bucket_targets_config_json)?; - self.bucket_target_config = Some(bucket_targets); + if let Err(e) = serde_json::from_slice::(&self.bucket_targets_config_json) + .map(|t| self.bucket_target_config = Some(t)) + { + tracing::warn!(bucket = %self.name, config = "bucket_targets", error = %e, "parse_all_configs: failed to parse"); + self.bucket_target_config = Some(BucketTargets::default()); + } } else { - self.bucket_target_config = Some(BucketTargets::default()) + self.bucket_target_config = Some(BucketTargets::default()); } - if !self.cors_config_xml.is_empty() { - self.cors_config = Some(deserialize::(&self.cors_config_xml)?); + if !self.cors_config_xml.is_empty() + && let Err(e) = deserialize::(&self.cors_config_xml).map(|c| self.cors_config = Some(c)) + { + tracing::warn!(bucket = %self.name, config = "cors", error = %e, "parse_all_configs: failed to parse"); } - if !self.public_access_block_config_xml.is_empty() { - self.public_access_block_config = - Some(deserialize::(&self.public_access_block_config_xml)?); + if !self.public_access_block_config_xml.is_empty() + && let Err(e) = deserialize::(&self.public_access_block_config_xml) + .map(|c| self.public_access_block_config = Some(c)) + { + tracing::warn!(bucket = %self.name, config = "public_access_block", error = %e, "parse_all_configs: failed to parse"); } - if !self.bucket_acl_config_json.is_empty() { - let acl = String::from_utf8(self.bucket_acl_config_json.clone()) - .map_err(|e| Error::other(format!("invalid UTF-8 in bucket ACL: {}", e)))?; - self.bucket_acl_config = Some(acl); + if !self.bucket_acl_config_json.is_empty() + && let Err(e) = String::from_utf8(self.bucket_acl_config_json.clone()).map(|acl| self.bucket_acl_config = Some(acl)) + { + tracing::warn!(bucket = %self.name, config = "bucket_acl", error = %e, "parse_all_configs: failed to parse"); } Ok(()) @@ -478,7 +641,6 @@ async fn read_bucket_metadata(api: Arc, bucket: &str) -> Result(t: &OffsetDateTime, s: S) -> std::result::Result where S: Serializer, diff --git a/crates/ecstore/src/bucket/metadata_sys.rs b/crates/ecstore/src/bucket/metadata_sys.rs index 531cefcea..b6ae15fc3 100644 --- a/crates/ecstore/src/bucket/metadata_sys.rs +++ b/crates/ecstore/src/bucket/metadata_sys.rs @@ -360,12 +360,14 @@ impl BucketMetadataSys { }; if !meta.lifecycle_config_xml.is_empty() { - let cfg = deserialize::(&meta.lifecycle_config_xml)?; - // TODO: FIXME: - // for _v in cfg.rules.iter() { - // break; - // } - if let Some(_v) = cfg.rules.first() {} + if let Ok(cfg) = deserialize::(&meta.lifecycle_config_xml) { + if let Some(_v) = cfg.rules.first() {} + } else { + tracing::warn!( + bucket = %bucket, + "delete: failed to parse lifecycle config XML" + ); + } } // TODO: other lifecycle handle diff --git a/crates/ecstore/src/bucket/metadata_test.rs b/crates/ecstore/src/bucket/metadata_test.rs new file mode 100644 index 000000000..335594275 --- /dev/null +++ b/crates/ecstore/src/bucket/metadata_test.rs @@ -0,0 +1,261 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::metadata::BucketMetadata; +use time::OffsetDateTime; + +/// Full BucketMetadata hex (all fields populated). +const TEST_BUCKET_METADATA_HEX: &str = "de0019a44e616d65b27275737466732d636f6d7061742d74657374a743726561746564c70c050000000065920080075bcd15ab4c6f636b456e61626c6564c3b0506f6c696379436f6e6669674a534f4ec4907b2256657273696f6e223a22323031322d31302d3137222c2253746174656d656e74223a5b7b22456666656374223a22416c6c6f77222c225072696e636970616c223a222a222c22416374696f6e223a2273333a4765744f626a656374222c225265736f75726365223a2261726e3a6177733a73333a3a3a7275737466732d636f6d7061742d746573742f2a227d5d7db54e6f74696669636174696f6e436f6e666967584d4cc4963c4e6f74696669636174696f6e436f6e66696775726174696f6e3e3c436c6f75645761746368436f6e66696775726174696f6e3e3c49643e6e313c2f49643e3c4576656e743e73333a4f626a656374437265617465643a2a3c2f4576656e743e3c2f436c6f75645761746368436f6e66696775726174696f6e3e3c2f4e6f74696669636174696f6e436f6e66696775726174696f6e3eb24c6966656379636c65436f6e666967584d4cc48c3c4c6966656379636c65436f6e66696775726174696f6e3e3c52756c653e3c49443e72756c65313c2f49443e3c5374617475733e456e61626c65643c2f5374617475733e3c45787069726174696f6e3e3c446179733e33303c2f446179733e3c2f45787069726174696f6e3e3c2f52756c653e3c2f4c6966656379636c65436f6e66696775726174696f6e3eb34f626a6563744c6f636b436f6e666967584d4cc4b83c4f626a6563744c6f636b436f6e66696775726174696f6e3e3c4f626a6563744c6f636b456e61626c65643e456e61626c65643c2f4f626a6563744c6f636b456e61626c65643e3c52756c653e3c44656661756c74526574656e74696f6e3e3c4d6f64653e474f5645524e414e43453c2f4d6f64653e3c446179733e373c2f446179733e3c2f44656661756c74526574656e74696f6e3e3c2f52756c653e3c2f4f626a6563744c6f636b436f6e66696775726174696f6e3eb356657273696f6e696e67436f6e666967584d4cc44b3c56657273696f6e696e67436f6e66696775726174696f6e3e3c5374617475733e456e61626c65643c2f5374617475733e3c2f56657273696f6e696e67436f6e66696775726174696f6e3eb3456e6372797074696f6e436f6e666967584d4cc4c03c53657276657253696465456e6372797074696f6e436f6e66696775726174696f6e3e3c52756c653e3c4170706c7953657276657253696465456e6372797074696f6e427944656661756c743e3c535345416c676f726974686d3e4145533235363c2f535345416c676f726974686d3e3c2f4170706c7953657276657253696465456e6372797074696f6e427944656661756c743e3c2f52756c653e3c2f53657276657253696465456e6372797074696f6e436f6e66696775726174696f6e3eb054616767696e67436f6e666967584d4cc4503c54616767696e673e3c5461675365743e3c5461673e3c4b65793e456e763c2f4b65793e3c56616c75653e546573743c2f56616c75653e3c2f5461673e3c2f5461675365743e3c2f54616767696e673eaf51756f7461436f6e6669674a534f4ec4707b2271756f7461223a313037333734313832342c2271756f74615f74797065223a2248617264222c22637265617465645f6174223a22323032342d30312d30315430303a30303a30305a222c22757064617465645f6174223a22323032342d30312d30315430303a30303a30305a227db45265706c69636174696f6e436f6e666967584d4cc4e73c5265706c69636174696f6e436f6e66696775726174696f6e3e3c526f6c653e61726e3a6177733a69616d3a3a3132333435363738393031323a726f6c652f7265706c3c2f526f6c653e3c52756c653e3c49443e72313c2f49443e3c5374617475733e456e61626c65643c2f5374617475733e3c5072656669783e646f632f3c2f5072656669783e3c44657374696e6174696f6e3e3c4275636b65743e61726e3a6177733a73333a3a3a646573743c2f4275636b65743e3c2f44657374696e6174696f6e3e3c2f52756c653e3c2f5265706c69636174696f6e436f6e66696775726174696f6e3eb74275636b657454617267657473436f6e6669674a534f4ec4535b7b22656e64706f696e74223a22687474703a2f2f7461726765742e6578616d706c652e636f6d222c227461726765744275636b6574223a227462222c22726567696f6e223a2275732d656173742d31227d5dbb4275636b657454617267657473436f6e6669674d6574614a534f4ec42d7b227265706c69636174696f6e4964223a227265706c2d31222c2273796e634d6f6465223a226173796e63227db5506f6c696379436f6e666967557064617465644174c70c050000000065a5022000000000b94f626a6563744c6f636b436f6e666967557064617465644174c70c050000000065a5022000000000b9456e6372797074696f6e436f6e666967557064617465644174c70c050000000065a5022000000000b654616767696e67436f6e666967557064617465644174c70c050000000065a5022000000000b451756f7461436f6e666967557064617465644174c70c050000000065a5022000000000ba5265706c69636174696f6e436f6e666967557064617465644174c70c050000000065a5022000000000b956657273696f6e696e67436f6e666967557064617465644174c70c050000000065a5022000000000b84c6966656379636c65436f6e666967557064617465644174c70c050000000065a5022000000000bb4e6f74696669636174696f6e436f6e666967557064617465644174c70c050000000065a5022000000000bc4275636b657454617267657473436f6e666967557064617465644174c70c050000000065a5022000000000d9204275636b657454617267657473436f6e6669674d657461557064617465644174c70c050000000065a5022000000000"; + +#[tokio::test] +async fn marshal_msg() { + let bm = BucketMetadata::new("dada"); + + let buf = bm.marshal_msg().unwrap(); + + let new = BucketMetadata::unmarshal(&buf).unwrap(); + + assert_eq!(bm.name, new.name); +} + +/// Verifies that serialized time uses msgp ext type 5. +#[tokio::test] +async fn marshal_msg_uses_time_format() { + let mut bm = BucketMetadata::new("test-bucket"); + bm.created = OffsetDateTime::from_unix_timestamp(1704067200).unwrap(); // 2024-01-01 00:00:00 UTC + + let buf = bm.marshal_msg().unwrap(); + + // msgp uses ext8 (0xc7), len 12, type 5 for time + assert!( + buf.windows(3).any(|w| w == [0xc7, 0x0c, 0x05]), + "serialized data should contain msgp time ext (0xc7 0x0c 0x05)" + ); +} + +#[tokio::test] +async fn unmarshal_test_bucket_metadata() { + use faster_hex::hex_decode; + + let mut bytes = vec![0u8; TEST_BUCKET_METADATA_HEX.len() / 2]; + hex_decode(TEST_BUCKET_METADATA_HEX.as_bytes(), &mut bytes).expect("valid hex"); + let bm = BucketMetadata::unmarshal(&bytes).expect("RustFS must unmarshal MinIO format"); + + assert_eq!(bm.name, "rustfs-compat-test"); + assert_eq!(bm.created.unix_timestamp(), 1704067200); + assert_eq!(bm.created.nanosecond(), 123456789); + assert!(bm.lock_enabled); + + assert!(!bm.policy_config_json.is_empty()); + assert!(bm.policy_config_json.starts_with(b"{\"Version\"")); + assert!(!bm.notification_config_xml.is_empty()); + assert!(bm.notification_config_xml.starts_with(b"rule1Enabled30"#; + bm.lifecycle_config_xml = lifecycle_xml.as_bytes().to_vec(); + bm.lifecycle_config_updated_at = OffsetDateTime::now_utc(); + + // Add versioning configuration + let versioning_xml = r#"Enabled"#; + bm.versioning_config_xml = versioning_xml.as_bytes().to_vec(); + bm.versioning_config_updated_at = OffsetDateTime::now_utc(); + + // Add encryption configuration + let encryption_xml = r#"AES256"#; + bm.encryption_config_xml = encryption_xml.as_bytes().to_vec(); + bm.encryption_config_updated_at = OffsetDateTime::now_utc(); + + // Add tagging configuration + let tagging_xml = r#"EnvironmentTestOwnerRustFS"#; + bm.tagging_config_xml = tagging_xml.as_bytes().to_vec(); + bm.tagging_config_updated_at = OffsetDateTime::now_utc(); + + // Add quota configuration + let quota_json = + r#"{"quota":1073741824,"quota_type":"Hard","created_at":"2024-01-01T00:00:00Z","updated_at":"2024-01-01T00:00:00Z"}"#; // 1GB quota + bm.quota_config_json = quota_json.as_bytes().to_vec(); + bm.quota_config_updated_at = OffsetDateTime::now_utc(); + + // Add object lock configuration + let object_lock_xml = r#"EnabledGOVERNANCE7"#; + bm.object_lock_config_xml = object_lock_xml.as_bytes().to_vec(); + bm.object_lock_config_updated_at = OffsetDateTime::now_utc(); + + // Add notification configuration + let notification_xml = r#"notification1s3:ObjectCreated:*test-log-group"#; + bm.notification_config_xml = notification_xml.as_bytes().to_vec(); + bm.notification_config_updated_at = OffsetDateTime::now_utc(); + + // Add replication configuration + let replication_xml = r#"arn:aws:iam::123456789012:role/replication-rolerule1Enableddocuments/arn:aws:s3:::destination-bucket"#; + bm.replication_config_xml = replication_xml.as_bytes().to_vec(); + bm.replication_config_updated_at = OffsetDateTime::now_utc(); + + // Add bucket targets configuration + let bucket_targets_json = r#"[{"endpoint":"http://target1.example.com","credentials":{"accessKey":"key1","secretKey":"secret1"},"targetBucket":"target-bucket-1","region":"us-east-1"},{"endpoint":"http://target2.example.com","credentials":{"accessKey":"key2","secretKey":"secret2"},"targetBucket":"target-bucket-2","region":"us-west-2"}]"#; + bm.bucket_targets_config_json = bucket_targets_json.as_bytes().to_vec(); + bm.bucket_targets_config_updated_at = OffsetDateTime::now_utc(); + + // Add bucket targets meta configuration + let bucket_targets_meta_json = r#"{"replicationId":"repl-123","syncMode":"async","bandwidth":"100MB"}"#; + bm.bucket_targets_config_meta_json = bucket_targets_meta_json.as_bytes().to_vec(); + bm.bucket_targets_config_meta_updated_at = OffsetDateTime::now_utc(); + + // Add public access block configuration + let public_access_block_xml = r#"truetruetruefalse"#; + bm.public_access_block_config_xml = public_access_block_xml.as_bytes().to_vec(); + bm.public_access_block_config_updated_at = OffsetDateTime::now_utc(); + + let bucket_acl = r#"{"owner":{"id":"rustfsadmin","display_name":"RustFS Tester"},"grants":[{"grantee":{"grantee_type":"CanonicalUser","id":"rustfsadmin","display_name":"RustFS Tester","uri":null,"email_address":null},"permission":"FULL_CONTROL"}]}"#; + bm.bucket_acl_config_json = bucket_acl.as_bytes().to_vec(); + bm.bucket_acl_config_updated_at = OffsetDateTime::now_utc(); + + // Test serialization + let buf = bm.marshal_msg().unwrap(); + assert!(!buf.is_empty(), "Serialized buffer should not be empty"); + + // Test deserialization + let deserialized_bm = BucketMetadata::unmarshal(&buf).unwrap(); + + // Verify all fields are correctly serialized and deserialized + assert_eq!(bm.name, deserialized_bm.name); + assert_eq!(bm.created.unix_timestamp(), deserialized_bm.created.unix_timestamp()); + assert_eq!(bm.lock_enabled, deserialized_bm.lock_enabled); + + // Verify configuration data + assert_eq!(bm.policy_config_json, deserialized_bm.policy_config_json); + assert_eq!(bm.lifecycle_config_xml, deserialized_bm.lifecycle_config_xml); + assert_eq!(bm.versioning_config_xml, deserialized_bm.versioning_config_xml); + assert_eq!(bm.encryption_config_xml, deserialized_bm.encryption_config_xml); + assert_eq!(bm.tagging_config_xml, deserialized_bm.tagging_config_xml); + assert_eq!(bm.quota_config_json, deserialized_bm.quota_config_json); + assert_eq!(bm.public_access_block_config_xml, deserialized_bm.public_access_block_config_xml); + assert_eq!(bm.bucket_acl_config_json, deserialized_bm.bucket_acl_config_json); + assert_eq!(bm.object_lock_config_xml, deserialized_bm.object_lock_config_xml); + assert_eq!(bm.notification_config_xml, deserialized_bm.notification_config_xml); + assert_eq!(bm.replication_config_xml, deserialized_bm.replication_config_xml); + assert_eq!(bm.bucket_targets_config_json, deserialized_bm.bucket_targets_config_json); + assert_eq!(bm.bucket_targets_config_meta_json, deserialized_bm.bucket_targets_config_meta_json); + + // Verify timestamps (comparing unix timestamps to avoid precision issues) + assert_eq!( + bm.policy_config_updated_at.unix_timestamp(), + deserialized_bm.policy_config_updated_at.unix_timestamp() + ); + assert_eq!( + bm.lifecycle_config_updated_at.unix_timestamp(), + deserialized_bm.lifecycle_config_updated_at.unix_timestamp() + ); + assert_eq!( + bm.versioning_config_updated_at.unix_timestamp(), + deserialized_bm.versioning_config_updated_at.unix_timestamp() + ); + assert_eq!( + bm.encryption_config_updated_at.unix_timestamp(), + deserialized_bm.encryption_config_updated_at.unix_timestamp() + ); + assert_eq!( + bm.tagging_config_updated_at.unix_timestamp(), + deserialized_bm.tagging_config_updated_at.unix_timestamp() + ); + assert_eq!( + bm.quota_config_updated_at.unix_timestamp(), + deserialized_bm.quota_config_updated_at.unix_timestamp() + ); + assert_eq!( + bm.object_lock_config_updated_at.unix_timestamp(), + deserialized_bm.object_lock_config_updated_at.unix_timestamp() + ); + assert_eq!( + bm.notification_config_updated_at.unix_timestamp(), + deserialized_bm.notification_config_updated_at.unix_timestamp() + ); + assert_eq!( + bm.replication_config_updated_at.unix_timestamp(), + deserialized_bm.replication_config_updated_at.unix_timestamp() + ); + assert_eq!( + bm.bucket_targets_config_updated_at.unix_timestamp(), + deserialized_bm.bucket_targets_config_updated_at.unix_timestamp() + ); + assert_eq!( + bm.bucket_targets_config_meta_updated_at.unix_timestamp(), + deserialized_bm.bucket_targets_config_meta_updated_at.unix_timestamp() + ); + + // Test that the serialized data contains expected content + let buf_str = String::from_utf8_lossy(&buf); + assert!(buf_str.contains("test-bucket"), "Serialized data should contain bucket name"); + + // Verify the buffer size is reasonable (should be larger due to all the config data) + assert!(buf.len() > 1000, "Buffer should be substantial in size due to all configurations"); + + println!("✅ Complete BucketMetadata serialization test passed"); + println!(" - Bucket name: {}", deserialized_bm.name); + println!(" - Lock enabled: {}", deserialized_bm.lock_enabled); + println!(" - Policy config size: {} bytes", deserialized_bm.policy_config_json.len()); + println!(" - Lifecycle config size: {} bytes", deserialized_bm.lifecycle_config_xml.len()); + println!(" - Serialized buffer size: {} bytes", buf.len()); +} diff --git a/crates/ecstore/src/bucket/migration.rs b/crates/ecstore/src/bucket/migration.rs new file mode 100644 index 000000000..bf7809bc8 --- /dev/null +++ b/crates/ecstore/src/bucket/migration.rs @@ -0,0 +1,420 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Migration of bucket metadata and IAM config from legacy format to RustFS format. + +use crate::bucket::metadata::BUCKET_METADATA_FILE; +use crate::bucket::replication::{decode_resync_file, encode_resync_file}; +use crate::disk::{BUCKET_META_PREFIX, MIGRATING_META_BUCKET, RUSTFS_META_BUCKET}; +use crate::store_api::{BucketOptions, ObjectOptions, PutObjReader, StorageAPI}; +use http::HeaderMap; +use rustfs_policy::auth::UserIdentity; +use rustfs_policy::policy::PolicyDoc; +use rustfs_utils::path::SLASH_SEPARATOR; +use serde::{Deserialize, Serialize}; +use std::sync::Arc; +use time::OffsetDateTime; +use tracing::{debug, info, warn}; + +/// IAM config prefix under meta bucket (e.g. config/iam/). +const IAM_CONFIG_PREFIX: &str = "config/iam"; +const IAM_FORMAT_FILE_PATH: &str = "config/iam/format.json"; +const IAM_USERS_PREFIX: &str = "config/iam/users/"; +const IAM_SERVICE_ACCOUNTS_PREFIX: &str = "config/iam/service-accounts/"; +const IAM_STS_PREFIX: &str = "config/iam/sts/"; +const IAM_GROUPS_PREFIX: &str = "config/iam/groups/"; +const IAM_POLICIES_PREFIX: &str = "config/iam/policies/"; +const IAM_POLICY_DB_PREFIX: &str = "config/iam/policydb/"; +const REPLICATION_META_DIR: &str = ".replication"; +const RESYNC_META_FILE: &str = "resync.bin"; + +#[derive(Debug, Serialize, Deserialize)] +struct CompatIamFormat { + #[serde(default)] + version: i64, +} + +#[derive(Debug, Serialize, Deserialize)] +struct CompatGroupInfo { + #[serde(default)] + version: i64, + #[serde(default = "default_group_status")] + status: String, + #[serde(default)] + members: Vec, + #[serde( + rename = "updatedAt", + alias = "update_at", + default, + with = "rustfs_policy::serde_datetime::option" + )] + update_at: Option, +} + +#[derive(Debug, Serialize, Deserialize)] +struct CompatMappedPolicy { + #[serde(default)] + version: i64, + #[serde(rename = "policy", alias = "policies", default)] + policy: String, + #[serde( + rename = "updatedAt", + alias = "update_at", + default, + with = "rustfs_policy::serde_datetime::option" + )] + update_at: Option, +} + +fn default_group_status() -> String { + "enabled".to_string() +} + +fn normalize_iam_config_blob(path: &str, data: &[u8]) -> std::result::Result>, String> { + if path == IAM_FORMAT_FILE_PATH { + let mut format: CompatIamFormat = + serde_json::from_slice(data).map_err(|err| format!("parse IAM format failed: {err}"))?; + if format.version <= 0 { + format.version = 1; + } + return serde_json::to_vec(&format) + .map(Some) + .map_err(|err| format!("serialize IAM format failed: {err}")); + } + + if is_identity_path(path) { + let mut identity: UserIdentity = + serde_json::from_slice(data).map_err(|err| format!("parse IAM identity failed: {err}"))?; + if identity.update_at.is_none() { + identity.update_at = Some(OffsetDateTime::now_utc()); + } + return serde_json::to_vec(&identity) + .map(Some) + .map_err(|err| format!("serialize IAM identity failed: {err}")); + } + + if is_group_path(path) { + let mut group: CompatGroupInfo = serde_json::from_slice(data).map_err(|err| format!("parse IAM group failed: {err}"))?; + if group.update_at.is_none() { + group.update_at = Some(OffsetDateTime::now_utc()); + } + return serde_json::to_vec(&group) + .map(Some) + .map_err(|err| format!("serialize IAM group failed: {err}")); + } + + if is_policy_doc_path(path) { + let mut doc = PolicyDoc::try_from(data.to_vec()).map_err(|err| format!("parse IAM policy doc failed: {err}"))?; + if doc.create_date.is_none() { + doc.create_date = doc.update_date; + } + if doc.update_date.is_none() { + doc.update_date = doc.create_date; + } + return serde_json::to_vec(&doc) + .map(Some) + .map_err(|err| format!("serialize IAM policy doc failed: {err}")); + } + + if is_policy_mapping_path(path) { + let mut mapped: CompatMappedPolicy = + serde_json::from_slice(data).map_err(|err| format!("parse IAM policy mapping failed: {err}"))?; + if mapped.update_at.is_none() { + mapped.update_at = Some(OffsetDateTime::now_utc()); + } + return serde_json::to_vec(&mapped) + .map(Some) + .map_err(|err| format!("serialize IAM policy mapping failed: {err}")); + } + + Ok(None) +} + +fn is_identity_path(path: &str) -> bool { + (path.starts_with(IAM_USERS_PREFIX) || path.starts_with(IAM_SERVICE_ACCOUNTS_PREFIX) || path.starts_with(IAM_STS_PREFIX)) + && path.ends_with("/identity.json") +} + +fn is_group_path(path: &str) -> bool { + path.starts_with(IAM_GROUPS_PREFIX) && path.ends_with("/members.json") +} + +fn is_policy_doc_path(path: &str) -> bool { + path.starts_with(IAM_POLICIES_PREFIX) && path.ends_with("/policy.json") +} + +fn is_policy_mapping_path(path: &str) -> bool { + path.starts_with(IAM_POLICY_DB_PREFIX) && path.ends_with(".json") +} + +fn is_resync_meta_path(path: &str) -> bool { + path.ends_with(&format!("{REPLICATION_META_DIR}/{RESYNC_META_FILE}")) +} + +fn normalize_bucket_meta_blob(path: &str, data: &[u8]) -> std::result::Result>, String> { + if !is_resync_meta_path(path) { + return Ok(None); + } + let status = decode_resync_file(data).map_err(|err| format!("decode resync meta failed: {err}"))?; + encode_resync_file(&status) + .map(Some) + .map_err(|err| format!("encode resync meta failed: {err}")) +} + +/// Migrates bucket metadata from legacy format to RustFS. +/// Uses list_bucket (from disk volumes) to get bucket names, since list_objects_v2 on the legacy +/// meta bucket may not work (legacy format differs from object layer expectations). +/// Skips buckets that already exist in RustFS (idempotent). +pub async fn try_migrate_bucket_metadata(store: Arc) { + let buckets_list = match store + .list_bucket(&BucketOptions { + no_metadata: true, + ..Default::default() + }) + .await + { + Ok(b) => b, + Err(e) => { + warn!("list buckets failed (skip migration): {e}"); + return; + } + }; + + let buckets: Vec = buckets_list.into_iter().map(|b| b.name).collect(); + + if buckets.is_empty() { + debug!("No migrating bucket metadata found"); + return; + } + + debug!("Found {} migrating bucket metadata, migrating...", buckets.len()); + + let opts = ObjectOptions { + max_parity: true, + no_lock: true, + ..Default::default() + }; + let h = HeaderMap::new(); + + for bucket in buckets { + let meta_path = format!("{BUCKET_META_PREFIX}{SLASH_SEPARATOR}{bucket}{SLASH_SEPARATOR}{BUCKET_METADATA_FILE}"); + migrate_one_if_missing(store.clone(), &opts, &h, &meta_path, &format!("bucket metadata: {bucket}")).await; + + let resync_path = format!( + "{BUCKET_META_PREFIX}{SLASH_SEPARATOR}{bucket}{SLASH_SEPARATOR}{REPLICATION_META_DIR}{SLASH_SEPARATOR}{RESYNC_META_FILE}" + ); + migrate_one_if_missing(store.clone(), &opts, &h, &resync_path, &format!("bucket replication resync: {bucket}")).await; + } +} + +async fn migrate_one_if_missing( + store: Arc, + opts: &ObjectOptions, + headers: &HeaderMap, + path: &str, + label: &str, +) { + if store + .get_object_info(RUSTFS_META_BUCKET, path, &ObjectOptions::default()) + .await + .is_ok() + { + debug!("{label} already exists in RustFS, skip"); + return; + } + + let mut rd = match store + .get_object_reader(MIGRATING_META_BUCKET, path, None, headers.clone(), opts) + .await + { + Ok(r) => r, + Err(e) => { + debug!("read migrating {label}: {e}"); + return; + } + }; + + let data = match rd.read_all().await { + Ok(d) if !d.is_empty() => d, + Ok(_) => return, + Err(e) => { + debug!("read migrating {label} body: {e}"); + return; + } + }; + + let data = match normalize_bucket_meta_blob(path, &data) { + Ok(Some(normalized)) => normalized, + Ok(None) => data, + Err(e) => { + warn!("skip {label} migration due to incompatible format: {e}"); + return; + } + }; + + if let Err(e) = store + .put_object(RUSTFS_META_BUCKET, path, &mut PutObjReader::from_vec(data), opts) + .await + { + warn!("write {label}: {e}"); + } else { + info!("Migrated {label}"); + } +} + +/// Migrates IAM config from legacy meta bucket `config/iam/` to RustFS meta bucket. +/// Lists all objects under the IAM prefix in the source, copies each to the target if not present. +/// Skips objects that already exist in RustFS (idempotent). +/// If list_objects_v2 on the legacy bucket fails (e.g. format differs), migration is skipped. +pub async fn try_migrate_iam_config(store: Arc) { + let opts = ObjectOptions { + max_parity: true, + no_lock: true, + ..Default::default() + }; + let h = HeaderMap::new(); + let prefix = format!("{IAM_CONFIG_PREFIX}/"); + let mut continuation: Option = None; + let mut total_migrated = 0usize; + + loop { + let list_result = match store + .clone() + .list_objects_v2(MIGRATING_META_BUCKET, &prefix, continuation, None, 500, false, None, false) + .await + { + Ok(r) => r, + Err(e) => { + debug!("list IAM config from legacy bucket failed (skip migration): {e}"); + return; + } + }; + + for obj in list_result.objects { + let path = &obj.name; + if path.is_empty() || path.ends_with('/') { + continue; + } + if store + .get_object_info(RUSTFS_META_BUCKET, path, &ObjectOptions::default()) + .await + .is_ok() + { + debug!("IAM config already exists in RustFS, skip: {path}"); + continue; + } + let mut rd = match store + .get_object_reader(MIGRATING_META_BUCKET, path, None, h.clone(), &opts) + .await + { + Ok(r) => r, + Err(e) => { + debug!("read migrating IAM config {path}: {e}"); + continue; + } + }; + let data = match rd.read_all().await { + Ok(d) if !d.is_empty() => d, + Ok(_) => continue, + Err(e) => { + debug!("read migrating IAM config {path} body: {e}"); + continue; + } + }; + let data = match normalize_iam_config_blob(path, &data) { + Ok(Some(normalized)) => normalized, + Ok(None) => { + debug!("skip unsupported IAM config path during migration: {path}"); + continue; + } + Err(e) => { + warn!("skip IAM config migration due to incompatible format, path: {path}, err: {e}"); + continue; + } + }; + if let Err(e) = store + .put_object(RUSTFS_META_BUCKET, path, &mut PutObjReader::from_vec(data), &opts) + .await + { + warn!("write IAM config {path}: {e}"); + } else { + info!("Migrated IAM config: {path}"); + total_migrated += 1; + } + } + + continuation = list_result.next_continuation_token.or(list_result.continuation_token); + if !list_result.is_truncated || continuation.is_none() { + break; + } + } + + if total_migrated > 0 { + info!("IAM migration complete: {} object(s) migrated", total_migrated); + } +} + +#[cfg(test)] +mod tests { + use super::{normalize_bucket_meta_blob, normalize_iam_config_blob}; + use crate::bucket::replication::{ + BucketReplicationResyncStatus, ResyncStatusType, TargetReplicationResyncStatus, decode_resync_file, encode_resync_file, + }; + use std::collections::HashMap; + + #[test] + fn test_normalize_policy_mapping_legacy_timestamp_and_fields() { + let path = "config/iam/policydb/users/alice.json"; + let input = r#"{"version":1,"policies":"readwrite","update_at":"2026-03-09 02:22:44.998954 +00:00:00"}"#; + + let output = normalize_iam_config_blob(path, input.as_bytes()) + .expect("normalize should succeed") + .expect("path should be supported"); + + let v: serde_json::Value = serde_json::from_slice(&output).expect("output should be valid JSON"); + assert_eq!(v.get("policy").and_then(|x| x.as_str()), Some("readwrite")); + assert!(v.get("policies").is_none(), "legacy field should be normalized"); + + let updated_at = v + .get("updatedAt") + .and_then(|x| x.as_str()) + .expect("updatedAt should exist as string"); + assert!(updated_at.contains('T'), "updatedAt should be RFC3339-like"); + } + + #[test] + fn test_normalize_bucket_meta_blob_resync_reencode() { + let path = ".buckets/test/.replication/resync.bin"; + let mut status = BucketReplicationResyncStatus::new(); + status.id = 123; + status.targets_map = HashMap::from([( + "arn:replication::1:dest".to_string(), + TargetReplicationResyncStatus { + resync_id: "reset-1".to_string(), + resync_status: ResyncStatusType::ResyncStarted, + replicated_count: 1, + ..Default::default() + }, + )]); + + let input = encode_resync_file(&status).expect("encode should succeed"); + let output = normalize_bucket_meta_blob(path, &input) + .expect("normalize should succeed") + .expect("resync path should be normalized"); + + let decoded = decode_resync_file(&output).expect("decode should succeed"); + assert_eq!(decoded.id, 123); + assert_eq!(decoded.targets_map["arn:replication::1:dest"].resync_id, "reset-1"); + } +} diff --git a/crates/ecstore/src/bucket/mod.rs b/crates/ecstore/src/bucket/mod.rs index ac04822fc..44aa3f5c7 100644 --- a/crates/ecstore/src/bucket/mod.rs +++ b/crates/ecstore/src/bucket/mod.rs @@ -18,6 +18,10 @@ pub mod error; pub mod lifecycle; pub mod metadata; pub mod metadata_sys; +#[cfg(test)] +mod metadata_test; +pub mod migration; +mod msgp_decode; pub mod object_lock; pub mod policy_sys; pub mod quota; diff --git a/crates/ecstore/src/bucket/msgp_decode.rs b/crates/ecstore/src/bucket/msgp_decode.rs new file mode 100644 index 000000000..2bddcd4e9 --- /dev/null +++ b/crates/ecstore/src/bucket/msgp_decode.rs @@ -0,0 +1,189 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! MessagePack decode helpers for bucket metadata, aligned with msgp format. + +use crate::error::{Error, Result}; +use byteorder::{BigEndian, ByteOrder}; +use rmp::Marker; +use std::io::{Read, Write}; +use time::OffsetDateTime; + +/// Skip a single MessagePack value. Used for unknown map keys. +pub(crate) fn skip_msgp_value(rd: &mut R) -> Result<()> { + let marker = rmp::decode::read_marker(rd).map_err(|e| Error::other(format!("{e:?}")))?; + let skip_len: usize = match marker { + Marker::Null | Marker::False | Marker::True => 0, + Marker::FixPos(_) | Marker::FixNeg(_) => 0, + Marker::U8 => 1, + Marker::U16 => 2, + Marker::U32 => 4, + Marker::U64 => 8, + Marker::I8 => 1, + Marker::I16 => 2, + Marker::I32 => 4, + Marker::I64 => 8, + Marker::F32 => 4, + Marker::F64 => 8, + Marker::FixStr(n) => n as usize, + Marker::Str8 => { + let mut b = [0u8; 1]; + rd.read_exact(&mut b).map_err(Error::other)?; + b[0] as usize + } + Marker::Str16 => { + let mut b = [0u8; 2]; + rd.read_exact(&mut b).map_err(Error::other)?; + u16::from_be_bytes(b) as usize + } + Marker::Str32 => { + let mut b = [0u8; 4]; + rd.read_exact(&mut b).map_err(Error::other)?; + u32::from_be_bytes(b) as usize + } + Marker::Bin8 => { + let mut b = [0u8; 1]; + rd.read_exact(&mut b).map_err(Error::other)?; + b[0] as usize + } + Marker::Bin16 => { + let mut b = [0u8; 2]; + rd.read_exact(&mut b).map_err(Error::other)?; + u16::from_be_bytes(b) as usize + } + Marker::Bin32 => { + let mut b = [0u8; 4]; + rd.read_exact(&mut b).map_err(Error::other)?; + u32::from_be_bytes(b) as usize + } + Marker::FixArray(n) => { + for _ in 0..n { + skip_msgp_value(rd)?; + } + return Ok(()); + } + Marker::Array16 => { + let mut b = [0u8; 2]; + rd.read_exact(&mut b).map_err(Error::other)?; + let n = u16::from_be_bytes(b); + for _ in 0..n { + skip_msgp_value(rd)?; + } + return Ok(()); + } + Marker::Array32 => { + let mut b = [0u8; 4]; + rd.read_exact(&mut b).map_err(Error::other)?; + let n = u32::from_be_bytes(b); + for _ in 0..n { + skip_msgp_value(rd)?; + } + return Ok(()); + } + Marker::FixMap(n) => { + for _ in 0..n { + skip_msgp_value(rd)?; + skip_msgp_value(rd)?; + } + return Ok(()); + } + Marker::Map16 => { + let mut b = [0u8; 2]; + rd.read_exact(&mut b).map_err(Error::other)?; + let n = u16::from_be_bytes(b); + for _ in 0..n { + skip_msgp_value(rd)?; + skip_msgp_value(rd)?; + } + return Ok(()); + } + Marker::Map32 => { + let mut b = [0u8; 4]; + rd.read_exact(&mut b).map_err(Error::other)?; + let n = u32::from_be_bytes(b); + for _ in 0..n { + skip_msgp_value(rd)?; + skip_msgp_value(rd)?; + } + return Ok(()); + } + Marker::FixExt1 => 1, + Marker::FixExt2 => 2, + Marker::FixExt4 => 4, + Marker::FixExt8 => 8, + Marker::FixExt16 => 16, + Marker::Ext8 => { + let mut b = [0u8; 1]; + rd.read_exact(&mut b).map_err(Error::other)?; + let len = b[0] as usize; + 1 + len // type byte + data + } + Marker::Ext16 => { + let mut b = [0u8; 2]; + rd.read_exact(&mut b).map_err(Error::other)?; + let len = u16::from_be_bytes(b) as usize; + 2 + len + } + Marker::Ext32 => { + let mut b = [0u8; 4]; + rd.read_exact(&mut b).map_err(Error::other)?; + let len = u32::from_be_bytes(b) as usize; + 4 + len + } + Marker::Reserved => 0, + }; + if skip_len > 0 { + let mut buf = vec![0u8; skip_len]; + rd.read_exact(&mut buf).map_err(Error::other)?; + } + Ok(()) +} + +/// msgp time format: ext8 (0xc7), len 12, type 5, 8 bytes sec (BE) + 4 bytes nsec (BE). +pub(crate) const MSGP_TIME_EXT_TYPE: i8 = 5; +pub(crate) const MSGP_TIME_LEN: u8 = 12; + +/// Read msgp ext8 time - caller must have already read the marker and verified it's ext8. +/// Ext8 format: 1 byte len, 1 byte type, then data bytes. +pub(crate) fn read_msgp_ext8_time(rd: &mut R) -> Result { + let mut len_buf = [0u8; 1]; + rd.read_exact(&mut len_buf).map_err(Error::other)?; + let len = len_buf[0] as usize; + if len != MSGP_TIME_LEN as usize { + return Err(Error::other(format!("invalid msgp time len: {len}"))); + } + let mut type_buf = [0u8; 1]; + rd.read_exact(&mut type_buf).map_err(Error::other)?; + if type_buf[0] != MSGP_TIME_EXT_TYPE as u8 { + return Err(Error::other(format!("invalid msgp time type: {}", type_buf[0]))); + } + let mut buf = [0u8; 12]; + rd.read_exact(&mut buf).map_err(Error::other)?; + let sec = BigEndian::read_i64(&buf[0..8]); + let nsec = BigEndian::read_u32(&buf[8..12]); + OffsetDateTime::from_unix_timestamp(sec) + .map_err(|_| Error::other("invalid timestamp"))? + .replace_nanosecond(nsec) + .map_err(|_| Error::other("invalid nanosecond")) +} + +/// Write msgp time as ext8 (0xc7), len 12, type 5. Always uses ext format (never nil). +pub(crate) fn write_msgp_time(wr: &mut W, t: OffsetDateTime) -> Result<()> { + wr.write_all(&[0xc7, MSGP_TIME_LEN, MSGP_TIME_EXT_TYPE as u8]) + .map_err(Error::other)?; + let mut buf = [0u8; 12]; + BigEndian::write_i64(&mut buf[0..8], t.unix_timestamp()); + BigEndian::write_u32(&mut buf[8..12], t.nanosecond()); + wr.write_all(&buf).map_err(Error::other) +} diff --git a/crates/ecstore/src/bucket/quota/mod.rs b/crates/ecstore/src/bucket/quota/mod.rs index 7152208fd..1593e3d44 100644 --- a/crates/ecstore/src/bucket/quota/mod.rs +++ b/crates/ecstore/src/bucket/quota/mod.rs @@ -15,7 +15,6 @@ pub mod checker; use crate::error::Result; -use rmp_serde::Serializer as rmpSerializer; use rustfs_config::{ QUOTA_API_PATH, QUOTA_EXCEEDED_ERROR_CODE, QUOTA_INTERNAL_ERROR_CODE, QUOTA_INVALID_CONFIG_ERROR_CODE, QUOTA_NOT_FOUND_ERROR_CODE, @@ -28,27 +27,35 @@ use time::OffsetDateTime; pub enum QuotaType { /// Hard quota: reject immediately when exceeded #[default] + #[serde(alias = "HARD", alias = "hard")] Hard, } +/// Bucket quota configuration. quota_type defaults to Hard when omitted. #[derive(Debug, Deserialize, Serialize, Default, Clone, PartialEq)] pub struct BucketQuota { + #[serde(default)] pub quota: Option, + /// Defaults to Hard when missing. + #[serde(default)] pub quota_type: QuotaType, /// Timestamp when this quota configuration was set (for audit purposes) + #[serde(default, with = "time::serde::rfc3339::option")] pub created_at: Option, + /// Accept updated_at for compatibility; not used. + #[serde(default, with = "time::serde::rfc3339::option", skip_serializing_if = "Option::is_none")] + pub updated_at: Option, } impl BucketQuota { + /// Serialize to JSON bytes. Same format as parse_all_configs. pub fn marshal_msg(&self) -> Result> { - let mut buf = Vec::new(); - self.serialize(&mut rmpSerializer::new(&mut buf).with_struct_map())?; - Ok(buf) + serde_json::to_vec(self).map_err(Into::into) } + /// Deserialize from JSON bytes. Same format as parse_all_configs. pub fn unmarshal(buf: &[u8]) -> Result { - let t: BucketQuota = rmp_serde::from_slice(buf)?; - Ok(t) + serde_json::from_slice(buf).map_err(Into::into) } pub fn new(quota: Option) -> Self { @@ -57,6 +64,7 @@ impl BucketQuota { quota, quota_type: QuotaType::Hard, created_at: Some(now), + updated_at: None, } } @@ -156,3 +164,57 @@ impl QuotaErrorResponse { } } } + +#[cfg(test)] +mod tests { + use super::*; + + /// Legacy format: quota, created_at, updated_at (no quota_type) + #[test] + fn deserialize_format_without_quota_type() { + let json = r#"{"quota":1073741824,"created_at":"2024-01-01T00:00:00Z","updated_at":"2024-01-01T00:00:00Z"}"#; + let q: BucketQuota = serde_json::from_slice(json.as_bytes()).expect("should parse"); + assert_eq!(q.quota, Some(1073741824)); + assert_eq!(q.quota_type, QuotaType::Hard); + assert!(q.created_at.is_some()); + assert!(q.updated_at.is_some()); + } + + /// RustFS format: quota, quota_type, created_at + #[test] + fn deserialize_rustfs_format() { + let json = r#"{"quota":1073741824,"quota_type":"Hard","created_at":"2024-01-01T00:00:00Z"}"#; + let q: BucketQuota = serde_json::from_slice(json.as_bytes()).expect("should parse"); + assert_eq!(q.quota, Some(1073741824)); + assert_eq!(q.quota_type, QuotaType::Hard); + assert!(q.created_at.is_some()); + assert!(q.created_at.is_some_and(|t| t.unix_timestamp() == 1704067200)); + } + + /// E2E format uses "HARD" (uppercase) + #[test] + fn deserialize_quota_type_hard_uppercase() { + let json = r#"{"quota":2048,"quota_type":"HARD"}"#; + let q: BucketQuota = serde_json::from_slice(json.as_bytes()).expect("should parse"); + assert_eq!(q.quota_type, QuotaType::Hard); + } + + /// marshal_msg/unmarshal use JSON, same as parse_all_configs + #[test] + fn marshal_unmarshal_roundtrip() { + let q = BucketQuota::new(Some(1073741824)); + let buf = q.marshal_msg().expect("marshal"); + let restored = BucketQuota::unmarshal(&buf).expect("unmarshal"); + assert_eq!(q.quota, restored.quota); + assert_eq!(q.quota_type, restored.quota_type); + } + + /// unmarshal accepts format without quota_type + #[test] + fn unmarshal_format_without_quota_type() { + let json = r#"{"quota":1073741824,"created_at":"2024-01-01T00:00:00Z","updated_at":"2024-01-01T00:00:00Z"}"#; + let q = BucketQuota::unmarshal(json.as_bytes()).expect("should parse"); + assert_eq!(q.quota, Some(1073741824)); + assert_eq!(q.quota_type, QuotaType::Hard); + } +} diff --git a/crates/ecstore/src/bucket/replication/replication_pool.rs b/crates/ecstore/src/bucket/replication/replication_pool.rs index 1398aaa3c..8d7a1734a 100644 --- a/crates/ecstore/src/bucket/replication/replication_pool.rs +++ b/crates/ecstore/src/bucket/replication/replication_pool.rs @@ -20,8 +20,8 @@ use crate::bucket::replication::ResyncStatusType; use crate::bucket::replication::replicate_delete; use crate::bucket::replication::replicate_object; use crate::bucket::replication::replication_resyncer::{ - BucketReplicationResyncStatus, DeletedObjectReplicationInfo, ReplicationConfig, ReplicationResyncer, - get_heal_replicate_object_info, + BucketReplicationResyncStatus, DeletedObjectReplicationInfo, REPLICATION_DIR, RESYNC_FILE_NAME, ReplicationConfig, + ReplicationResyncer, decode_resync_file, get_heal_replicate_object_info, }; use crate::bucket::replication::replication_state::ReplicationStats; use crate::config::com::read_config; @@ -41,7 +41,7 @@ use rustfs_filemeta::VersionPurgeStatusType; use rustfs_filemeta::replication_statuses_map; use rustfs_filemeta::version_purge_statuses_map; use rustfs_filemeta::{REPLICATE_EXISTING, REPLICATE_HEAL, REPLICATE_HEAL_DELETE}; -use rustfs_utils::http::RESERVED_METADATA_PREFIX_LOWER; +use rustfs_utils::http::{SUFFIX_REPLICATION_TIMESTAMP, get_str}; use std::any::Any; use std::sync::Arc; use std::sync::atomic::AtomicI32; @@ -861,17 +861,8 @@ async fn load_bucket_resync_metadata( bucket: &str, obj_api: Arc, ) -> Result { - use std::convert::TryInto; - let mut brs = BucketReplicationResyncStatus::new(); - // Constants that would be defined elsewhere - const REPLICATION_DIR: &str = "replication"; - const RESYNC_FILE_NAME: &str = "resync.bin"; - const RESYNC_META_FORMAT: u16 = 1; - const RESYNC_META_VERSION: u16 = 1; - const RESYNC_META_VERSION_V1: u16 = 1; - let resync_dir_path = format!("{BUCKET_META_PREFIX}/{bucket}/{REPLICATION_DIR}"); let resync_file_path = format!("{resync_dir_path}/{RESYNC_FILE_NAME}"); @@ -886,27 +877,7 @@ async fn load_bucket_resync_metadata( return Ok(brs); } - if data.len() <= 4 { - return Err(EcstoreError::CorruptedFormat); - } - - // Read resync meta header - let format = u16::from_le_bytes(data[0..2].try_into().unwrap()); - if format != RESYNC_META_FORMAT { - return Err(EcstoreError::CorruptedFormat); - } - - let version = u16::from_le_bytes(data[2..4].try_into().unwrap()); - if version != RESYNC_META_VERSION { - return Err(EcstoreError::CorruptedFormat); - } - - // Parse data - brs = BucketReplicationResyncStatus::unmarshal_msg(&data[4..])?; - - if brs.version != RESYNC_META_VERSION_V1 { - return Err(EcstoreError::CorruptedFormat); - } + brs = decode_resync_file(&data)?; Ok(brs) } @@ -984,10 +955,8 @@ pub fn get_global_replication_pool() -> Option> { pub async fn schedule_replication(oi: ObjectInfo, o: Arc, dsc: ReplicateDecision, op_type: ReplicationType) { let tgt_statuses = replication_statuses_map(&oi.replication_status_internal.clone().unwrap_or_default()); let purge_statuses = version_purge_statuses_map(&oi.version_purge_status_internal.clone().unwrap_or_default()); - let tm = oi - .user_defined - .get(&format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, "replication-timestamp")) - .map(|v| OffsetDateTime::parse(v, &Rfc3339).unwrap_or(OffsetDateTime::UNIX_EPOCH)); + let tm = get_str(&oi.user_defined, SUFFIX_REPLICATION_TIMESTAMP) + .map(|v| OffsetDateTime::parse(&v, &Rfc3339).unwrap_or(OffsetDateTime::UNIX_EPOCH)); let mut rstate = oi.replication_state(); rstate.replicate_decision_str = dsc.to_string(); let asz = oi.get_actual_size().unwrap_or_default(); diff --git a/crates/ecstore/src/bucket/replication/replication_resyncer.rs b/crates/ecstore/src/bucket/replication/replication_resyncer.rs index 98b4a2645..d043018a8 100644 --- a/crates/ecstore/src/bucket/replication/replication_resyncer.rs +++ b/crates/ecstore/src/bucket/replication/replication_resyncer.rs @@ -17,6 +17,7 @@ use crate::bucket::bucket_target_sys::{ AdvancedPutOptions, BucketTargetSys, PutObjectOptions, PutObjectPartOptions, RemoveObjectOptions, TargetClient, }; use crate::bucket::metadata_sys; +use crate::bucket::msgp_decode::{read_msgp_ext8_time, skip_msgp_value, write_msgp_time}; use crate::bucket::replication::ResyncStatusType; use crate::bucket::replication::replication_pool::GLOBAL_REPLICATION_STATS; use crate::bucket::replication::{ObjectOpts, ReplicationConfigurationExt as _}; @@ -50,7 +51,7 @@ use http_body::Frame; use http_body_util::StreamBody; use regex::Regex; use rustfs_filemeta::{ - MrfReplicateEntry, REPLICATE_EXISTING, REPLICATE_EXISTING_DELETE, REPLICATION_RESET, ReplicateDecision, ReplicateObjectInfo, + MrfReplicateEntry, REPLICATE_EXISTING, REPLICATE_EXISTING_DELETE, ReplicateDecision, ReplicateObjectInfo, ReplicateTargetDecision, ReplicatedInfos, ReplicatedTargetInfo, ReplicationAction, ReplicationState, ReplicationStatusType, ReplicationType, ReplicationWorkerOperation, ResyncDecision, ResyncTargetDecision, VersionPurgeStatusType, get_replication_state, parse_replicate_decision, replication_statuses_map, target_reset_header, version_purge_statuses_map, @@ -58,8 +59,13 @@ use rustfs_filemeta::{ use rustfs_s3_common::EventName; use rustfs_utils::http::{ AMZ_BUCKET_REPLICATION_STATUS, AMZ_OBJECT_TAGGING, AMZ_TAGGING_DIRECTIVE, CONTENT_ENCODING, HeaderExt as _, - RESERVED_METADATA_PREFIX, RESERVED_METADATA_PREFIX_LOWER, RUSTFS_REPLICATION_ACTUAL_OBJECT_SIZE, - RUSTFS_REPLICATION_RESET_STATUS, SSEC_ALGORITHM_HEADER, SSEC_KEY_HEADER, SSEC_KEY_MD5_HEADER, headers, + SSEC_ALGORITHM_HEADER, SSEC_KEY_HEADER, SSEC_KEY_MD5_HEADER, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, + SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, SUFFIX_REPLICATION_RESET, SUFFIX_REPLICATION_RESET_ARN_PREFIX, + SUFFIX_REPLICATION_STATUS, SUFFIX_TAGGING_TIMESTAMP, headers, +}; +use rustfs_utils::http::{ + SUFFIX_REPLICATION_ACTUAL_OBJECT_SIZE, SUFFIX_REPLICATION_RESET_STATUS, SUFFIX_REPLICATION_SSEC_CRC, get_header_map, get_str, + has_internal_suffix, insert_header_map, insert_str, internal_key_strip_suffix_prefix, is_internal_key, }; use rustfs_utils::path::path_join_buf; use rustfs_utils::string::strings_has_prefix_fold; @@ -69,7 +75,8 @@ use serde::Deserialize; use serde::Serialize; use std::any::Any; use std::collections::HashMap; -use std::sync::Arc; +use std::io::{Cursor, Read}; +use std::sync::{Arc, LazyLock}; use time::OffsetDateTime; use time::format_description::well_known::Rfc3339; use tokio::io::AsyncRead; @@ -80,14 +87,29 @@ use tokio_util::io::ReaderStream; use tokio_util::sync::CancellationToken; use tracing::{error, info, instrument, warn}; -const REPLICATION_DIR: &str = ".replication"; -const RESYNC_FILE_NAME: &str = "resync.bin"; -const RESYNC_META_FORMAT: u16 = 1; -const RESYNC_META_VERSION: u16 = 1; +pub(crate) const REPLICATION_DIR: &str = ".replication"; +pub(crate) const RESYNC_FILE_NAME: &str = "resync.bin"; +pub(crate) const RESYNC_META_FORMAT: u16 = 1; +pub(crate) const RESYNC_META_VERSION: u16 = 1; const RESYNC_TIME_INTERVAL: TokioDuration = TokioDuration::from_secs(60); +const WIRE_ZERO_TIME_UNIX: i64 = -62_135_596_800; + +static WIRE_ZERO_TIME: LazyLock = + LazyLock::new(|| OffsetDateTime::from_unix_timestamp(WIRE_ZERO_TIME_UNIX).unwrap_or(OffsetDateTime::UNIX_EPOCH)); static WARNED_MONITOR_UNINIT: std::sync::Once = std::sync::Once::new(); +fn wire_time_or_default(value: Option) -> OffsetDateTime { + value.unwrap_or(*WIRE_ZERO_TIME) +} + +fn normalize_wire_time(value: Option) -> Option { + match value { + Some(v) if v == *WIRE_ZERO_TIME || v == OffsetDateTime::UNIX_EPOCH => None, + other => other, + } +} + #[derive(Debug, Clone, Default)] pub struct ResyncOpts { pub bucket: String, @@ -139,14 +161,199 @@ impl BucketReplicationResyncStatus { } pub fn marshal_msg(&self) -> Result> { - Ok(rmp_serde::to_vec(&self)?) + let mut wr = Vec::new(); + rmp::encode::write_map_len(&mut wr, 4)?; + rmp::encode::write_str(&mut wr, "v")?; + rmp::encode::write_i32(&mut wr, i32::from(self.version))?; + rmp::encode::write_str(&mut wr, "brs")?; + rmp::encode::write_map_len(&mut wr, self.targets_map.len() as u32)?; + for (arn, status) in &self.targets_map { + rmp::encode::write_str(&mut wr, arn)?; + status.marshal_wire_msg(&mut wr)?; + } + rmp::encode::write_str(&mut wr, "id")?; + rmp::encode::write_i32(&mut wr, self.id)?; + rmp::encode::write_str(&mut wr, "lu")?; + write_msgp_time(&mut wr, wire_time_or_default(self.last_update))?; + Ok(wr) } pub fn unmarshal_msg(data: &[u8]) -> Result { + let mut rd = Cursor::new(data); + let mut out = Self::new(); + let mut fields = rmp::decode::read_map_len(&mut rd)?; + + while fields > 0 { + fields -= 1; + let key = read_msgp_str(&mut rd)?; + match key.as_str() { + "v" => { + let v: i32 = rmp::decode::read_int(&mut rd)?; + out.version = u16::try_from(v).map_err(|_| Error::other("invalid resync version"))?; + } + "brs" => { + let map_len = rmp::decode::read_map_len(&mut rd)?; + let mut targets = HashMap::with_capacity(map_len as usize); + for _ in 0..map_len { + let arn = read_msgp_str(&mut rd)?; + let status = TargetReplicationResyncStatus::unmarshal_wire_msg(&mut rd)?; + targets.insert(arn, status); + } + out.targets_map = targets; + } + "id" => { + out.id = rmp::decode::read_int::(&mut rd)?; + } + "lu" => { + out.last_update = normalize_wire_time(read_msgp_time_or_nil(&mut rd)?); + } + _ => skip_msgp_value(&mut rd)?, + } + } + Ok(out) + } + + pub fn unmarshal_legacy_msg(data: &[u8]) -> Result { Ok(rmp_serde::from_slice(data)?) } } +pub(crate) fn encode_resync_file(status: &BucketReplicationResyncStatus) -> Result> { + let payload = status.marshal_msg()?; + let mut data = Vec::with_capacity(4 + payload.len()); + let mut major = [0u8; 2]; + byteorder::LittleEndian::write_u16(&mut major, RESYNC_META_FORMAT); + data.extend_from_slice(&major); + let mut minor = [0u8; 2]; + byteorder::LittleEndian::write_u16(&mut minor, RESYNC_META_VERSION); + data.extend_from_slice(&minor); + data.extend_from_slice(&payload); + Ok(data) +} + +pub(crate) fn decode_resync_file(data: &[u8]) -> Result { + if data.len() <= 4 { + return Err(Error::CorruptedFormat); + } + + let mut major = [0u8; 2]; + major.copy_from_slice(&data[0..2]); + if byteorder::LittleEndian::read_u16(&major) != RESYNC_META_FORMAT { + return Err(Error::CorruptedFormat); + } + + let mut minor = [0u8; 2]; + minor.copy_from_slice(&data[2..4]); + if byteorder::LittleEndian::read_u16(&minor) != RESYNC_META_VERSION { + return Err(Error::CorruptedFormat); + } + + let status = match BucketReplicationResyncStatus::unmarshal_msg(&data[4..]) { + Ok(v) => v, + Err(_) => BucketReplicationResyncStatus::unmarshal_legacy_msg(&data[4..])?, + }; + if status.version != RESYNC_META_VERSION { + return Err(Error::CorruptedFormat); + } + Ok(status) +} + +impl TargetReplicationResyncStatus { + fn marshal_wire_msg(&self, wr: &mut Vec) -> Result<()> { + rmp::encode::write_map_len(wr, 11)?; + rmp::encode::write_str(wr, "st")?; + write_msgp_time(wr, wire_time_or_default(self.start_time))?; + rmp::encode::write_str(wr, "lst")?; + write_msgp_time(wr, wire_time_or_default(self.last_update))?; + rmp::encode::write_str(wr, "id")?; + rmp::encode::write_str(wr, &self.resync_id)?; + rmp::encode::write_str(wr, "rdt")?; + write_msgp_time(wr, wire_time_or_default(self.resync_before_date))?; + rmp::encode::write_str(wr, "rst")?; + rmp::encode::write_i32(wr, resync_status_to_i32(self.resync_status))?; + rmp::encode::write_str(wr, "fs")?; + rmp::encode::write_i64(wr, self.failed_size)?; + rmp::encode::write_str(wr, "frc")?; + rmp::encode::write_i64(wr, self.failed_count)?; + rmp::encode::write_str(wr, "rs")?; + rmp::encode::write_i64(wr, self.replicated_size)?; + rmp::encode::write_str(wr, "rrc")?; + rmp::encode::write_i64(wr, self.replicated_count)?; + rmp::encode::write_str(wr, "bkt")?; + rmp::encode::write_str(wr, &self.bucket)?; + rmp::encode::write_str(wr, "obj")?; + rmp::encode::write_str(wr, &self.object)?; + Ok(()) + } + + fn unmarshal_wire_msg(rd: &mut R) -> Result { + let mut out = Self::new(); + let mut fields = rmp::decode::read_map_len(rd)?; + + while fields > 0 { + fields -= 1; + let key = read_msgp_str(rd)?; + match key.as_str() { + "st" => out.start_time = normalize_wire_time(read_msgp_time_or_nil(rd)?), + "lst" => out.last_update = normalize_wire_time(read_msgp_time_or_nil(rd)?), + "id" => out.resync_id = read_msgp_str(rd)?, + "rdt" => out.resync_before_date = normalize_wire_time(read_msgp_time_or_nil(rd)?), + "rst" => { + let v: i32 = rmp::decode::read_int(rd)?; + out.resync_status = resync_status_from_i32(v)?; + } + "fs" => out.failed_size = rmp::decode::read_int(rd)?, + "frc" => out.failed_count = rmp::decode::read_int(rd)?, + "rs" => out.replicated_size = rmp::decode::read_int(rd)?, + "rrc" => out.replicated_count = rmp::decode::read_int(rd)?, + "bkt" => out.bucket = read_msgp_str(rd)?, + "obj" => out.object = read_msgp_str(rd)?, + _ => skip_msgp_value(rd)?, + } + } + Ok(out) + } +} + +fn read_msgp_str(rd: &mut R) -> Result { + let len = rmp::decode::read_str_len(rd)? as usize; + let mut buf = vec![0u8; len]; + rd.read_exact(&mut buf)?; + Ok(String::from_utf8(buf)?) +} + +fn read_msgp_time_or_nil(rd: &mut R) -> Result> { + let marker = rmp::decode::read_marker(rd).map_err(|e| Error::other(format!("{e:?}")))?; + match marker { + rmp::Marker::Null => Ok(None), + rmp::Marker::Ext8 => Ok(Some(read_msgp_ext8_time(rd)?)), + other => Err(Error::other(format!("expected time ext or nil, got marker: {other:?}"))), + } +} + +fn resync_status_to_i32(status: ResyncStatusType) -> i32 { + match status { + ResyncStatusType::NoResync => 0, + ResyncStatusType::ResyncPending => 1, + ResyncStatusType::ResyncCanceled => 2, + ResyncStatusType::ResyncStarted => 3, + ResyncStatusType::ResyncCompleted => 4, + ResyncStatusType::ResyncFailed => 5, + } +} + +fn resync_status_from_i32(code: i32) -> Result { + match code { + 0 => Ok(ResyncStatusType::NoResync), + 1 => Ok(ResyncStatusType::ResyncPending), + 2 => Ok(ResyncStatusType::ResyncCanceled), + 3 => Ok(ResyncStatusType::ResyncStarted), + 4 => Ok(ResyncStatusType::ResyncCompleted), + 5 => Ok(ResyncStatusType::ResyncFailed), + _ => Err(Error::other(format!("invalid resync status code: {code}"))), + } +} + static RESYNC_WORKER_COUNT: usize = 10; #[derive(Debug)] @@ -617,7 +824,7 @@ pub async fn get_heal_replicate_object_info(oi: &ObjectInfo, rcfg: &ReplicationC let keys_to_update: Vec<_> = user_defined .iter() - .filter(|(k, _)| k.eq_ignore_ascii_case(format!("{RESERVED_METADATA_PREFIX_LOWER}{REPLICATION_RESET}").as_str())) + .filter(|(k, _)| has_internal_suffix(k, SUFFIX_REPLICATION_RESET)) .map(|(k, v)| (k.clone(), v.clone())) .collect(); @@ -695,19 +902,7 @@ pub async fn get_heal_replicate_object_info(oi: &ObjectInfo, rcfg: &ReplicationC } async fn save_resync_status(bucket: &str, status: &BucketReplicationResyncStatus, api: Arc) -> Result<()> { - let buf = status.marshal_msg()?; - - let mut data = Vec::new(); - - let mut major = [0u8; 2]; - byteorder::LittleEndian::write_u16(&mut major, RESYNC_META_FORMAT); - data.extend_from_slice(&major); - - let mut minor = [0u8; 2]; - byteorder::LittleEndian::write_u16(&mut minor, RESYNC_META_VERSION); - data.extend_from_slice(&minor); - - data.extend_from_slice(&buf); + let data = encode_resync_file(status)?; let config_file = path_join_buf(&[BUCKET_META_PREFIX, bucket, REPLICATION_DIR, RESYNC_FILE_NAME]); save_config(api, &config_file, data).await?; @@ -900,8 +1095,8 @@ pub fn resync_target( let rs = oi .user_defined .get(target_reset_header(arn).as_str()) - .or(oi.user_defined.get(RUSTFS_REPLICATION_RESET_STATUS)) - .map(|s| s.to_string()); + .cloned() + .or_else(|| get_header_map(&oi.user_defined, SUFFIX_REPLICATION_RESET_STATUS)); let mut dec = ResyncTargetDecision::default(); @@ -1132,15 +1327,7 @@ impl ObjectInfoExt for ObjectInfo { .user_defined .iter() .filter_map(|(k, v)| { - if k.starts_with(&format!("{RESERVED_METADATA_PREFIX_LOWER}-{REPLICATION_RESET}")) { - Some(( - k.trim_start_matches(&format!("{RESERVED_METADATA_PREFIX_LOWER}-{REPLICATION_RESET}")) - .to_string(), - v.clone(), - )) - } else { - None - } + internal_key_strip_suffix_prefix(k, SUFFIX_REPLICATION_RESET_ARN_PREFIX).map(|arn| (arn, v.clone())) }) .collect(), ..Default::default() @@ -1893,7 +2080,7 @@ pub async fn replicate_object(roi: ReplicateObjectInfo, storage: if roi.replication_status_internal != new_replication_internal || rinfos.replication_resynced() { let mut eval_metadata = HashMap::new(); if let Some(ref s) = new_replication_internal { - eval_metadata.insert(format!("{RESERVED_METADATA_PREFIX_LOWER}replication-status"), s.clone()); + insert_str(&mut eval_metadata, SUFFIX_REPLICATION_STATUS, s.clone()); } let popts = ObjectOptions { version_id: roi.version_id.map(|v| v.to_string()), @@ -2549,8 +2736,6 @@ static VALID_SSE_REPLICATION_HEADERS: &[(&str, &str)] = &[ ("X-Rustfs-Internal-Actual-Object-Size", "X-Rustfs-Replication-Actual-Object-Size"), ]; -const REPLICATION_SSEC_CHECKSUM_HEADER: &str = "X-Rustfs-Replication-Ssec-Crc"; - fn is_valid_sse_header(k: &str) -> Option<&str> { VALID_SSE_REPLICATION_HEADERS .iter() @@ -2574,7 +2759,7 @@ fn put_replication_opts(sc: &str, object_info: &ObjectInfo) -> Result<(PutObject // In case of SSE-C objects copy the allowed internal headers as well if !is_ssec || !has_valid_sse_header { - if strings_has_prefix_fold(k, RESERVED_METADATA_PREFIX) { + if is_internal_key(k) { continue; } if is_standard_header(k) { @@ -2598,7 +2783,7 @@ fn put_replication_opts(sc: &str, object_info: &ObjectInfo) -> Result<(PutObject // Add encrypted CRC to metadata for SSE-C objects if is_ssec { let encoded = BASE64_STANDARD.encode(checksum_data); - meta.insert(REPLICATION_SSEC_CHECKSUM_HEADER.to_string(), encoded); + insert_header_map(&mut meta, SUFFIX_REPLICATION_SSEC_CRC, encoded); } else { // Get checksum metadata for non-SSE-C objects let (cs_meta, is_mp) = object_info.decrypt_checksums(0, &HeaderMap::new())?; @@ -2658,11 +2843,8 @@ fn put_replication_opts(sc: &str, object_info: &ObjectInfo) -> Result<(PutObject if !tags.is_empty() { put_op.user_tags = tags; // set tag timestamp in opts - put_op.internal.tagging_timestamp = if let Some(ts) = object_info - .user_defined - .get(&format!("{RESERVED_METADATA_PREFIX_LOWER}tagging-timestamp")) - { - OffsetDateTime::parse(ts, &Rfc3339) + put_op.internal.tagging_timestamp = if let Some(ts) = get_str(&object_info.user_defined, SUFFIX_TAGGING_TIMESTAMP) { + OffsetDateTime::parse(&ts, &Rfc3339) .map_err(|e| Error::other(format!("Failed to parse tagging timestamp: {}", e)))? } else { object_info.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH) @@ -2694,28 +2876,24 @@ fn put_replication_opts(sc: &str, object_info: &ObjectInfo) -> Result<(PutObject put_op.retain_until_date = OffsetDateTime::parse(v, &Rfc3339).map_err(|e| Error::other(format!("Failed to parse retain until date: {}", e)))?; // set retention timestamp in opts - put_op.internal.retention_timestamp = if let Some(v) = object_info - .user_defined - .get(&format!("{RESERVED_METADATA_PREFIX_LOWER}objectlock-retention-timestamp")) - { - OffsetDateTime::parse(v, &Rfc3339).unwrap_or(OffsetDateTime::UNIX_EPOCH) - } else { - object_info.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH) - }; + put_op.internal.retention_timestamp = + if let Some(v) = get_str(&object_info.user_defined, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP) { + OffsetDateTime::parse(&v, &Rfc3339).unwrap_or(OffsetDateTime::UNIX_EPOCH) + } else { + object_info.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH) + }; } if let Some(v) = lk_map.lookup(AMZ_OBJECT_LOCK_LEGAL_HOLD) { let hold = v.to_uppercase(); put_op.legalhold = Some(ObjectLockLegalHoldStatus::from(hold.as_str())); // set legalhold timestamp in opts - put_op.internal.legalhold_timestamp = if let Some(v) = object_info - .user_defined - .get(&format!("{RESERVED_METADATA_PREFIX_LOWER}objectlock-legalhold-timestamp")) - { - OffsetDateTime::parse(v, &Rfc3339).unwrap_or(OffsetDateTime::UNIX_EPOCH) - } else { - object_info.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH) - }; + put_op.internal.legalhold_timestamp = + if let Some(v) = get_str(&object_info.user_defined, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP) { + OffsetDateTime::parse(&v, &Rfc3339).unwrap_or(OffsetDateTime::UNIX_EPOCH) + } else { + object_info.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH) + }; } // Handle SSE-S3 encryption @@ -2736,7 +2914,7 @@ fn put_replication_opts(sc: &str, object_info: &ObjectInfo) -> Result<(PutObject // If KMS key ID replication is enabled (as by default) // we include the object's KMS key ID. In any case, we // always set the SSE-KMS header. If no KMS key ID is - // specified, MinIO is supposed to use whatever default + // specified, the server uses the default applicable // config applies on the site or bucket. // TODO: Implement SSE-KMS support with key ID replication // let key_id = if kms::replicate_key_id() { @@ -2859,13 +3037,10 @@ async fn replicate_object_with_multipart(ctx: MultipartReplicatio let mut user_metadata = HashMap::new(); - user_metadata.insert( - RUSTFS_REPLICATION_ACTUAL_OBJECT_SIZE.to_string(), - object_info - .user_defined - .get(&format!("{RESERVED_METADATA_PREFIX}actual-size")) - .map(|v| v.to_string()) - .unwrap_or_default(), + insert_header_map( + &mut user_metadata, + SUFFIX_REPLICATION_ACTUAL_OBJECT_SIZE, + rustfs_utils::http::get_str(&object_info.user_defined, rustfs_utils::http::SUFFIX_ACTUAL_SIZE).unwrap_or_default(), ); cli.complete_multipart_upload( @@ -2994,6 +3169,9 @@ fn get_replication_action(oi1: &ObjectInfo, oi2: &HeadObjectOutput, op_type: Rep #[cfg(test)] mod tests { use super::*; + use crate::bucket::msgp_decode::write_msgp_time; + use std::collections::HashMap; + use time::OffsetDateTime; use uuid::Uuid; #[test] @@ -3010,6 +3188,176 @@ mod tests { assert!(part_range_spec_from_actual_size(0, -1).is_err()); } + #[test] + fn test_unmarshal_resync_payload() { + let start = OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("valid ts"); + let last = OffsetDateTime::from_unix_timestamp(1_700_000_123).expect("valid ts"); + let before = OffsetDateTime::from_unix_timestamp(1_699_000_000).expect("valid ts"); + let bucket_last = OffsetDateTime::from_unix_timestamp(1_700_111_111).expect("valid ts"); + + let mut payload = Vec::new(); + rmp::encode::write_map_len(&mut payload, 4).expect("write map"); + rmp::encode::write_str(&mut payload, "v").expect("write key"); + rmp::encode::write_i32(&mut payload, 1).expect("write version"); + rmp::encode::write_str(&mut payload, "brs").expect("write key"); + rmp::encode::write_map_len(&mut payload, 1).expect("write target map"); + rmp::encode::write_str(&mut payload, "arn:replication::1:dest").expect("write arn"); + rmp::encode::write_map_len(&mut payload, 11).expect("write target"); + rmp::encode::write_str(&mut payload, "st").expect("write key"); + write_msgp_time(&mut payload, start).expect("write time"); + rmp::encode::write_str(&mut payload, "lst").expect("write key"); + write_msgp_time(&mut payload, last).expect("write time"); + rmp::encode::write_str(&mut payload, "id").expect("write key"); + rmp::encode::write_str(&mut payload, "resync-1").expect("write id"); + rmp::encode::write_str(&mut payload, "rdt").expect("write key"); + write_msgp_time(&mut payload, before).expect("write time"); + rmp::encode::write_str(&mut payload, "rst").expect("write key"); + rmp::encode::write_i32(&mut payload, 3).expect("write status"); + rmp::encode::write_str(&mut payload, "fs").expect("write key"); + rmp::encode::write_i64(&mut payload, 11).expect("write fs"); + rmp::encode::write_str(&mut payload, "frc").expect("write key"); + rmp::encode::write_i64(&mut payload, 2).expect("write frc"); + rmp::encode::write_str(&mut payload, "rs").expect("write key"); + rmp::encode::write_i64(&mut payload, 101).expect("write rs"); + rmp::encode::write_str(&mut payload, "rrc").expect("write key"); + rmp::encode::write_i64(&mut payload, 9).expect("write rrc"); + rmp::encode::write_str(&mut payload, "bkt").expect("write key"); + rmp::encode::write_str(&mut payload, "bucket-a").expect("write bucket"); + rmp::encode::write_str(&mut payload, "obj").expect("write key"); + rmp::encode::write_str(&mut payload, "object-a").expect("write obj"); + rmp::encode::write_str(&mut payload, "id").expect("write key"); + rmp::encode::write_i32(&mut payload, 42).expect("write id"); + rmp::encode::write_str(&mut payload, "lu").expect("write key"); + write_msgp_time(&mut payload, bucket_last).expect("write lu"); + + let got = BucketReplicationResyncStatus::unmarshal_msg(&payload).expect("decode"); + assert_eq!(got.version, 1); + assert_eq!(got.id, 42); + assert_eq!(got.last_update, Some(bucket_last)); + let tgt = got.targets_map.get("arn:replication::1:dest").expect("target exists"); + assert_eq!(tgt.resync_id, "resync-1"); + assert_eq!(tgt.resync_status, ResyncStatusType::ResyncStarted); + assert_eq!(tgt.bucket, "bucket-a"); + assert_eq!(tgt.object, "object-a"); + assert_eq!(tgt.start_time, Some(start)); + assert_eq!(tgt.last_update, Some(last)); + assert_eq!(tgt.resync_before_date, Some(before)); + } + + #[test] + fn test_unmarshal_legacy_resync_payload() { + let mut status = BucketReplicationResyncStatus::new(); + status.id = 7; + status.version = 1; + status.last_update = Some(OffsetDateTime::from_unix_timestamp(1_700_222_222).expect("valid ts")); + status.targets_map = HashMap::from([( + "legacy-arn".to_string(), + TargetReplicationResyncStatus { + resync_id: "legacy-1".to_string(), + resync_status: ResyncStatusType::ResyncCompleted, + ..Default::default() + }, + )]); + + let old_payload = rmp_serde::to_vec(&status).expect("legacy encode"); + let got = BucketReplicationResyncStatus::unmarshal_legacy_msg(&old_payload).expect("legacy decode"); + assert_eq!(got.id, 7); + assert_eq!(got.version, 1); + assert_eq!(got.targets_map["legacy-arn"].resync_id, "legacy-1"); + assert_eq!(got.targets_map["legacy-arn"].resync_status, ResyncStatusType::ResyncCompleted); + } + + #[test] + fn test_resync_file_roundtrip_wire_format() { + let mut status = BucketReplicationResyncStatus::new(); + status.id = 19; + status.last_update = Some(OffsetDateTime::from_unix_timestamp(1_700_333_333).expect("valid ts")); + status.targets_map = HashMap::from([( + "arn:replication::1:dest".to_string(), + TargetReplicationResyncStatus { + resync_id: "wire-1".to_string(), + resync_status: ResyncStatusType::ResyncStarted, + replicated_count: 5, + ..Default::default() + }, + )]); + + let bytes = encode_resync_file(&status).expect("encode file"); + assert_eq!(&bytes[0..2], &RESYNC_META_FORMAT.to_le_bytes()); + assert_eq!(&bytes[2..4], &RESYNC_META_VERSION.to_le_bytes()); + + let got = decode_resync_file(&bytes).expect("decode file"); + assert_eq!(got.version, RESYNC_META_VERSION); + assert_eq!(got.id, 19); + assert_eq!(got.targets_map["arn:replication::1:dest"].resync_id, "wire-1"); + assert_eq!(got.targets_map["arn:replication::1:dest"].replicated_count, 5); + } + + #[test] + fn test_resync_file_decodes_legacy_payload() { + let mut status = BucketReplicationResyncStatus::new(); + status.id = 7; + status.version = RESYNC_META_VERSION; + status.targets_map = HashMap::from([( + "legacy-arn".to_string(), + TargetReplicationResyncStatus { + resync_id: "legacy-v1".to_string(), + resync_status: ResyncStatusType::ResyncCompleted, + ..Default::default() + }, + )]); + + let legacy_payload = rmp_serde::to_vec(&status).expect("legacy encode"); + let mut file_bytes = Vec::new(); + file_bytes.extend_from_slice(&RESYNC_META_FORMAT.to_le_bytes()); + file_bytes.extend_from_slice(&RESYNC_META_VERSION.to_le_bytes()); + file_bytes.extend_from_slice(&legacy_payload); + + let got = decode_resync_file(&file_bytes).expect("decode legacy"); + assert_eq!(got.id, 7); + assert_eq!(got.targets_map["legacy-arn"].resync_id, "legacy-v1"); + assert_eq!(got.targets_map["legacy-arn"].resync_status, ResyncStatusType::ResyncCompleted); + } + + #[test] + fn test_resync_none_time_encodes_as_wire_zero_and_decodes_to_none() { + let wire_zero = OffsetDateTime::from_unix_timestamp(WIRE_ZERO_TIME_UNIX).expect("valid wire zero timestamp"); + + let mut with_none = BucketReplicationResyncStatus::new(); + with_none.id = 77; + with_none.targets_map = HashMap::from([( + "arn:replication::1:dest".to_string(), + TargetReplicationResyncStatus { + resync_id: "wire-none".to_string(), + resync_status: ResyncStatusType::ResyncStarted, + replicated_count: 1, + ..Default::default() + }, + )]); + + let mut with_zero = with_none.clone(); + with_zero.last_update = Some(wire_zero); + if let Some(target) = with_zero.targets_map.get_mut("arn:replication::1:dest") { + target.start_time = Some(wire_zero); + target.last_update = Some(wire_zero); + target.resync_before_date = Some(wire_zero); + } + + let encoded_none = encode_resync_file(&with_none).expect("encode with none"); + let encoded_zero = encode_resync_file(&with_zero).expect("encode with zero"); + assert_eq!(encoded_none, encoded_zero); + + let decoded = decode_resync_file(&encoded_none).expect("decode"); + let target = decoded + .targets_map + .get("arn:replication::1:dest") + .expect("target should exist"); + assert_eq!(decoded.last_update, None); + assert_eq!(target.start_time, None); + assert_eq!(target.last_update, None); + assert_eq!(target.resync_before_date, None); + } + #[test] fn test_heal_should_use_check_replicate_delete_failed_non_delete_marker() { let oi = ObjectInfo { diff --git a/crates/ecstore/src/bucket/utils.rs b/crates/ecstore/src/bucket/utils.rs index 3ea13301f..c165a12e6 100644 --- a/crates/ecstore/src/bucket/utils.rs +++ b/crates/ecstore/src/bucket/utils.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::disk::RUSTFS_META_BUCKET; +use crate::disk::{MIGRATING_META_BUCKET, RUSTFS_META_BUCKET}; use crate::error::{Error, Result, StorageError}; use regex::Regex; use rustfs_utils::path::SLASH_SEPARATOR; @@ -20,7 +20,7 @@ use s3s::xml; use tracing::instrument; pub fn is_meta_bucketname(name: &str) -> bool { - name.starts_with(RUSTFS_META_BUCKET) + name.starts_with(RUSTFS_META_BUCKET) || name.starts_with(MIGRATING_META_BUCKET) } lazy_static::lazy_static! { diff --git a/crates/ecstore/src/config/com.rs b/crates/ecstore/src/config/com.rs index 7552582c4..fb8b17b68 100644 --- a/crates/ecstore/src/config/com.rs +++ b/crates/ecstore/src/config/com.rs @@ -13,16 +13,17 @@ // limitations under the License. use crate::config::{Config, GLOBAL_STORAGE_CLASS, storageclass}; -use crate::disk::RUSTFS_META_BUCKET; +use crate::disk::{MIGRATING_META_BUCKET, RUSTFS_META_BUCKET}; use crate::error::{Error, Result}; use crate::store_api::{ObjectInfo, ObjectOptions, PutObjReader, StorageAPI}; use http::HeaderMap; -use rustfs_config::DEFAULT_DELIMITER; +use rustfs_config::{DEFAULT_DELIMITER, RUSTFS_REGION}; use rustfs_utils::path::SLASH_SEPARATOR; -use std::collections::HashSet; +use serde_json::{Map, Value}; +use std::collections::{HashMap, HashSet}; use std::sync::Arc; use std::sync::LazyLock; -use tracing::{error, instrument, warn}; +use tracing::{debug, error, info, instrument, warn}; pub const CONFIG_PREFIX: &str = "config"; const CONFIG_FILE: &str = "config.json"; @@ -136,6 +137,226 @@ fn get_config_file() -> String { format!("{CONFIG_PREFIX}{SLASH_SEPARATOR}{CONFIG_FILE}") } +fn storage_class_kvs_mut(cfg: &mut Config) -> &mut crate::config::KVS { + let sub_cfg = cfg.0.entry(STORAGE_CLASS_SUB_SYS.to_string()).or_insert_with(|| { + let mut section = HashMap::new(); + section.insert(DEFAULT_DELIMITER.to_string(), storageclass::DEFAULT_KVS.clone()); + section + }); + sub_cfg + .entry(DEFAULT_DELIMITER.to_string()) + .or_insert_with(|| storageclass::DEFAULT_KVS.clone()) +} + +fn parse_storage_class_value(value: &Value) -> Option { + match value { + Value::String(v) => Some(v.trim().to_string()), + Value::Object(m) => m + .get("parity") + .and_then(Value::as_u64) + .map(|parity| if parity == 0 { String::new() } else { format!("EC:{parity}") }), + _ => None, + } +} + +fn parse_inline_block_value(value: &Value) -> Option { + match value { + Value::String(v) if !v.trim().is_empty() => Some(v.trim().to_string()), + Value::Number(v) => Some(v.to_string()), + _ => None, + } +} + +fn apply_external_storage_class_map(cfg: &mut Config, root: &Map) -> bool { + let sc = root.get("storageclass").or_else(|| root.get("storage_class")); + let Some(Value::Object(sc_obj)) = sc else { + return false; + }; + + let mut applied = false; + let kvs = storage_class_kvs_mut(cfg); + + if let Some(v) = sc_obj.get("standard").and_then(parse_storage_class_value) { + kvs.insert(storageclass::CLASS_STANDARD.to_string(), v); + applied = true; + } + if let Some(v) = sc_obj.get("rrs").and_then(parse_storage_class_value) { + kvs.insert(storageclass::CLASS_RRS.to_string(), v); + applied = true; + } + if let Some(Value::String(v)) = sc_obj.get("optimize") + && !v.trim().is_empty() + { + kvs.insert(storageclass::OPTIMIZE.to_string(), v.clone()); + applied = true; + } + if let Some(v) = sc_obj.get("inline_block").and_then(parse_inline_block_value) { + kvs.insert(storageclass::INLINE_BLOCK.to_string(), v); + applied = true; + } + + applied +} + +fn decode_server_config_blob(data: &[u8]) -> Result { + if let Ok(cfg) = Config::unmarshal(data) { + return Ok(cfg); + } + + let value: Value = serde_json::from_slice(data)?; + let Value::Object(root) = value else { + return Err(Error::other("unrecognized external server config shape")); + }; + + let mut cfg = Config::new(); + let has_storage = apply_external_storage_class_map(&mut cfg, &root); + let has_header = root.contains_key("version") || root.contains_key("region") || root.contains_key("credential"); + if !has_storage && !has_header { + return Err(Error::other("unrecognized external server config shape")); + } + Ok(cfg) +} + +fn parse_object_seed(data: &[u8]) -> Option> { + let value: Value = serde_json::from_slice(data).ok()?; + value.as_object().cloned() +} + +fn build_storageclass_object(cfg: &Config) -> Map { + let kvs = cfg.get_value(STORAGE_CLASS_SUB_SYS, DEFAULT_DELIMITER).unwrap_or_default(); + let mut sc_obj = Map::new(); + sc_obj.insert( + "standard".to_string(), + Value::String(kvs.lookup(storageclass::CLASS_STANDARD).unwrap_or_default()), + ); + sc_obj.insert("rrs".to_string(), Value::String(kvs.lookup(storageclass::CLASS_RRS).unwrap_or_default())); + let optimize = kvs + .lookup(storageclass::OPTIMIZE) + .filter(|v| !v.trim().is_empty()) + .unwrap_or_else(|| "availability".to_string()); + sc_obj.insert("optimize".to_string(), Value::String(optimize)); + if let Some(v) = kvs.lookup(storageclass::INLINE_BLOCK).filter(|v| !v.trim().is_empty()) { + sc_obj.insert("inline_block".to_string(), Value::String(v)); + } + sc_obj +} + +fn encode_server_config_blob(cfg: &Config, seed: Option<&[u8]>) -> Result> { + let mut root = seed.and_then(parse_object_seed).unwrap_or_default(); + + if !matches!(root.get("version"), Some(Value::String(v)) if !v.trim().is_empty()) { + root.insert("version".to_string(), Value::String("33".to_string())); + } + if !matches!(root.get("region"), Some(Value::String(v)) if !v.trim().is_empty()) { + root.insert("region".to_string(), Value::String(RUSTFS_REGION.to_string())); + } + + let mut sc_obj = match root.remove("storageclass") { + Some(Value::Object(v)) => v, + _ => Map::new(), + }; + for (k, v) in build_storageclass_object(cfg) { + sc_obj.insert(k, v); + } + root.insert("storageclass".to_string(), Value::Object(sc_obj)); + root.remove("storage_class"); + + Ok(serde_json::to_vec(&Value::Object(root))?) +} + +fn is_standard_object_server_config(data: &[u8]) -> bool { + let Ok(value) = serde_json::from_slice::(data) else { + return false; + }; + let Value::Object(root) = value else { + return false; + }; + matches!(root.get("version"), Some(Value::String(v)) if !v.trim().is_empty()) + && matches!(root.get("storageclass"), Some(Value::Object(_))) + && !root.contains_key("storage_class") +} + +fn configs_semantically_equal(lhs: &Config, rhs: &Config) -> bool { + build_storageclass_object(lhs) == build_storageclass_object(rhs) +} + +fn is_object_not_found(err: &Error) -> bool { + *err == Error::FileNotFound || matches!(err, Error::ObjectNotFound(_, _) | Error::BucketNotFound(_)) +} + +pub async fn try_migrate_server_config(api: Arc) { + let config_file = get_config_file(); + match api + .get_object_info(RUSTFS_META_BUCKET, &config_file, &ObjectOptions::default()) + .await + { + Ok(_) => { + debug!("server config already exists in RustFS metadata bucket, skip migration"); + return; + } + Err(err) if is_object_not_found(&err) => {} + Err(err) => { + warn!("check target server config failed, skip migration: {:?}", err); + return; + } + } + + let opts = ObjectOptions { + max_parity: true, + no_lock: true, + ..Default::default() + }; + + let mut rd = match api + .get_object_reader(MIGRATING_META_BUCKET, &config_file, None, HeaderMap::new(), &opts) + .await + { + Ok(v) => v, + Err(err) => { + if !is_object_not_found(&err) { + warn!("read legacy server config failed: {:?}", err); + } + return; + } + }; + + let data = match rd.read_all().await { + Ok(v) if !v.is_empty() => v, + Ok(_) => { + debug!("legacy server config is empty, skip migration"); + return; + } + Err(err) => { + warn!("read legacy server config body failed: {:?}", err); + return; + } + }; + + let cfg = match decode_server_config_blob(&data) { + Ok(v) => v, + Err(err) => { + warn!("legacy server config format is incompatible, skip migration: {:?}", err); + return; + } + }; + let normalized = match encode_server_config_blob(&cfg, Some(&data)) { + Ok(v) => v, + Err(err) => { + warn!("serialize migrated server config failed, skip migration: {:?}", err); + return; + } + }; + + match save_config(api, &config_file, normalized).await { + Ok(()) => { + info!("Migrated compatible server config from legacy metadata bucket"); + } + Err(err) => { + warn!("write migrated server config failed: {:?}", err); + } + } +} + /// Handle the situation where the configuration file does not exist, create and save a new configuration async fn handle_missing_config(api: Arc, context: &str) -> Result { warn!("Configuration not found ({}): Start initializing new configuration", context); @@ -171,7 +392,7 @@ async fn read_server_config(api: Arc, data: &[u8]) -> Result { // TODO: decrypt - let cfg = Config::unmarshal(&cfg_data)?; + let cfg = decode_server_config_blob(&cfg_data)?; return Ok(cfg.merge()); } Err(Error::ConfigNotFound) => return handle_missing_config(api, "Read alternate configuration").await, @@ -180,14 +401,35 @@ async fn read_server_config(api: Arc, data: &[u8]) -> Result(api: Arc, cfg: &Config) -> Result<()> { - let data = cfg.marshal()?; - let config_file = get_config_file(); + let existing = match read_config(api.clone(), &config_file).await { + Ok(v) => Some(v), + Err(Error::ConfigNotFound) => None, + Err(err) => { + warn!("read existing server config before save failed, continue with clean output: {:?}", err); + None + } + }; + + if let Some(current) = existing.as_deref() + && is_standard_object_server_config(current) + && let Ok(decoded_current) = decode_server_config_blob(current) + && configs_semantically_equal(&decoded_current, cfg) + { + debug!("server config unchanged and already in standard object shape, skip write"); + return Ok(()); + } + + let data = encode_server_config_blob(cfg, existing.as_deref())?; + if existing.as_deref().is_some_and(|current| current == data.as_slice()) { + debug!("server config bytes unchanged after encode, skip write"); + return Ok(()); + } save_config(api, &config_file, data).await } @@ -232,3 +474,84 @@ async fn apply_dynamic_config_for_sub_sys(cfg: &mut Config, api: Ok(()) } + +#[cfg(test)] +mod tests { + use super::{ + configs_semantically_equal, decode_server_config_blob, encode_server_config_blob, is_standard_object_server_config, + storage_class_kvs_mut, + }; + use crate::config::Config; + use serde_json::Value; + + #[test] + fn test_decode_server_config_accepts_legacy_hidden_if_empty_alias() { + let input = r#"{"storage_class":{"_":[{"key":"standard","value":"EC:2","hiddenIfEmpty":true}]}}"#; + let cfg = decode_server_config_blob(input.as_bytes()).expect("decode should succeed"); + let kvs = cfg.get_value("storage_class", "_").expect("storage_class should exist"); + assert!(kvs.0[0].hidden_if_empty); + } + + #[test] + fn test_decode_server_config_accepts_missing_hidden_if_empty() { + let input = r#"{"storage_class":{"_":[{"key":"standard","value":"EC:2"}]}}"#; + let cfg = decode_server_config_blob(input.as_bytes()).expect("decode should succeed"); + let kvs = cfg.get_value("storage_class", "_").expect("storage_class should exist"); + assert!(!kvs.0[0].hidden_if_empty); + } + + #[test] + fn test_decode_server_config_accepts_v33_object_shape() { + let input = r#"{ + "version":"33", + "credential":{"accessKey":"test","secretKey":"testtesttest"}, + "region":"us-east-1", + "worm":"off", + "storageclass":{"standard":"EC:2","rrs":"EC:1"}, + "notify":{}, + "logger":{}, + "compress":{"enabled":false}, + "openid":{}, + "policy":{"opa":{}}, + "ldapserverconfig":{} + }"#; + + let cfg = decode_server_config_blob(input.as_bytes()).expect("decode should succeed"); + let kvs = cfg.get_value("storage_class", "_").expect("storage_class should exist"); + assert_eq!(kvs.get("standard"), "EC:2"); + assert_eq!(kvs.get("rrs"), "EC:1"); + assert_eq!(kvs.get("optimize"), "availability"); + } + + #[test] + fn test_encode_server_config_writes_external_object_shape() { + let mut cfg = Config::new(); + let kvs = storage_class_kvs_mut(&mut cfg); + kvs.insert("standard".to_string(), "EC:2".to_string()); + kvs.insert("rrs".to_string(), "EC:1".to_string()); + + let out = encode_server_config_blob(&cfg, None).expect("encode should succeed"); + let v: Value = serde_json::from_slice(&out).expect("output should be json"); + assert!(v.get("version").is_some(), "external object should have version"); + assert!(v.get("storageclass").is_some(), "external object should have storageclass"); + assert!(v.get("storage_class").is_none(), "should not write rustfs map shape"); + } + + #[test] + fn test_is_standard_object_server_config_detection() { + let external = br#"{"version":"33","storageclass":{"standard":"EC:2","rrs":"EC:1"}}"#; + assert!(is_standard_object_server_config(external)); + + let legacy = br#"{"storage_class":{"_":[{"key":"standard","value":"EC:2"}]}}"#; + assert!(!is_standard_object_server_config(legacy)); + } + + #[test] + fn test_configs_semantically_equal_for_equivalent_shapes() { + let external = br#"{"version":"33","storageclass":{"standard":"EC:2","rrs":"EC:1","optimize":"availability"}}"#; + let legacy = br#"{"storage_class":{"_":[{"key":"standard","value":"EC:2"},{"key":"rrs","value":"EC:1"},{"key":"optimize","value":"availability"}]}}"#; + let lhs = decode_server_config_blob(external).expect("decode external"); + let rhs = decode_server_config_blob(legacy).expect("decode legacy"); + assert!(configs_semantically_equal(&lhs, &rhs)); + } +} diff --git a/crates/ecstore/src/config/mod.rs b/crates/ecstore/src/config/mod.rs index ab31b2515..be2c26cda 100644 --- a/crates/ecstore/src/config/mod.rs +++ b/crates/ecstore/src/config/mod.rs @@ -75,10 +75,15 @@ pub async fn init_global_config_sys(api: Arc) -> Result<()> { GLOBAL_CONFIG_SYS.init(api).await } +pub async fn try_migrate_server_config(api: Arc) { + com::try_migrate_server_config(api).await +} + #[derive(Debug, Deserialize, Serialize, Clone)] pub struct KV { pub key: String, pub value: String, + #[serde(default, alias = "hiddenIfEmpty")] pub hidden_if_empty: bool, } diff --git a/crates/ecstore/src/disk/local.rs b/crates/ecstore/src/disk/local.rs index a4160c1c7..d5fcf72ff 100644 --- a/crates/ecstore/src/disk/local.rs +++ b/crates/ecstore/src/disk/local.rs @@ -697,7 +697,6 @@ impl LocalDisk { match self.read_metadata_with_dmtime(meta_path).await { Ok(res) => Ok(res), Err(err) => { - warn!("read_raw: error: {:?}", err); if err == Error::FileNotFound && !skip_access_checks(volume_dir.as_ref().to_string_lossy().to_string().as_str()) && let Err(e) = access(volume_dir.as_ref()).await @@ -1493,6 +1492,12 @@ impl DiskAPI for LocalDisk { let erasure = &fi.erasure; for (i, part) in fi.parts.iter().enumerate() { let checksum_info = erasure.get_checksum_info(part.number); + let checksum_algo = + if fi.uses_legacy_checksum && checksum_info.algorithm == rustfs_utils::HashAlgorithm::HighwayHash256S { + rustfs_utils::HashAlgorithm::HighwayHash256SLegacy + } else { + checksum_info.algorithm + }; let part_path = self.get_object_path( volume, path_join_buf(&[ @@ -1506,7 +1511,7 @@ impl DiskAPI for LocalDisk { .bitrot_verify( &part_path, erasure.shard_file_size(part.size as i64) as usize, - checksum_info.algorithm, + checksum_algo, &checksum_info.hash, erasure.shard_size(), ) @@ -2058,7 +2063,6 @@ impl DiskAPI for LocalDisk { let search_version_id = fi.version_id.or(Some(Uuid::nil())); // Check if there's an existing version with the same version_id that has a data_dir to clean up - // Note: For non-versioned buckets, fi.version_id is None, but in xl.meta it's stored as Some(Uuid::nil()) let has_old_data_dir = { xlmeta.find_version(search_version_id).ok().and_then(|(_, ver)| { // shard_count == 0 means no other version shares this data_dir diff --git a/crates/ecstore/src/disk/mod.rs b/crates/ecstore/src/disk/mod.rs index ea7ebdbb3..4ef8cc515 100644 --- a/crates/ecstore/src/disk/mod.rs +++ b/crates/ecstore/src/disk/mod.rs @@ -23,6 +23,7 @@ pub mod local; pub mod os; pub const RUSTFS_META_BUCKET: &str = ".rustfs.sys"; +pub const MIGRATING_META_BUCKET: &str = ".minio.sys"; pub const RUSTFS_META_MULTIPART_BUCKET: &str = ".rustfs.sys/multipart"; pub const RUSTFS_META_TMP_BUCKET: &str = ".rustfs.sys/tmp"; pub const RUSTFS_META_TMP_DELETED_BUCKET: &str = ".rustfs.sys/tmp/.trash"; diff --git a/crates/ecstore/src/erasure_coding/bitrot.rs b/crates/ecstore/src/erasure_coding/bitrot.rs index 3b11b1d4c..4f4b7f1e7 100644 --- a/crates/ecstore/src/erasure_coding/bitrot.rs +++ b/crates/ecstore/src/erasure_coding/bitrot.rs @@ -173,7 +173,7 @@ where } pub fn bitrot_shard_file_size(size: usize, shard_size: usize, algo: HashAlgorithm) -> usize { - if algo != HashAlgorithm::HighwayHash256S { + if algo != HashAlgorithm::HighwayHash256S && algo != HashAlgorithm::HighwayHash256SLegacy { return size; } size.div_ceil(shard_size) * algo.size() + size diff --git a/crates/ecstore/src/erasure_coding/erasure.rs b/crates/ecstore/src/erasure_coding/erasure.rs index 8b46b3eb4..8942ab4e7 100644 --- a/crates/ecstore/src/erasure_coding/erasure.rs +++ b/crates/ecstore/src/erasure_coding/erasure.rs @@ -12,31 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Erasure coding implementation using Reed-Solomon SIMD backend. +//! Erasure coding implementation using reed-solomon-erasure (GF(2^8)). +//! Supports legacy (reed-solomon-simd) for reading/healing old-version files. //! -//! This module provides erasure coding functionality with high-performance SIMD -//! Reed-Solomon implementation: -//! -//! ## Reed-Solomon Implementation -//! -//! ### SIMD Mode (Only) -//! - **Performance**: Uses SIMD optimization for high-performance encoding/decoding -//! - **Compatibility**: Works with any shard size through SIMD implementation -//! - **Reliability**: High-performance SIMD implementation for large data processing -//! - **Use case**: Optimized for maximum performance in large data processing scenarios -//! -//! ## Example -//! -//! ```ignore -//! use rustfs_ecstore::erasure_coding::Erasure; -//! -//! let erasure = Erasure::new(4, 2, 1024); // 4 data shards, 2 parity shards, 1KB block size -//! let data = b"hello world"; -//! let shards = erasure.encode_data(data).unwrap(); -//! // Simulate loss and recovery... -//! ``` use bytes::{Bytes, BytesMut}; +use reed_solomon_erasure::galois_8::ReedSolomon; use reed_solomon_simd; use smallvec::SmallVec; use std::io; @@ -44,132 +25,88 @@ use tokio::io::AsyncRead; use tracing::warn; use uuid::Uuid; -/// Reed-Solomon encoder using SIMD implementation. -pub struct ReedSolomonEncoder { +/// Legacy calc_shard_size formula: (block_size.div_ceil(data_shards) + 1) & !1 +/// Matches main branch and filemeta::ErasureInfo for old-version files. +pub fn calc_shard_size_legacy(block_size: usize, data_shards: usize) -> usize { + (block_size.div_ceil(data_shards) + 1) & !1 +} + +/// Reed-Solomon encoder for legacy (main branch) format using reed-solomon-simd. +/// Used when decoding/encoding files with uses_legacy_checksum == true. +struct LegacyReedSolomonEncoder { data_shards: usize, parity_shards: usize, - // Use RwLock to ensure thread safety, implementing Send + Sync encoder_cache: std::sync::RwLock>, decoder_cache: std::sync::RwLock>, } -impl Clone for ReedSolomonEncoder { +impl Clone for LegacyReedSolomonEncoder { fn clone(&self) -> Self { Self { data_shards: self.data_shards, parity_shards: self.parity_shards, - // Create an empty cache for the new instance instead of sharing one encoder_cache: std::sync::RwLock::new(None), decoder_cache: std::sync::RwLock::new(None), } } } -impl ReedSolomonEncoder { - /// Create a new Reed-Solomon encoder with specified data and parity shards. - pub fn new(data_shards: usize, parity_shards: usize) -> io::Result { - Ok(ReedSolomonEncoder { - data_shards, - parity_shards, +impl LegacyReedSolomonEncoder { + fn new(_data_shards: usize, _parity_shards: usize) -> io::Result { + Ok(Self { + data_shards: _data_shards, + parity_shards: _parity_shards, encoder_cache: std::sync::RwLock::new(None), decoder_cache: std::sync::RwLock::new(None), }) } - /// Encode data shards with parity. - pub fn encode(&self, shards: SmallVec<[&mut [u8]; 16]>) -> io::Result<()> { + fn encode(&self, shards: SmallVec<[&mut [u8]; 16]>) -> io::Result<()> { let mut shards_vec: Vec<&mut [u8]> = shards.into_vec(); if shards_vec.is_empty() { return Ok(()); } - - let simd_result = self.encode_with_simd(&mut shards_vec); - - match simd_result { - Ok(()) => Ok(()), - Err(simd_error) => { - warn!("SIMD encoding failed: {}", simd_error); - Err(simd_error) - } - } - } - - fn encode_with_simd(&self, shards_vec: &mut [&mut [u8]]) -> io::Result<()> { let shard_len = shards_vec[0].len(); - - // Get or create encoder let mut encoder = { let mut cache_guard = self .encoder_cache .write() .map_err(|_| io::Error::other("Failed to acquire encoder cache lock"))?; - match cache_guard.take() { - Some(mut cached_encoder) => { - // Use reset method to reset existing encoder to adapt to new parameters - if let Err(e) = cached_encoder.reset(self.data_shards, self.parity_shards, shard_len) { - warn!("Failed to reset SIMD encoder: {:?}, creating new one", e); - // If reset fails, create new encoder + Some(mut cached) => { + if cached.reset(self.data_shards, self.parity_shards, shard_len).is_err() { reed_solomon_simd::ReedSolomonEncoder::new(self.data_shards, self.parity_shards, shard_len) .map_err(|e| io::Error::other(format!("Failed to create SIMD encoder: {e:?}")))? } else { - cached_encoder + cached } } - None => { - // First use, create new encoder - reed_solomon_simd::ReedSolomonEncoder::new(self.data_shards, self.parity_shards, shard_len) - .map_err(|e| io::Error::other(format!("Failed to create SIMD encoder: {e:?}")))? - } + None => reed_solomon_simd::ReedSolomonEncoder::new(self.data_shards, self.parity_shards, shard_len) + .map_err(|e| io::Error::other(format!("Failed to create SIMD encoder: {e:?}")))?, } }; - - // Add original shards for (i, shard) in shards_vec.iter().enumerate().take(self.data_shards) { encoder .add_original_shard(shard) .map_err(|e| io::Error::other(format!("Failed to add shard {i}: {e:?}")))?; } - - // Encode and get recovery shards let result = encoder .encode() .map_err(|e| io::Error::other(format!("SIMD encoding failed: {e:?}")))?; - - // Copy recovery shards to output buffer for (i, recovery_shard) in result.recovery_iter().enumerate() { if i + self.data_shards < shards_vec.len() { shards_vec[i + self.data_shards].copy_from_slice(recovery_shard); } } - - // Return encoder to cache (encoder is automatically reset after result is dropped, can be reused) - drop(result); // Explicitly drop result to ensure encoder is reset - + drop(result); *self .encoder_cache .write() .map_err(|_| io::Error::other("Failed to return encoder to cache"))? = Some(encoder); - Ok(()) } - /// Reconstruct missing shards. - pub fn reconstruct(&self, shards: &mut [Option>]) -> io::Result<()> { - // Use SIMD for reconstruction - let simd_result = self.reconstruct_with_simd(shards); - - match simd_result { - Ok(()) => Ok(()), - Err(simd_error) => { - warn!("SIMD reconstruction failed: {}", simd_error); - Err(simd_error) - } - } - } - - fn reconstruct_with_simd(&self, shards: &mut [Option>]) -> io::Result<()> { - // Find a valid shard to determine length + fn reconstruct(&self, shards: &mut [Option>]) -> io::Result<()> { let shard_len = shards .iter() .find_map(|s| s.as_ref().map(|v| v.len())) @@ -185,7 +122,6 @@ impl ReedSolomonEncoder { Some(mut cached_decoder) => { if let Err(e) = cached_decoder.reset(self.data_shards, self.parity_shards, shard_len) { warn!("Failed to reset SIMD decoder: {:?}, creating new one", e); - reed_solomon_simd::ReedSolomonDecoder::new(self.data_shards, self.parity_shards, shard_len) .map_err(|e| io::Error::other(format!("Failed to create SIMD decoder: {e:?}")))? } else { @@ -197,7 +133,6 @@ impl ReedSolomonEncoder { } }; - // Add available shards (both data and parity) for (i, shard_opt) in shards.iter().enumerate() { if let Some(shard) = shard_opt { if i < self.data_shards { @@ -217,7 +152,6 @@ impl ReedSolomonEncoder { .decode() .map_err(|e| io::Error::other(format!("SIMD decode error: {e:?}")))?; - // Fill in missing data shards from reconstruction result for (i, shard_opt) in shards.iter_mut().enumerate() { if shard_opt.is_none() && i < self.data_shards { for (restored_index, restored_data) in result.restored_original_iter() { @@ -240,6 +174,67 @@ impl ReedSolomonEncoder { } } +/// Reed-Solomon encoder using reed-solomon-erasure +pub struct ReedSolomonEncoder { + data_shards: usize, + parity_shards: usize, + encoder: Option, +} + +impl Clone for ReedSolomonEncoder { + fn clone(&self) -> Self { + Self { + data_shards: self.data_shards, + parity_shards: self.parity_shards, + encoder: self.encoder.clone(), + } + } +} + +impl ReedSolomonEncoder { + /// Create a new Reed-Solomon encoder with specified data and parity shards. + pub fn new(data_shards: usize, parity_shards: usize) -> io::Result { + let encoder = if parity_shards > 0 { + ReedSolomon::new(data_shards, parity_shards) + .map_err(|e| io::Error::other(format!("Failed to create Reed-Solomon encoder: {e:?}"))) + .map(Some)? + } else { + None + }; + + Ok(ReedSolomonEncoder { + data_shards, + parity_shards, + encoder, + }) + } + + /// Encode data shards with parity. + pub fn encode(&self, shards: SmallVec<[&mut [u8]; 16]>) -> io::Result<()> { + let mut shards_vec: Vec<&mut [u8]> = shards.into_vec(); + if shards_vec.is_empty() { + return Ok(()); + } + + if let Some(ref rs) = self.encoder { + rs.encode(&mut shards_vec) + .map_err(|e| io::Error::other(format!("Reed-Solomon encode failed: {e:?}"))) + } else { + Ok(()) + } + } + + /// Reconstruct missing shards. + pub fn reconstruct(&self, shards: &mut [Option>]) -> io::Result<()> { + if let Some(ref rs) = self.encoder { + rs.reconstruct_data(shards) + .map_err(|e| io::Error::other(format!("Reed-Solomon reconstruct failed: {e:?}"))) + } else { + Ok(()) + } + } +} + /// Erasure coding utility for data reliability using Reed-Solomon codes. /// /// This struct provides encoding and decoding of data into data and parity shards. @@ -262,24 +257,41 @@ impl ReedSolomonEncoder { /// let shards = erasure.encode_data(data).unwrap(); /// // Simulate loss and recovery... /// ``` - -#[derive(Default)] pub struct Erasure { pub data_shards: usize, pub parity_shards: usize, encoder: Option, + legacy_encoder: Option, pub block_size: usize, + uses_legacy: bool, _id: Uuid, _buf: Vec, } +impl Default for Erasure { + fn default() -> Self { + Self { + data_shards: 0, + parity_shards: 0, + encoder: None, + legacy_encoder: None, + block_size: 0, + uses_legacy: false, + _id: Uuid::nil(), + _buf: vec![], + } + } +} + impl Clone for Erasure { fn clone(&self) -> Self { Self { data_shards: self.data_shards, parity_shards: self.parity_shards, encoder: self.encoder.clone(), + legacy_encoder: self.legacy_encoder.clone(), block_size: self.block_size, + uses_legacy: self.uses_legacy, _id: Uuid::new_v4(), // Generate new ID for clone _buf: vec![0u8; self.block_size], } @@ -287,28 +299,44 @@ impl Clone for Erasure { } pub fn calc_shard_size(block_size: usize, data_shards: usize) -> usize { - (block_size.div_ceil(data_shards) + 1) & !1 + block_size.div_ceil(data_shards) } impl Erasure { - /// Create a new Erasure instance. + /// Create a new Erasure instance /// /// # Arguments /// * `data_shards` - Number of data shards. /// * `parity_shards` - Number of parity shards. /// * `block_size` - Block size for each shard. pub fn new(data_shards: usize, parity_shards: usize, block_size: usize) -> Self { - let encoder = if parity_shards > 0 { + Self::new_with_options(data_shards, parity_shards, block_size, false) + } + + /// Create a new Erasure instance with legacy format support. + /// + /// When `uses_legacy` is true, uses main-branch shard_size formula and reed-solomon-simd + /// for decode/reconstruct (for reading and healing old-version files). + pub fn new_with_options(data_shards: usize, parity_shards: usize, block_size: usize, uses_legacy: bool) -> Self { + let encoder = if !uses_legacy && parity_shards > 0 { Some(ReedSolomonEncoder::new(data_shards, parity_shards).unwrap()) } else { None }; + let legacy_encoder = if uses_legacy && parity_shards > 0 { + Some(LegacyReedSolomonEncoder::new(data_shards, parity_shards).unwrap()) + } else { + None + }; + Erasure { data_shards, parity_shards, block_size, encoder, + legacy_encoder, + uses_legacy, _id: Uuid::new_v4(), _buf: vec![0u8; block_size], } @@ -323,28 +351,29 @@ impl Erasure { /// A vector of encoded shards as `Bytes`. #[tracing::instrument(level = "debug", skip_all, fields(data_len=data.len()))] pub fn encode_data(&self, data: &[u8]) -> io::Result> { - // let shard_size = self.shard_size(); - // let total_size = shard_size * self.total_shard_count(); - - // Data shard count - let per_shard_size = calc_shard_size(data.len(), self.data_shards); - // Total required size + let shard_size_fn = if self.uses_legacy { + calc_shard_size_legacy + } else { + calc_shard_size + }; + let per_shard_size = shard_size_fn(data.len(), self.data_shards); let need_total_size = per_shard_size * self.total_shard_count(); - // Create a new buffer with the required total length for all shards let mut data_buffer = BytesMut::with_capacity(need_total_size); - - // Copy source data data_buffer.extend_from_slice(data); data_buffer.resize(need_total_size, 0u8); { - // EC encode, the result will be written into data_buffer let data_slices: SmallVec<[&mut [u8]; 16]> = data_buffer.chunks_exact_mut(per_shard_size).collect(); - // Only do EC if parity_shards > 0 if self.parity_shards > 0 { - if let Some(encoder) = self.encoder.as_ref() { + if self.uses_legacy { + if let Some(encoder) = self.legacy_encoder.as_ref() { + encoder.encode(data_slices)?; + } else { + warn!("parity_shards > 0, uses_legacy but legacy_encoder is None"); + } + } else if let Some(encoder) = self.encoder.as_ref() { encoder.encode(data_slices)?; } else { warn!("parity_shards > 0, but encoder is None"); @@ -372,7 +401,13 @@ impl Erasure { /// Ok if reconstruction succeeds, error otherwise. pub fn decode_data(&self, shards: &mut [Option>]) -> io::Result<()> { if self.parity_shards > 0 { - if let Some(encoder) = self.encoder.as_ref() { + if self.uses_legacy { + if let Some(encoder) = self.legacy_encoder.as_ref() { + encoder.reconstruct(shards)?; + } else { + warn!("parity_shards > 0, uses_legacy but legacy_encoder is None"); + } + } else if let Some(encoder) = self.encoder.as_ref() { encoder.reconstruct(shards)?; } else { warn!("parity_shards > 0, but encoder is None"); @@ -395,7 +430,11 @@ impl Erasure { /// Calculate the size of each shard. pub fn shard_size(&self) -> usize { - calc_shard_size(self.block_size, self.data_shards) + if self.uses_legacy { + calc_shard_size_legacy(self.block_size, self.data_shards) + } else { + calc_shard_size(self.block_size, self.data_shards) + } } /// Calculate the total erasure file size for a given original size. // Returns the final erasure size from the original size @@ -408,10 +447,15 @@ impl Erasure { } let total_length = total_length as usize; + let shard_size_fn = if self.uses_legacy { + calc_shard_size_legacy + } else { + calc_shard_size + }; let num_shards = total_length / self.block_size; let last_block_size = total_length % self.block_size; - let last_shard_size = calc_shard_size(last_block_size, self.data_shards); + let last_shard_size = shard_size_fn(last_block_size, self.data_shards); (num_shards * self.shard_size() + last_shard_size) as i64 } @@ -494,8 +538,7 @@ mod tests { #[test] fn test_shard_file_size_cases2() { let erasure = Erasure::new(12, 4, 1024 * 1024); - - assert_eq!(erasure.shard_file_size(1572864), 131074); + assert_eq!(erasure.shard_file_size(1572864), 131073); } #[test] @@ -517,11 +560,14 @@ mod tests { // Case 5: total_length > block_size, aligned assert_eq!(erasure.shard_file_size(16), 4); // 16/8=2, last=0, 2*2+0=4 - assert_eq!(erasure.shard_file_size(1248739), 312186); // 1248739/8=156092, last=3, 3 div_ceil 4=1, 156092*2+1=312185 + // MinIO-compatible: 1248739/8=156092, last=3, ceil(3/4)=1, 156092*2+1=312185 + assert_eq!(erasure.shard_file_size(1248739), 312185); - assert_eq!(erasure.shard_file_size(43), 12); // 43/8=5, last=3, 3 div_ceil 4=1, 5*2+1=11 + // MinIO-compatible: 43/8=5, last=3, ceil(3/4)=1, 5*2+1=11 + assert_eq!(erasure.shard_file_size(43), 11); - assert_eq!(erasure.shard_file_size(1572864), 393216); // 43/8=5, last=3, 3 div_ceil 4=1, 5*2+1=11 + // 1572864 with block_size=8: 196608 full blocks, last=0, 196608*2+0=393216 + assert_eq!(erasure.shard_file_size(1572864), 393216); } #[test] @@ -601,10 +647,70 @@ mod tests { #[test] fn test_shard_size_and_file_size() { let erasure = Erasure::new(4, 2, 8); + assert_eq!(erasure.shard_file_size(33), 9); + assert_eq!(erasure.shard_file_size(0), 0); + } + + #[test] + fn test_legacy_shard_size_and_file_size() { + let erasure = Erasure::new_with_options(4, 2, 8, true); + assert_eq!(erasure.shard_size(), 2); + assert_eq!(calc_shard_size_legacy(8, 4), 2); + assert_eq!(calc_shard_size_legacy(1, 4), 2); assert_eq!(erasure.shard_file_size(33), 10); assert_eq!(erasure.shard_file_size(0), 0); } + #[test] + fn test_legacy_encode_decode_roundtrip() { + let data_shards = 4; + let parity_shards = 2; + let block_size = 1024; + let erasure = Erasure::new_with_options(data_shards, parity_shards, block_size, true); + + let data = b"Legacy encode/decode roundtrip test data with sufficient length.".repeat(20); + let encoded_shards = erasure.encode_data(&data).unwrap(); + assert_eq!(encoded_shards.len(), data_shards + parity_shards); + + let mut decode_input: Vec>> = vec![None; data_shards + parity_shards]; + for i in 0..data_shards { + decode_input[i] = Some(encoded_shards[i].to_vec()); + } + + erasure.decode_data(&mut decode_input).unwrap(); + + let mut recovered = Vec::new(); + for shard in decode_input.iter().take(data_shards) { + recovered.extend_from_slice(shard.as_ref().unwrap()); + } + recovered.truncate(data.len()); + assert_eq!(&recovered, &data); + } + + #[test] + fn test_legacy_decode_with_missing_shards() { + let data_shards = 4; + let parity_shards = 2; + let block_size = 256; + let erasure = Erasure::new_with_options(data_shards, parity_shards, block_size, true); + + let data = b"Legacy decode with missing shards test.".repeat(10); + let encoded_shards = erasure.encode_data(&data).unwrap(); + + let mut shards_opt: Vec>> = encoded_shards.iter().map(|s| Some(s.to_vec())).collect(); + shards_opt[1] = None; + shards_opt[5] = None; + + erasure.decode_data(&mut shards_opt).unwrap(); + + let mut recovered = Vec::new(); + for shard in shards_opt.iter().take(data_shards) { + recovered.extend_from_slice(shard.as_ref().unwrap()); + } + recovered.truncate(data.len()); + assert_eq!(&recovered, &data); + } + #[test] fn test_shard_file_offset() { let erasure = Erasure::new(8, 8, 1024 * 1024); @@ -887,6 +993,57 @@ mod tests { assert_eq!(&recovered, &data); } + /// Generates 7557 bytes identical to MinIO generateCompatTestData. + fn generate_compat_test_data(size: usize) -> Vec { + (0..size).map(|i| ((i * 7 + 13) % 256) as u8).collect() + } + + /// Verifies reed-solomon-simd produces same shards. + /// Data shards (0-3) must match for MinIO to read RustFS part files. + /// Parity shards (4-5) differ: reed-solomon-simd vs klauspost use different RS encoding. + /// Run: cargo test -p rustfs-ecstore test_reed_solomon_compat + #[test] + fn test_reed_solomon_compat() { + let data = generate_compat_test_data(7557); + let erasure = Erasure::new(4, 2, 7557); + let shards = erasure.encode_data(&data).unwrap(); + assert_eq!(shards.len(), 6, "expected 6 shards (4 data + 2 parity)"); + + // Per-shard HighwayHash + let expected_hashes: [&str; 6] = [ + "fb3db9338e610cec541504ddae4b0bfd54445bcbd45318cf21f35f024240914d", // data 0 + "a545269a3196e18e77ef9f5ec6e735a4f4ebe82d342db666b11a5256eb305720", // data 1 + "2adbf0058f36c4cbcb5c9c16c38a6530c54198dfe504179a6f92d2349f245318", // data 2 + "898e6d060b0cb4f0e830add7e1f936bc8b78442bf582283ee244a3a058602db8", // data 3 + "4a20460bca044b3a777b26f2b0bcd371e3eab2f156f84778be3ccd8edd521ef2", // parity 4 + "eb8ba4c0db15ca910d58d031f74e4601ba2fed62ad03ec29cadde3367ab0d415", // parity 5 + ]; + + let mut data_shards_match = true; + let mut parity_shards_match = true; + for (i, shard) in shards.iter().enumerate() { + let hash = rustfs_utils::HashAlgorithm::HighwayHash256S.hash_encode(shard); + let got = hex_simd::encode_to_string(hash.as_ref(), hex_simd::AsciiCase::Lower); + let matches = got == expected_hashes[i]; + if i < 4 { + data_shards_match &= matches; + } else { + parity_shards_match &= matches; + } + if !matches { + eprintln!( + "Shard {} ({}): got {} want {}", + i, + if i < 4 { "data" } else { "parity" }, + got, + expected_hashes[i] + ); + } + } + assert!(data_shards_match, "Data shards (0-3) must match"); + assert!(parity_shards_match, "Parity shards (4-5): reed-solomon-simd differs"); + } + #[test] fn test_simd_small_data_handling() { let data_shards = 4; diff --git a/crates/ecstore/src/erasure_coding/mod.rs b/crates/ecstore/src/erasure_coding/mod.rs index 766562a93..56d4a5f0d 100644 --- a/crates/ecstore/src/erasure_coding/mod.rs +++ b/crates/ecstore/src/erasure_coding/mod.rs @@ -19,4 +19,4 @@ pub mod erasure; pub mod heal; pub use bitrot::*; -pub use erasure::{Erasure, ReedSolomonEncoder, calc_shard_size}; +pub use erasure::{Erasure, ReedSolomonEncoder, calc_shard_size, calc_shard_size_legacy}; diff --git a/crates/ecstore/src/pools.rs b/crates/ecstore/src/pools.rs index 2d9222cae..a135818c3 100644 --- a/crates/ecstore/src/pools.rs +++ b/crates/ecstore/src/pools.rs @@ -13,6 +13,17 @@ // limitations under the License. use crate::bucket::versioning_sys::BucketVersioningSys; +use crate::bucket::{ + lifecycle::{ + bucket_lifecycle_audit::LcEventSrc, + bucket_lifecycle_ops::{ + LifecycleOps, apply_expiry_on_transitioned_object, apply_expiry_rule, eval_action_from_lifecycle, + }, + lifecycle::IlmAction, + }, + metadata_sys, + object_lock::objectlock_sys::BucketObjectLockSys, +}; use crate::cache_value::metacache_set::{ListPathRawOptions, list_path_raw}; use crate::config::com::{CONFIG_PREFIX, read_config, save_config}; use crate::data_usage::DATA_USAGE_CACHE_NAME; @@ -30,25 +41,32 @@ use crate::store_api::{ BucketOperations, BucketOptions, CompletePart, GetObjectReader, HealOperations, MakeBucketOptions, MultipartOperations, ObjectIO, ObjectOperations, ObjectOptions, PutObjReader, StorageAPI, }; -use crate::{sets::Sets, store::ECStore}; +use crate::{global::GLOBAL_LifecycleSys, sets::Sets, store::ECStore}; use byteorder::{ByteOrder, LittleEndian, WriteBytesExt}; +use bytes::Bytes; use futures::future::BoxFuture; use http::HeaderMap; use rmp_serde::{Deserializer, Serializer}; use rustfs_common::defer; use rustfs_common::heal_channel::HealOpts; -use rustfs_filemeta::{MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams}; -use rustfs_rio::{HashReader, WarpReader}; +use rustfs_filemeta::{FileInfoVersions, MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams}; +use rustfs_rio::{EtagResolvable, HashReader, HashReaderDetector, Index, Reader, TryGetIndex, WarpReader}; use rustfs_utils::path::{SLASH_SEPARATOR, encode_dir_object, path_join}; use rustfs_workers::workers::Workers; +use s3s::dto::{BucketLifecycleConfiguration, DefaultRetention, ReplicationConfiguration}; use serde::{Deserialize, Serialize}; use std::collections::{HashMap, HashSet}; use std::fmt::Display; use std::io::{Cursor, Write}; use std::path::PathBuf; -use std::sync::Arc; +use std::pin::Pin; +use std::sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, +}; +use std::task::{Context, Poll}; use time::{Duration, OffsetDateTime}; -use tokio::io::{AsyncReadExt, BufReader}; +use tokio::io::{AsyncRead, AsyncReadExt, BufReader, ReadBuf}; use tokio_util::sync::CancellationToken; use tracing::{debug, error, info, warn}; @@ -75,6 +93,147 @@ pub struct PoolMeta { pub dont_save: bool, } +#[derive(Debug, Clone, Serialize, Deserialize)] +struct PersistedPoolMeta { + pub version: u16, + pub pools: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +struct PersistedPoolStatus { + #[serde(rename = "id")] + pub id: usize, + #[serde(rename = "cmdline")] + pub cmd_line: String, + #[serde(rename = "lastUpdate", with = "time::serde::rfc3339")] + pub last_update: OffsetDateTime, + #[serde(rename = "decommissionInfo")] + pub decommission: Option, +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +struct PersistedPoolDecommissionInfo { + #[serde(rename = "startTime", with = "time::serde::rfc3339::option")] + pub start_time: Option, + #[serde(rename = "startSize")] + pub start_size: usize, + #[serde(rename = "totalSize")] + pub total_size: usize, + #[serde(rename = "currentSize")] + pub current_size: usize, + #[serde(rename = "complete")] + pub complete: bool, + #[serde(rename = "failed")] + pub failed: bool, + #[serde(rename = "canceled")] + pub canceled: bool, + #[serde(rename = "queuedBuckets", default)] + pub queued_buckets: Vec, + #[serde(rename = "decommissionedBuckets", default)] + pub decommissioned_buckets: Vec, + #[serde(rename = "bucket", default)] + pub bucket: String, + #[serde(rename = "prefix", default)] + pub prefix: String, + #[serde(rename = "object", default)] + pub object: String, + #[serde(rename = "objectsDecommissioned")] + pub items_decommissioned: usize, + #[serde(rename = "objectsDecommissionedFailed")] + pub items_decommission_failed: usize, + #[serde(rename = "bytesDecommissioned")] + pub bytes_done: usize, + #[serde(rename = "bytesDecommissionedFailed")] + pub bytes_failed: usize, +} + +impl From for PoolMeta { + fn from(value: PersistedPoolMeta) -> Self { + Self { + version: value.version, + pools: value.pools.into_iter().map(Into::into).collect(), + dont_save: false, + } + } +} + +impl From for PoolStatus { + fn from(value: PersistedPoolStatus) -> Self { + Self { + id: value.id, + cmd_line: value.cmd_line, + last_update: value.last_update, + decommission: value.decommission.map(Into::into), + } + } +} + +impl From for PoolDecommissionInfo { + fn from(value: PersistedPoolDecommissionInfo) -> Self { + Self { + start_time: value.start_time, + start_size: value.start_size, + total_size: value.total_size, + current_size: value.current_size, + complete: value.complete, + failed: value.failed, + canceled: value.canceled, + queued_buckets: value.queued_buckets, + decommissioned_buckets: value.decommissioned_buckets, + bucket: value.bucket, + prefix: value.prefix, + object: value.object, + items_decommissioned: value.items_decommissioned, + items_decommission_failed: value.items_decommission_failed, + bytes_done: value.bytes_done, + bytes_failed: value.bytes_failed, + } + } +} + +impl From<&PoolMeta> for PersistedPoolMeta { + fn from(value: &PoolMeta) -> Self { + Self { + version: value.version, + pools: value.pools.iter().map(Into::into).collect(), + } + } +} + +impl From<&PoolStatus> for PersistedPoolStatus { + fn from(value: &PoolStatus) -> Self { + Self { + id: value.id, + cmd_line: value.cmd_line.clone(), + last_update: value.last_update, + decommission: value.decommission.as_ref().map(Into::into), + } + } +} + +impl From<&PoolDecommissionInfo> for PersistedPoolDecommissionInfo { + fn from(value: &PoolDecommissionInfo) -> Self { + Self { + start_time: value.start_time, + start_size: value.start_size, + total_size: value.total_size, + current_size: value.current_size, + complete: value.complete, + failed: value.failed, + canceled: value.canceled, + queued_buckets: value.queued_buckets.clone(), + decommissioned_buckets: value.decommissioned_buckets.clone(), + bucket: value.bucket.clone(), + prefix: value.prefix.clone(), + object: value.object.clone(), + items_decommissioned: value.items_decommissioned, + items_decommission_failed: value.items_decommission_failed, + bytes_done: value.bytes_done, + bytes_failed: value.bytes_failed, + } + } +} + impl PoolMeta { pub fn new(pools: &[Arc], prev_meta: &PoolMeta) -> Self { let mut new_meta = Self { @@ -144,8 +303,8 @@ impl PoolMeta { } let mut buf = Deserializer::new(Cursor::new(&data[4..])); - let meta: PoolMeta = Deserialize::deserialize(&mut buf)?; - *self = meta; + let meta: PersistedPoolMeta = Deserialize::deserialize(&mut buf)?; + *self = meta.into(); if self.version != POOL_META_VERSION { return Err(Error::other(format!("unexpected PoolMeta version: {}", self.version))); @@ -161,7 +320,7 @@ impl PoolMeta { data.write_u16::(POOL_META_FORMAT)?; data.write_u16::(POOL_META_VERSION)?; let mut buf = Vec::new(); - self.serialize(&mut Serializer::new(&mut buf))?; + PersistedPoolMeta::from(self).serialize(&mut Serializer::new(&mut buf))?; data.write_all(&buf)?; for pool in pools { @@ -178,7 +337,6 @@ impl PoolMeta { stats.last_update = OffsetDateTime::now_utc(); let mut pd = d.clone(); - pd.start_time = None; pd.canceled = true; pd.failed = false; pd.complete = false; @@ -202,7 +360,6 @@ impl PoolMeta { stats.last_update = OffsetDateTime::now_utc(); let mut pd = d.clone(); - pd.start_time = None; pd.canceled = false; pd.failed = true; pd.complete = false; @@ -226,7 +383,6 @@ impl PoolMeta { stats.last_update = OffsetDateTime::now_utc(); let mut pd = d.clone(); - pd.start_time = None; pd.canceled = false; pd.failed = false; pd.complete = true; @@ -576,6 +732,132 @@ impl Display for DecomBucketInfo { } } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum DecommissionFinalState { + Complete, + Failed, +} + +fn determine_decommission_final_state(items_failed: usize, was_cancelled: bool) -> DecommissionFinalState { + if items_failed > 0 || was_cancelled { + DecommissionFinalState::Failed + } else { + DecommissionFinalState::Complete + } +} + +fn remaining_versions_after_decommission(fivs: &FileInfoVersions) -> usize { + fivs.versions.iter().filter(|version| !version.deleted).count() +} + +fn decommission_delete_marker_opts( + version: &rustfs_filemeta::FileInfo, + version_id: Option, + src_pool_idx: usize, +) -> ObjectOptions { + ObjectOptions { + versioned: true, + version_id, + mod_time: version.mod_time, + src_pool_idx, + data_movement: true, + delete_marker: true, + skip_decommissioned: true, + delete_replication: version.replication_state_internal.clone(), + ..Default::default() + } +} + +async fn should_skip_lifecycle_for_decommission( + store: Arc, + bucket: &str, + version: &rustfs_filemeta::FileInfo, + lifecycle_config: Option<&BucketLifecycleConfiguration>, + lock_retention: Option, + replication_config: Option<(ReplicationConfiguration, OffsetDateTime)>, + apply_actions: bool, +) -> bool { + let Some(lifecycle_config) = lifecycle_config else { + return false; + }; + + let versioned = BucketVersioningSys::prefix_enabled(bucket, &version.name).await; + let object_info = crate::store_api::ObjectInfo::from_file_info(version, bucket, &version.name, versioned); + let event = eval_action_from_lifecycle(lifecycle_config, lock_retention, replication_config, &object_info).await; + + match event.action { + IlmAction::DeleteRestoredAction | IlmAction::DeleteRestoredVersionAction => { + if apply_actions && object_info.is_remote() { + let _ = apply_expiry_on_transitioned_object(store, &object_info, &event, &LcEventSrc::Decom).await; + } + false + } + IlmAction::DeleteAction + | IlmAction::DeleteVersionAction + | IlmAction::DeleteAllVersionsAction + | IlmAction::DelMarkerDeleteAllVersionsAction => { + if apply_actions { + let _ = apply_expiry_rule(&event, &LcEventSrc::Decom, &object_info).await; + } + true + } + _ => false, + } +} + +struct IndexedDecommissionReader { + inner: R, + index: Option, +} + +impl IndexedDecommissionReader { + fn new(inner: R, index: Option) -> Self { + Self { inner, index } + } +} + +impl AsyncRead for IndexedDecommissionReader { + fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll> { + Pin::new(&mut self.inner).poll_read(cx, buf) + } +} + +impl EtagResolvable for IndexedDecommissionReader {} + +impl HashReaderDetector for IndexedDecommissionReader {} + +impl TryGetIndex for IndexedDecommissionReader { + fn try_get_index(&self) -> Option<&Index> { + self.index.as_ref() + } +} + +impl Reader for IndexedDecommissionReader {} + +fn decode_part_index(index: Option<&Bytes>) -> Option { + let bytes = index?; + let mut decoded = Index::new(); + if decoded.load(bytes.as_ref()).is_ok() { + Some(decoded) + } else { + None + } +} + +fn put_obj_reader_from_chunk(chunk: Vec, size: i64, actual_size: i64, index: Option) -> Result { + use sha2::{Digest, Sha256}; + + let sha256hex = if !chunk.is_empty() { + Some(hex_simd::encode_to_string(Sha256::digest(&chunk), hex_simd::AsciiCase::Lower)) + } else { + None + }; + + let reader = IndexedDecommissionReader::new(WarpReader::new(Cursor::new(chunk)), index); + let hash_reader = HashReader::new(Box::new(reader), size, actual_size, None, sha256hex, false)?; + Ok(PutObjReader::new(hash_reader)) +} + impl ECStore { pub async fn status(&self, idx: usize) -> Result { let space_info = self.get_decommission_pool_space_info(idx).await?; @@ -621,13 +903,18 @@ impl ECStore { return Err(Error::other("InvalidArgument")); } - let Some(has_canceler) = self.decommission_cancelers.get(idx) else { - return Err(Error::other("InvalidArgument")); - }; + let canceler = { + let mut cancelers = self.decommission_cancelers.write().await; + let Some(slot) = cancelers.get_mut(idx) else { + return Err(Error::other("InvalidArgument")); + }; - if has_canceler.is_none() { - return Err(StorageError::DecommissionNotStarted); - } + let Some(canceler) = slot.take() else { + return Err(StorageError::DecommissionNotStarted); + }; + + canceler + }; let mut lock = self.pool_meta.write().await; if lock.decommission_cancel(idx) { @@ -640,6 +927,8 @@ impl ECStore { } } + canceler.cancel(); + Ok(()) } pub async fn is_decommission_running(&self) -> bool { @@ -684,8 +973,8 @@ impl ECStore { Ok(()) } - #[allow(unused_assignments)] - #[tracing::instrument(skip(self, set, wk, rcfg))] + #[allow(unused_assignments, clippy::too_many_arguments)] + #[tracing::instrument(skip(self, set, wk, lifecycle_config, lock_retention, replication_config))] async fn decommission_entry( self: &Arc, idx: usize, @@ -693,7 +982,9 @@ impl ECStore { bucket: String, set: Arc, wk: Arc, - rcfg: Option, + lifecycle_config: Option, + lock_retention: Option, + replication_config: Option<(ReplicationConfiguration, OffsetDateTime)>, ) { warn!("decommission_entry: {} {}", &bucket, &entry.name); wk.give().await; @@ -713,12 +1004,27 @@ impl ECStore { fivs.versions.sort_by(|a, b| b.mod_time.cmp(&a.mod_time)); let mut decommissioned: usize = 0; - let expired: usize = 0; + let mut expired: usize = 0; for version in fivs.versions.iter() { - // TODO: filterLifecycle + if should_skip_lifecycle_for_decommission( + self.clone(), + &bucket, + version, + lifecycle_config.as_ref(), + lock_retention.clone(), + replication_config.clone(), + true, + ) + .await + { + expired += 1; + decommissioned += 1; + continue; + } + let remaining_versions = fivs.versions.len() - expired; - if version.deleted && remaining_versions == 1 && rcfg.is_none() { + if version.deleted && remaining_versions == 1 && replication_config.is_none() { // decommissioned += 1; info!("decommission_pool: DELETE marked object with no other non-current versions will be skipped"); @@ -731,25 +1037,19 @@ impl ECStore { let mut failure = false; let mut error = None; if version.deleted { - // TODO: other params if let Err(err) = self .delete_object( bucket.as_str(), &version.name, - ObjectOptions { - versioned: true, - version_id: version_id.clone(), - mod_time: version.mod_time, - src_pool_idx: idx, - data_movement: true, - delete_marker: true, - skip_decommissioned: true, - ..Default::default() - }, + decommission_delete_marker_opts(version, version_id.clone(), idx), ) .await { if is_err_object_not_found(&err) || is_err_version_not_found(&err) || is_err_data_movement_overwrite(&err) { + warn!( + "decommission_pool: ignore delete-marker copy for {}/{} version {:?}: {:?}", + &bucket, &version.name, &version_id, &err + ); ignore = true; continue; } @@ -776,7 +1076,33 @@ impl ECStore { for _i in 0..3 { if version.is_remote() { - // TODO: DecomTieredObject + if let Err(err) = self + .decommission_tiered_object( + bucket.as_str(), + &version.name, + version, + &ObjectOptions { + version_id: version_id.clone(), + mod_time: version.mod_time, + user_defined: version.metadata.clone(), + src_pool_idx: idx, + data_movement: true, + ..Default::default() + }, + ) + .await + { + if is_err_object_not_found(&err) || is_err_version_not_found(&err) || is_err_data_movement_overwrite(&err) + { + ignore = true; + break; + } + + failure = true; + error!("decommission_pool: decommission_tiered_object err {:?}", &err); + error = Some(err); + } + break; } let bucket = bucket.clone(); @@ -847,7 +1173,8 @@ impl ECStore { } { - self.pool_meta.write().await.count_item(idx, decommissioned, failure); + let size = usize::try_from(version.size).unwrap_or_default(); + self.pool_meta.write().await.count_item(idx, size, failure); } if failure { @@ -872,6 +1199,14 @@ impl ECStore { .await { error!("decommission_pool: delete_object err {:?}", &err); + } else if decommissioned != fivs.versions.len() { + warn!( + "decommission_pool: source object retained for {}/{} because only {}/{} versions were decommissioned", + &bucket, + &entry.name, + decommissioned, + fivs.versions.len() + ); } { @@ -903,16 +1238,19 @@ impl ECStore { ) -> Result<()> { let wk = Workers::new(pool.disk_set.len() * 2).map_err(Error::other)?; - // let mut vc = None; - // replication - let rcfg: Option = None; + let mut lifecycle_config = None; + let mut lock_retention = None; + let mut replication_config = None; if bi.name != RUSTFS_META_BUCKET { - let _versioning = BucketVersioningSys::get(&bi.name).await?; - // vc = Some(versioning); - // TODO: LifecycleSys - // TODO: BucketObjectLockSys - // TODO: ReplicationConfig + let _ = BucketVersioningSys::get(&bi.name).await?; + lifecycle_config = GLOBAL_LifecycleSys.get(&bi.name).await; + lock_retention = BucketObjectLockSys::get(&bi.name).await; + replication_config = match metadata_sys::get_replication_config(&bi.name).await { + Ok(config) => Some(config), + Err(Error::ConfigNotFound) => None, + Err(err) => return Err(err), + }; } for (set_idx, set) in pool.disk_set.iter().enumerate() { @@ -925,17 +1263,22 @@ impl ECStore { let bucket = bi.name.clone(); let wk = wk.clone(); let set = set.clone(); - let rcfg = rcfg.clone(); + let lifecycle_config = lifecycle_config.clone(); + let lock_retention = lock_retention.clone(); + let replication_config = replication_config.clone(); move |entry: MetaCacheEntry| { let this = this.clone(); let bucket = bucket.clone(); let wk = wk.clone(); let set = set.clone(); - let rcfg = rcfg.clone(); + let lifecycle_config = lifecycle_config.clone(); + let lock_retention = lock_retention.clone(); + let replication_config = replication_config.clone(); Box::pin(async move { wk.take().await; - this.decommission_entry(idx, entry, bucket, set, wk, rcfg).await + this.decommission_entry(idx, entry, bucket, set, wk, lifecycle_config, lock_retention, replication_config) + .await }) } }); @@ -988,7 +1331,15 @@ impl ECStore { #[tracing::instrument(skip(self, rx))] pub async fn do_decommission_in_routine(self: &Arc, rx: CancellationToken, idx: usize) { - if let Err(err) = self.decommission_in_background(rx, idx).await { + let decommission_token = rx.child_token(); + { + let mut cancelers = self.decommission_cancelers.write().await; + if let Some(slot) = cancelers.get_mut(idx) { + *slot = Some(decommission_token.clone()); + } + } + + if let Err(err) = self.decommission_in_background(decommission_token.clone(), idx).await { error!("decom err {:?}", &err); if let Err(er) = self.decommission_failed(idx).await { error!("decom failed err {:?}", &er); @@ -1001,29 +1352,49 @@ impl ECStore { warn!("decommission: decommission_in_background complete {}", idx); - let (failed, cmd_line) = { + let (final_state, cmd_line) = { let pool_meta = self.pool_meta.read().await; - let failed = { + let final_state = { if let Some(info) = &pool_meta.pools[idx].decommission { - info.items_decommission_failed > 0 + determine_decommission_final_state(info.items_decommission_failed, info.canceled) } else { - false + DecommissionFinalState::Failed } }; let cmd_line = pool_meta.pools[idx].cmd_line.clone(); - (failed, cmd_line) + (final_state, cmd_line) }; - if !failed { + let mut completed_successfully = false; + + if final_state == DecommissionFinalState::Complete { warn!("Decommissioning complete for pool {}, verifying for any pending objects", cmd_line); - if let Err(er) = self.decommission_failed(idx).await { - error!("decom failed err {:?}", &er); + if let Err(err) = self.check_after_decommission(idx).await { + error!("decom post-check err {:?}", &err); + if let Err(er) = self.decommission_failed(idx).await { + error!("decom failed err {:?}", &er); + } + } else if let Err(er) = self.complete_decommission(idx).await { + error!("decom complete err {:?}", &er); + } else { + completed_successfully = true; } - } else if let Err(er) = self.complete_decommission(idx).await { - error!("decom complete err {:?}", &er); + } else if let Err(er) = self.decommission_failed(idx).await { + error!("decom failed err {:?}", &er); } - warn!("Decommissioning complete for pool {}", cmd_line); + { + let mut cancelers = self.decommission_cancelers.write().await; + if let Some(slot) = cancelers.get_mut(idx) { + *slot = None; + } + } + + if completed_successfully { + warn!("Decommissioning complete for pool {}", cmd_line); + } else { + warn!("Decommissioning finished in failed state for pool {}", cmd_line); + } } #[tracing::instrument(skip(self))] @@ -1043,6 +1414,14 @@ impl ECStore { } } + let canceler = { + let mut cancelers = self.decommission_cancelers.write().await; + cancelers.get_mut(idx).and_then(Option::take) + }; + if let Some(canceler) = canceler { + canceler.cancel(); + } + Ok(()) } @@ -1061,6 +1440,14 @@ impl ECStore { } } + let canceler = { + let mut cancelers = self.decommission_cancelers.write().await; + cancelers.get_mut(idx).and_then(Option::take) + }; + if let Some(canceler) = canceler { + canceler.cancel(); + } + Ok(()) } @@ -1190,6 +1577,94 @@ impl ECStore { Ok(ret) } + async fn check_after_decommission(self: &Arc, idx: usize) -> Result<()> { + let buckets = self.get_buckets_to_decommission().await?; + let pool = self.pools[idx].clone(); + + for set in &pool.disk_set { + for bucket_info in &buckets { + let mut lifecycle_config = None; + let mut lock_retention = None; + let mut replication_config = None; + if bucket_info.name != RUSTFS_META_BUCKET { + lifecycle_config = GLOBAL_LifecycleSys.get(&bucket_info.name).await; + lock_retention = BucketObjectLockSys::get(&bucket_info.name).await; + replication_config = match metadata_sys::get_replication_config(&bucket_info.name).await { + Ok(config) => Some(config), + Err(Error::ConfigNotFound) => None, + Err(err) => return Err(err), + }; + } + + let versions_found = Arc::new(AtomicUsize::new(0)); + let versions_found_cb = versions_found.clone(); + let bucket_name = bucket_info.name.clone(); + let lifecycle_config_cb = lifecycle_config.clone(); + let lock_retention_cb = lock_retention.clone(); + let replication_config_cb = replication_config.clone(); + let store = Arc::clone(self); + + let callback: ListCallback = Arc::new(move |entry: MetaCacheEntry| { + let versions_found = versions_found_cb.clone(); + let bucket_name = bucket_name.clone(); + let lifecycle_config = lifecycle_config_cb.clone(); + let lock_retention = lock_retention_cb.clone(); + let replication_config = replication_config_cb.clone(); + let store = Arc::clone(&store); + Box::pin(async move { + if !entry.is_object() { + return; + } + + if bucket_name == RUSTFS_META_BUCKET && entry.name.contains(DATA_USAGE_CACHE_NAME) { + return; + } + + let Ok(fivs) = entry.file_info_versions(&bucket_name) else { + return; + }; + + let mut remaining = 0; + for version in &fivs.versions { + if version.deleted { + continue; + } + if should_skip_lifecycle_for_decommission( + Arc::clone(&store), + &bucket_name, + version, + lifecycle_config.as_ref(), + lock_retention.clone(), + replication_config.clone(), + false, + ) + .await + { + continue; + } + remaining += 1; + } + + versions_found.fetch_add(remaining, Ordering::Relaxed); + }) + }); + + set.list_objects_to_decommission(CancellationToken::new(), bucket_info.clone(), callback) + .await?; + + let versions_found = versions_found.load(Ordering::Relaxed); + if versions_found > 0 { + return Err(Error::other(format!( + "at least {versions_found} object(s)/version(s) were found in bucket `{}` after decommissioning", + bucket_info.name + ))); + } + } + } + + Ok(()) + } + #[tracing::instrument(skip(self, rd))] async fn decommission_object(self: Arc, pool_idx: usize, bucket: String, rd: GetObjectReader) -> Result<()> { warn!("decommission_object: start {} {}", &bucket, &rd.object_info.name); @@ -1238,7 +1713,10 @@ impl ECStore { reader.read_exact(&mut chunk).await?; - let mut data = PutObjReader::from_vec(chunk); + let part_size = i64::try_from(part.size).map_err(|_| Error::other("part size overflow"))?; + let part_actual_size = if part.actual_size > 0 { part.actual_size } else { part_size }; + let index = decode_part_index(part.index.as_ref()); + let mut data = put_obj_reader_from_chunk(chunk, part_size, part_actual_size, index)?; let pi = match self .put_object_part( @@ -1294,8 +1772,13 @@ impl ECStore { return Ok(()); } - let reader = BufReader::new(rd.stream); - let hrd = HashReader::new(Box::new(WarpReader::new(reader)), object_info.size, object_info.size, None, None, false)?; + let actual_size = object_info.get_actual_size()?; + let index = object_info + .parts + .first() + .and_then(|part| decode_part_index(part.index.as_ref())); + let reader = IndexedDecommissionReader::new(WarpReader::new(BufReader::new(rd.stream)), index); + let hrd = HashReader::new(Box::new(reader), object_info.size, actual_size, object_info.etag.clone(), None, false)?; let mut data = PutObjReader::new(hrd); if let Err(err) = self @@ -1325,6 +1808,159 @@ impl ECStore { } } +#[cfg(test)] +#[allow(clippy::items_after_test_module)] +mod tests { + use super::*; + + #[test] + fn determine_decommission_final_state_marks_failures_and_cancellations() { + assert_eq!(determine_decommission_final_state(0, false), DecommissionFinalState::Complete); + assert_eq!(determine_decommission_final_state(1, false), DecommissionFinalState::Failed); + assert_eq!(determine_decommission_final_state(0, true), DecommissionFinalState::Failed); + } + + #[test] + fn remaining_versions_after_decommission_ignores_delete_markers() { + let fivs = FileInfoVersions { + versions: vec![ + rustfs_filemeta::FileInfo { + deleted: false, + size: 128, + ..Default::default() + }, + rustfs_filemeta::FileInfo { + deleted: true, + size: 0, + ..Default::default() + }, + ], + ..Default::default() + }; + + assert_eq!(remaining_versions_after_decommission(&fivs), 1); + } + + #[test] + fn decommission_delete_marker_opts_preserves_replication_state() { + let mod_time = OffsetDateTime::now_utc(); + let version = rustfs_filemeta::FileInfo { + mod_time: Some(mod_time), + replication_state_internal: Some(rustfs_filemeta::ReplicationState { + replica_status: rustfs_filemeta::ReplicationStatusType::Replica, + delete_marker: true, + replicate_decision_str: "existing".to_string(), + ..Default::default() + }), + ..Default::default() + }; + + let opts = decommission_delete_marker_opts(&version, Some("version-id".to_string()), 7); + let replication = opts.delete_replication.expect("replication state should be preserved"); + + assert!(opts.versioned); + assert!(opts.data_movement); + assert!(opts.delete_marker); + assert!(opts.skip_decommissioned); + assert_eq!(opts.src_pool_idx, 7); + assert_eq!(opts.version_id.as_deref(), Some("version-id")); + assert_eq!(opts.mod_time, Some(mod_time)); + assert_eq!(replication.replica_status, rustfs_filemeta::ReplicationStatusType::Replica); + assert!(replication.delete_marker); + assert_eq!(replication.replicate_decision_str, "existing"); + } + + #[test] + fn decommission_state_transitions_preserve_start_time() { + let start_time = OffsetDateTime::now_utc(); + let mut pool_meta = PoolMeta { + version: POOL_META_VERSION, + pools: vec![PoolStatus { + id: 0, + cmd_line: "/tmp/pool".to_string(), + last_update: start_time, + decommission: Some(PoolDecommissionInfo { + start_time: Some(start_time), + ..Default::default() + }), + }], + dont_save: true, + }; + + assert!(pool_meta.decommission_failed(0)); + assert_eq!( + pool_meta.pools[0].decommission.as_ref().and_then(|info| info.start_time), + Some(start_time) + ); + + assert!(pool_meta.decommission_complete(0)); + assert_eq!( + pool_meta.pools[0].decommission.as_ref().and_then(|info| info.start_time), + Some(start_time) + ); + + assert!(pool_meta.decommission_cancel(0)); + assert_eq!( + pool_meta.pools[0].decommission.as_ref().and_then(|info| info.start_time), + Some(start_time) + ); + } + + #[test] + fn pool_meta_persists_decommission_resume_queues() { + let start_time = OffsetDateTime::now_utc(); + let pool_meta = PoolMeta { + version: POOL_META_VERSION, + pools: vec![PoolStatus { + id: 1, + cmd_line: "/data/pool1/disk{1...4}".to_string(), + last_update: start_time, + decommission: Some(PoolDecommissionInfo { + start_time: Some(start_time), + queued_buckets: vec!["bucket-a".to_string(), "bucket-b/prefix".to_string()], + decommissioned_buckets: vec!["bucket-done".to_string()], + bucket: "bucket-b".to_string(), + prefix: "prefix".to_string(), + object: "object.txt".to_string(), + items_decommissioned: 7, + items_decommission_failed: 1, + bytes_done: 1024, + bytes_failed: 128, + ..Default::default() + }), + }], + dont_save: false, + }; + + let mut buf = Vec::new(); + PersistedPoolMeta::from(&pool_meta) + .serialize(&mut Serializer::new(&mut buf)) + .expect("pool meta should serialize"); + + let mut deserializer = Deserializer::new(Cursor::new(&buf)); + let restored: PoolMeta = PersistedPoolMeta::deserialize(&mut deserializer) + .expect("pool meta should deserialize") + .into(); + + let restored_decommission = restored.pools[0] + .decommission + .as_ref() + .expect("decommission info should survive round-trip"); + assert_eq!( + restored_decommission.queued_buckets, + vec!["bucket-a".to_string(), "bucket-b/prefix".to_string()] + ); + assert_eq!(restored_decommission.decommissioned_buckets, vec!["bucket-done".to_string()]); + assert_eq!(restored_decommission.bucket, "bucket-b"); + assert_eq!(restored_decommission.prefix, "prefix"); + assert_eq!(restored_decommission.object, "object.txt"); + assert_eq!(restored_decommission.items_decommissioned, 7); + assert_eq!(restored_decommission.items_decommission_failed, 1); + assert_eq!(restored_decommission.bytes_done, 1024); + assert_eq!(restored_decommission.bytes_failed, 128); + } +} + // impl Fn(MetaCacheEntry) -> impl Future> pub type ListCallback = Arc BoxFuture<'static, ()> + Send + Sync + 'static>; diff --git a/crates/ecstore/src/set_disk.rs b/crates/ecstore/src/set_disk.rs index 8d86ad568..0c787ec4e 100644 --- a/crates/ecstore/src/set_disk.rs +++ b/crates/ecstore/src/set_disk.rs @@ -80,9 +80,12 @@ use rustfs_lock::{FastLockGuard, NamespaceLock, NamespaceLockGuard, NamespaceLoc use rustfs_madmin::heal_commands::{HealDriveInfo, HealResultItem}; use rustfs_rio::{EtagResolvable, HashReader, HashReaderMut, TryGetIndex as _, WarpReader}; use rustfs_s3_common::EventName; -use rustfs_utils::http::RUSTFS_BUCKET_REPLICATION_SSEC_CHECKSUM; +use rustfs_utils::http::headers::AMZ_OBJECT_TAGGING; use rustfs_utils::http::headers::AMZ_STORAGE_CLASS; -use rustfs_utils::http::headers::{AMZ_OBJECT_TAGGING, RESERVED_METADATA_PREFIX, RESERVED_METADATA_PREFIX_LOWER}; +use rustfs_utils::http::{ + SUFFIX_ACTUAL_OBJECT_SIZE_CAP, SUFFIX_ACTUAL_SIZE, SUFFIX_COMPRESSION, SUFFIX_COMPRESSION_SIZE, SUFFIX_REPLICATION_SSEC_CRC, + contains_key_str, get_header_map, get_str, insert_str, remove_header_map, +}; use rustfs_utils::{ HashAlgorithm, crypto::hex, @@ -136,6 +139,30 @@ pub fn get_lock_acquire_timeout() -> Duration { Duration::from_secs(rustfs_utils::get_env_u64("RUSTFS_LOCK_ACQUIRE_TIMEOUT", 5)) } +fn build_tiered_decommission_file_info( + bucket: &str, + object: &str, + fi: &FileInfo, + disk_count: usize, + default_parity_count: usize, + storage_class: Option<&str>, +) -> (FileInfo, usize) { + let parity_drives = GLOBAL_STORAGE_CLASS + .get() + .and_then(|sc| sc.get_parity_for_sc(storage_class.unwrap_or_default())) + .unwrap_or(default_parity_count); + let data_drives = disk_count - parity_drives; + let mut write_quorum = data_drives; + if data_drives == parity_drives { + write_quorum += 1; + } + + let mut updated = fi.clone(); + updated.erasure = FileInfo::new([bucket, object].join("/").as_str(), data_drives, parity_drives).erasure; + + (updated, write_quorum) +} + #[derive(Clone, Debug)] pub struct SetDisks { pub locker_owner: String, @@ -620,7 +647,7 @@ impl ObjectIO for SetDisks { &tmp_object, erasure.shard_file_size(data.size()), erasure.shard_size(), - HashAlgorithm::HighwayHash256, + HashAlgorithm::HighwayHash256S, ) .await { @@ -678,8 +705,8 @@ impl ObjectIO for SetDisks { ))); } - if user_defined.contains_key(&format!("{RESERVED_METADATA_PREFIX_LOWER}compression")) { - user_defined.insert(format!("{RESERVED_METADATA_PREFIX_LOWER}compression-size"), w_size.to_string()); + if contains_key_str(&user_defined, SUFFIX_COMPRESSION) { + insert_str(&mut user_defined, SUFFIX_COMPRESSION_SIZE, w_size.to_string()); } let index_op = data.stream.try_get_index().map(|v| v.clone().into_vec()); @@ -765,7 +792,7 @@ impl ObjectIO for SetDisks { .await?; if let Some(old_dir) = op_old_dir { - self.commit_rename_data_dir(&shuffle_disks, bucket, object, &old_dir.to_string(), write_quorum) + self.commit_rename_data_dir(&online_disks, bucket, object, &old_dir.to_string(), write_quorum) .await?; } @@ -1336,6 +1363,12 @@ impl ObjectOperations for SetDisks { let mut delete_marker = opts.versioned; if opts.version_id.is_some() { + // Decommission/rebalance may recreate a delete marker on a new pool before that + // exact version exists there, so we must still treat it as a mark-delete write. + if opts.data_movement && opts.delete_marker && !version_found { + mark_delete = true; + } + if version_found && opts.delete_marker_replication_status() == ReplicationStatusType::Replica { mark_delete = false; } @@ -1889,6 +1922,68 @@ impl ObjectOperations for SetDisks { } } +impl SetDisks { + #[tracing::instrument(skip(self, fi, opts))] + pub(crate) async fn decommission_tiered_object( + &self, + bucket: &str, + object: &str, + fi: &FileInfo, + opts: &ObjectOptions, + ) -> Result<()> { + let _lock_guard = if !opts.no_lock { + Some( + self.new_ns_lock(bucket, object) + .await? + .get_write_lock(get_lock_acquire_timeout()) + .await + .map_err(|e| { + Error::other(format!( + "Failed to acquire write lock: {}", + self.format_lock_error_from_error(bucket, object, "write", &e) + )) + })?, + ) + } else { + None + }; + + let disks = self.disks.read().await.clone(); + let storage_class = opts.user_defined.get(AMZ_STORAGE_CLASS).map(String::as_str); + let (fi, write_quorum) = + build_tiered_decommission_file_info(bucket, object, fi, disks.len(), self.default_parity_count, storage_class); + let parts_metadata = vec![fi.clone(); disks.len()]; + let (shuffle_disks, parts_metadata) = Self::shuffle_disks_and_parts_metadata(&disks, &parts_metadata, &fi); + + let mut errs = Vec::with_capacity(shuffle_disks.len()); + let mut futures = Vec::with_capacity(shuffle_disks.len()); + for (index, disk) in shuffle_disks.iter().enumerate() { + let mut file_info = parts_metadata[index].clone(); + file_info.erasure.index = index + 1; + futures.push(async move { + if let Some(disk) = disk { + disk.write_metadata("", bucket, object, file_info).await + } else { + Err(DiskError::DiskNotFound) + } + }); + } + + for result in join_all(futures).await { + match result { + Ok(_) => errs.push(None), + Err(err) => errs.push(Some(err)), + } + } + + if let Some(err) = reduce_write_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, write_quorum) { + return Err(to_object_err(err.into(), vec![bucket, object])); + } + + Ok(()) + } +} + #[async_trait::async_trait] impl ListOperations for SetDisks { #[tracing::instrument(skip(self))] @@ -2007,7 +2102,7 @@ impl MultipartOperations for SetDisks { &tmp_part_path, erasure.shard_file_size(data.size()), erasure.shard_size(), - HashAlgorithm::HighwayHash256, + HashAlgorithm::HighwayHash256S, ) .await { @@ -2454,11 +2549,11 @@ impl MultipartOperations for SetDisks { fi.data_dir = Some(Uuid::new_v4()); - if let Some(cssum) = user_defined.get(RUSTFS_BUCKET_REPLICATION_SSEC_CHECKSUM) + if let Some(cssum) = get_header_map(&user_defined, SUFFIX_REPLICATION_SSEC_CRC) && !cssum.is_empty() { - fi.checksum = base64_simd::STANDARD.decode_to_vec(cssum).ok().map(Bytes::from); - user_defined.remove(RUSTFS_BUCKET_REPLICATION_SSEC_CHECKSUM); + fi.checksum = base64_simd::STANDARD.decode_to_vec(&cssum).ok().map(Bytes::from); + remove_header_map(&mut user_defined, SUFFIX_REPLICATION_SSEC_CRC); } let parts_metadata = vec![fi.clone(); disks.len()]; @@ -2809,8 +2904,8 @@ impl MultipartOperations for SetDisks { } } - if let Some(rc_crc) = opts.user_defined.get(RUSTFS_BUCKET_REPLICATION_SSEC_CHECKSUM) { - if let Ok(rc_crc_bytes) = base64_simd::STANDARD.decode_to_vec(rc_crc) { + if let Some(rc_crc) = get_header_map(&opts.user_defined, SUFFIX_REPLICATION_SSEC_CRC) { + if let Ok(rc_crc_bytes) = base64_simd::STANDARD.decode_to_vec(&rc_crc) { fi.checksum = Some(Bytes::from(rc_crc_bytes)); } else { error!("complete_multipart_upload decode rc_crc failed rc_crc={}", rc_crc); @@ -2849,25 +2944,19 @@ impl MultipartOperations for SetDisks { fi.metadata.insert("etag".to_owned(), etag); if opts.replication_request { - if let Some(actual_size) = opts - .user_defined - .get(format!("{RESERVED_METADATA_PREFIX_LOWER}Actual-Object-Size").as_str()) - { + if let Some(actual_size) = get_str(&opts.user_defined, SUFFIX_ACTUAL_OBJECT_SIZE_CAP) { + insert_str(&mut fi.metadata, SUFFIX_ACTUAL_SIZE, actual_size.clone()); fi.metadata - .insert(format!("{RESERVED_METADATA_PREFIX}actual-size"), actual_size.clone()); - fi.metadata - .insert("x-rustfs-encryption-original-size".to_string(), actual_size.to_string()); + .insert("x-rustfs-encryption-original-size".to_string(), actual_size); } } else { - fi.metadata - .insert(format!("{RESERVED_METADATA_PREFIX}actual-size"), object_actual_size.to_string()); + insert_str(&mut fi.metadata, SUFFIX_ACTUAL_SIZE, object_actual_size.to_string()); fi.metadata .insert("x-rustfs-encryption-original-size".to_string(), object_actual_size.to_string()); } if fi.is_compressed() { - fi.metadata - .insert(format!("{RESERVED_METADATA_PREFIX_LOWER}compression-size"), object_size.to_string()); + insert_str(&mut fi.metadata, SUFFIX_COMPRESSION_SIZE, object_size.to_string()); } if opts.data_movement { @@ -2927,7 +3016,7 @@ impl MultipartOperations for SetDisks { .await?; if let Some(old_dir) = op_old_dir { - self.commit_rename_data_dir(&shuffle_disks, bucket, object, &old_dir.to_string(), write_quorum) + self.commit_rename_data_dir(&online_disks, bucket, object, &old_dir.to_string(), write_quorum) .await?; } @@ -3356,12 +3445,18 @@ async fn disks_with_all_parts( if (meta.data.is_some() || meta.size == 0) && !meta.parts.is_empty() { if let Some(data) = &meta.data { let checksum_info = meta.erasure.get_checksum_info(meta.parts[0].number); + let checksum_algo = + if meta.uses_legacy_checksum && checksum_info.algorithm == rustfs_utils::HashAlgorithm::HighwayHash256S { + rustfs_utils::HashAlgorithm::HighwayHash256SLegacy + } else { + checksum_info.algorithm + }; let data_len = data.len(); let verify_err = bitrot_verify( Box::new(Cursor::new(data.clone())), data_len, meta.erasure.shard_file_size(meta.size) as usize, - checksum_info.algorithm, + checksum_algo, checksum_info.hash, meta.erasure.shard_size(), ) @@ -4161,6 +4256,33 @@ mod tests { assert!(e_tag_matches("\"abc\"", "*")); } + #[test] + fn test_build_tiered_decommission_file_info_preserves_transition_metadata() { + let version_id = Uuid::new_v4(); + let transition_version_id = Uuid::new_v4(); + let original = FileInfo { + version_id: Some(version_id), + transition_status: TRANSITION_COMPLETE.to_string(), + transitioned_objname: "remote/object".to_string(), + transition_tier: "WARM-TIER".to_string(), + transition_version_id: Some(transition_version_id), + erasure: FileInfo::new("old-bucket/old-object", 8, 8).erasure, + ..Default::default() + }; + + let (updated, write_quorum) = build_tiered_decommission_file_info("bucket", "object", &original, 16, 4, None); + + assert_eq!(updated.version_id, original.version_id); + assert_eq!(updated.transition_status, original.transition_status); + assert_eq!(updated.transitioned_objname, original.transitioned_objname); + assert_eq!(updated.transition_tier, original.transition_tier); + assert_eq!(updated.transition_version_id, original.transition_version_id); + assert_eq!(updated.erasure.data_blocks, 12); + assert_eq!(updated.erasure.parity_blocks, 4); + assert_eq!(write_quorum, 12); + assert_ne!(updated.erasure.distribution, original.erasure.distribution); + } + #[test] fn test_should_prevent_write() { let oi = ObjectInfo { diff --git a/crates/ecstore/src/set_disk/heal.rs b/crates/ecstore/src/set_disk/heal.rs index cdc3cb079..3418f6d29 100644 --- a/crates/ecstore/src/set_disk/heal.rs +++ b/crates/ecstore/src/set_disk/heal.rs @@ -124,11 +124,12 @@ impl SetDisks { ); let erasure = if !latest_meta.deleted && !latest_meta.is_remote() { - // Initialize erasure coding - erasure_coding::Erasure::new( + // Initialize erasure coding; use legacy mode for old-version files + erasure_coding::Erasure::new_with_options( latest_meta.erasure.data_blocks, latest_meta.erasure.parity_blocks, latest_meta.erasure.block_size, + latest_meta.uses_legacy_checksum, ) } else { erasure_coding::Erasure::default() @@ -347,7 +348,14 @@ impl SetDisks { for (part_index, part) in latest_meta.parts.iter().enumerate() { let till_offset = erasure.shard_file_offset(0, part.size, part.size); - let checksum_algo = erasure_info.get_checksum_info(part.number).algorithm; + let checksum_info = erasure_info.get_checksum_info(part.number); + let checksum_algo = if latest_meta.uses_legacy_checksum + && checksum_info.algorithm == rustfs_utils::HashAlgorithm::HighwayHash256S + { + rustfs_utils::HashAlgorithm::HighwayHash256SLegacy + } else { + checksum_info.algorithm + }; let mut readers = Vec::with_capacity(latest_disks.len()); let mut writers = Vec::with_capacity(out_dated_disks.len()); // let mut errors = Vec::with_capacity(out_dated_disks.len()); @@ -420,7 +428,7 @@ impl SetDisks { ]), erasure.shard_file_size(part.size as i64), erasure.shard_size(), - HashAlgorithm::HighwayHash256, + HashAlgorithm::HighwayHash256S, ) .await { diff --git a/crates/ecstore/src/set_disk/read.rs b/crates/ecstore/src/set_disk/read.rs index be059408a..e8306f91c 100644 --- a/crates/ecstore/src/set_disk/read.rs +++ b/crates/ecstore/src/set_disk/read.rs @@ -603,7 +603,12 @@ impl SetDisks { object, offset, length, end_offset, part_index, last_part_index, last_part_relative_offset, "Multipart read bounds" ); - let erasure = erasure_coding::Erasure::new(fi.erasure.data_blocks, fi.erasure.parity_blocks, fi.erasure.block_size); + let erasure = erasure_coding::Erasure::new_with_options( + fi.erasure.data_blocks, + fi.erasure.parity_blocks, + fi.erasure.block_size, + fi.uses_legacy_checksum, + ); let part_indices: Vec = (part_index..=last_part_index).collect(); debug!(bucket, object, ?part_indices, "Multipart part indices to stream"); @@ -648,6 +653,14 @@ impl SetDisks { "Streaming multipart part" ); + let checksum_info = fi.erasure.get_checksum_info(part_number); + let checksum_algo = + if fi.uses_legacy_checksum && checksum_info.algorithm == rustfs_utils::HashAlgorithm::HighwayHash256S { + rustfs_utils::HashAlgorithm::HighwayHash256SLegacy + } else { + checksum_info.algorithm + }; + let mut readers = Vec::with_capacity(disks.len()); let mut errors = Vec::with_capacity(disks.len()); for (idx, disk_op) in disks.iter().enumerate() { @@ -659,7 +672,7 @@ impl SetDisks { read_offset, till_offset, erasure.shard_size(), - HashAlgorithm::HighwayHash256, + checksum_algo.clone(), skip_verify_bitrot, ) .await diff --git a/crates/ecstore/src/store.rs b/crates/ecstore/src/store.rs index 6c63e4d72..c58955a24 100644 --- a/crates/ecstore/src/store.rs +++ b/crates/ecstore/src/store.rs @@ -156,7 +156,7 @@ pub struct ECStore { // pub local_disks: Vec, pub pool_meta: RwLock, pub rebalance_meta: RwLock>, - pub decommission_cancelers: Vec>, + pub decommission_cancelers: RwLock>>, } // impl Clone for ECStore { diff --git a/crates/ecstore/src/store/init.rs b/crates/ecstore/src/store/init.rs index 364b7a1e4..2947cf082 100644 --- a/crates/ecstore/src/store/init.rs +++ b/crates/ecstore/src/store/init.rs @@ -149,7 +149,7 @@ impl ECStore { let mut pool_meta = PoolMeta::new(&pools, &PoolMeta::default()); pool_meta.dont_save = true; - let decommission_cancelers = vec![None; pools.len()]; + let decommission_cancelers = RwLock::new(vec![None; pools.len()]); let ec = Arc::new(ECStore { id: deployment_id.unwrap(), disk_map, @@ -265,6 +265,7 @@ impl ECStore { init_background_expiry(self.clone()).await; TransitionState::init(self.clone()).await; + crate::tier::tier::try_migrate_tiering_config(self.clone()).await; if let Err(err) = GLOBAL_TierConfigMgr.write().await.init(self.clone()).await { info!("TierConfigMgr init error: {}", err); diff --git a/crates/ecstore/src/store/object.rs b/crates/ecstore/src/store/object.rs index 9d56bde1a..63dda34e1 100644 --- a/crates/ecstore/src/store/object.rs +++ b/crates/ecstore/src/store/object.rs @@ -14,7 +14,64 @@ use super::*; +fn select_data_movement_target_pool( + existing_pool_idx: Result, + src_pool_idx: usize, + delete_marker: bool, +) -> Result> { + match existing_pool_idx { + Ok(pool_idx) => { + if delete_marker && pool_idx == src_pool_idx { + Ok(None) + } else { + Ok(Some(pool_idx)) + } + } + Err(err) => { + if is_err_read_quorum(&err) { + return Err(StorageError::ErasureWriteQuorum); + } + if delete_marker && (is_err_object_not_found(&err) || is_err_version_not_found(&err)) { + Ok(None) + } else { + Err(err) + } + } + } +} + impl ECStore { + #[instrument(skip(self, fi, opts))] + pub(crate) async fn decommission_tiered_object( + &self, + bucket: &str, + object: &str, + fi: &rustfs_filemeta::FileInfo, + opts: &ObjectOptions, + ) -> Result<()> { + check_put_object_args(bucket, object)?; + + let object = encode_dir_object(object); + + if self.single_pool() { + return Err(Error::other(format!("error decommissioning {bucket}/{object}"))); + } + + let idx = self.get_pool_idx_no_lock(bucket, &object, fi.size).await?; + if opts.data_movement && idx == opts.src_pool_idx { + return Err(StorageError::DataMovementOverwriteErr( + bucket.to_owned(), + object.to_owned(), + opts.version_id.clone().unwrap_or_default(), + )); + } + + self.pools[idx] + .get_disks_by_key(&object) + .decommission_tiered_object(bucket, &object, fi, opts) + .await + } + #[instrument(level = "debug", skip(self))] pub(super) async fn handle_get_object_reader( &self, @@ -179,6 +236,30 @@ impl ECStore { let mut gopts = opts.clone(); gopts.no_lock = true; + if opts.data_movement { + let existing_pool_idx = self + .get_pool_info_existing_with_opts(bucket, object, &gopts) + .await + .map(|(pinfo, _)| pinfo.index); + let target_pool_idx = + match select_data_movement_target_pool(existing_pool_idx, opts.src_pool_idx, opts.delete_marker)? { + Some(pool_idx) => pool_idx, + None => self.get_pool_idx_no_lock(bucket, object, 0).await?, + }; + + if opts.src_pool_idx == target_pool_idx { + return Err(StorageError::DataMovementOverwriteErr( + bucket.to_owned(), + object.to_owned(), + opts.version_id.unwrap_or_default(), + )); + } + + let mut obj = self.pools[target_pool_idx].delete_object(bucket, object, opts).await?; + obj.name = decode_dir_object(obj.name.as_str()); + return Ok(obj); + } + // Determine which pool contains it let (mut pinfo, errs) = self .get_pool_info_existing_with_opts(bucket, object, &gopts) @@ -204,12 +285,6 @@ impl ECStore { )); } - if opts.data_movement { - let mut obj = self.pools[pinfo.index].delete_object(bucket, object, opts).await?; - obj.name = decode_dir_object(obj.name.as_str()); - return Ok(obj); - } - if !errs.is_empty() && !opts.versioned && !opts.version_suspended { return self.delete_object_from_all_pools(bucket, object, &opts, errs).await; } @@ -565,3 +640,27 @@ impl ECStore { Ok(()) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn delete_marker_data_movement_falls_back_when_only_source_pool_has_object() { + let target = select_data_movement_target_pool(Ok(1), 1, true).unwrap(); + assert_eq!(target, None); + } + + #[test] + fn delete_marker_data_movement_falls_back_when_version_does_not_exist_yet() { + let err = StorageError::ObjectNotFound("bucket".to_string(), "object".to_string()); + let target = select_data_movement_target_pool(Err(err), 1, true).unwrap(); + assert_eq!(target, None); + } + + #[test] + fn non_delete_marker_data_movement_keeps_existing_pool() { + let target = select_data_movement_target_pool(Ok(0), 1, false).unwrap(); + assert_eq!(target, Some(0)); + } +} diff --git a/crates/ecstore/src/store_api.rs b/crates/ecstore/src/store_api.rs index cc40648c0..7ce1d2355 100644 --- a/crates/ecstore/src/store_api.rs +++ b/crates/ecstore/src/store_api.rs @@ -27,8 +27,8 @@ use bytes::Bytes; use http::{HeaderMap, HeaderValue}; use rustfs_common::heal_channel::HealOpts; use rustfs_filemeta::{ - FileInfo, MetaCacheEntriesSorted, ObjectPartInfo, REPLICATION_RESET, REPLICATION_STATUS, ReplicateDecision, ReplicationState, - ReplicationStatusType, RestoreStatusOps as _, VersionPurgeStatusType, parse_restore_obj_status, replication_statuses_map, + FileInfo, MetaCacheEntriesSorted, ObjectPartInfo, ReplicateDecision, ReplicationState, ReplicationStatusType, + RestoreStatusOps as _, VersionPurgeStatusType, parse_restore_obj_status, replication_statuses_map, version_purge_statuses_map, }; use rustfs_lock::NamespaceLockWrapper; @@ -36,7 +36,7 @@ use rustfs_madmin::heal_commands::HealResultItem; use rustfs_rio::Checksum; use rustfs_rio::{DecompressReader, HashReader, LimitReader, WarpReader}; use rustfs_utils::CompressionAlgorithm; -use rustfs_utils::http::headers::{AMZ_OBJECT_TAGGING, RESERVED_METADATA_PREFIX_LOWER}; +use rustfs_utils::http::headers::AMZ_OBJECT_TAGGING; use rustfs_utils::http::{AMZ_BUCKET_REPLICATION_STATUS, AMZ_RESTORE, AMZ_STORAGE_CLASS}; use rustfs_utils::path::decode_dir_object; use serde::{Deserialize, Serialize}; diff --git a/crates/ecstore/src/store_api/types.rs b/crates/ecstore/src/store_api/types.rs index 5a41ac1cd..9acc239df 100644 --- a/crates/ecstore/src/store_api/types.rs +++ b/crates/ecstore/src/store_api/types.rs @@ -118,11 +118,8 @@ impl ObjectOptions { } pub fn put_replication_state(&self) -> ReplicationState { - let rs = match self - .user_defined - .get(format!("{RESERVED_METADATA_PREFIX_LOWER}{REPLICATION_STATUS}").as_str()) - { - Some(v) => v.to_string(), + let rs = match rustfs_utils::http::get_str(&self.user_defined, rustfs_utils::http::SUFFIX_REPLICATION_STATUS) { + Some(v) => v, None => return ReplicationState::default(), }; @@ -341,15 +338,11 @@ impl Clone for ObjectInfo { impl ObjectInfo { pub fn is_compressed(&self) -> bool { - self.user_defined - .contains_key(&format!("{RESERVED_METADATA_PREFIX_LOWER}compression")) + rustfs_utils::http::contains_key_str(&self.user_defined, rustfs_utils::http::SUFFIX_COMPRESSION) } pub fn is_compressed_ok(&self) -> Result<(CompressionAlgorithm, bool)> { - let scheme = self - .user_defined - .get(&format!("{RESERVED_METADATA_PREFIX_LOWER}compression")) - .cloned(); + let scheme = rustfs_utils::http::get_str(&self.user_defined, rustfs_utils::http::SUFFIX_COMPRESSION); if let Some(scheme) = scheme { let algorithm = CompressionAlgorithm::from_str(&scheme)?; @@ -369,7 +362,7 @@ impl ObjectInfo { } if self.is_compressed() { - if let Some(size_str) = self.user_defined.get(&format!("{RESERVED_METADATA_PREFIX_LOWER}actual-size")) + if let Some(size_str) = rustfs_utils::http::get_str(&self.user_defined, rustfs_utils::http::SUFFIX_ACTUAL_SIZE) && !size_str.is_empty() { // Todo: deal with error @@ -745,15 +738,11 @@ impl ObjectInfo { .user_defined .iter() .filter_map(|(k, v)| { - if k.starts_with(&format!("{RESERVED_METADATA_PREFIX_LOWER}{REPLICATION_RESET}")) { - Some(( - k.trim_start_matches(&format!("{RESERVED_METADATA_PREFIX_LOWER}{REPLICATION_RESET}-")) - .to_string(), - v.clone(), - )) - } else { - None - } + rustfs_utils::http::internal_key_strip_suffix_prefix( + k, + rustfs_utils::http::SUFFIX_REPLICATION_RESET_ARN_PREFIX, + ) + .map(|arn| (arn, v.clone())) }) .collect(), ..Default::default() @@ -1035,8 +1024,8 @@ mod tests { fn get_actual_size_uses_compressed_metadata_size() { let user_defined = { let mut map = HashMap::new(); - map.insert(format!("{RESERVED_METADATA_PREFIX_LOWER}compression"), "zstd".to_string()); - map.insert(format!("{RESERVED_METADATA_PREFIX_LOWER}actual-size"), "42".to_string()); + rustfs_utils::http::insert_str(&mut map, rustfs_utils::http::SUFFIX_COMPRESSION, "zstd".to_string()); + rustfs_utils::http::insert_str(&mut map, rustfs_utils::http::SUFFIX_ACTUAL_SIZE, "42".to_string()); map }; @@ -1072,7 +1061,7 @@ mod tests { fn get_actual_size_uses_compressed_parts_actual_size_when_metadata_missing() { let user_defined = { let mut map = HashMap::new(); - map.insert(format!("{RESERVED_METADATA_PREFIX_LOWER}compression"), "zstd".to_string()); + rustfs_utils::http::insert_str(&mut map, rustfs_utils::http::SUFFIX_COMPRESSION, "zstd".to_string()); map }; @@ -1100,7 +1089,7 @@ mod tests { fn get_actual_size_returns_error_when_compressed_parts_missing_and_size_mismatch() { let user_defined = { let mut map = HashMap::new(); - map.insert(format!("{RESERVED_METADATA_PREFIX_LOWER}compression"), "zstd".to_string()); + rustfs_utils::http::insert_str(&mut map, rustfs_utils::http::SUFFIX_COMPRESSION, "zstd".to_string()); map }; diff --git a/crates/ecstore/src/store_init.rs b/crates/ecstore/src/store_init.rs index 0d4303dea..6d49ec858 100644 --- a/crates/ecstore/src/store_init.rs +++ b/crates/ecstore/src/store_init.rs @@ -18,7 +18,7 @@ use crate::disk::{self, DiskAPI}; use crate::error::{Error, Result}; use crate::{ disk::{ - DiskInfoOptions, DiskOption, DiskStore, FORMAT_CONFIG_FILE, RUSTFS_META_BUCKET, + DiskInfoOptions, DiskOption, DiskStore, FORMAT_CONFIG_FILE, MIGRATING_META_BUCKET, RUSTFS_META_BUCKET, error::DiskError, format::{FormatErasureVersion, FormatMetaVersion, FormatV3}, new_disk, @@ -71,11 +71,13 @@ pub async fn connect_load_init_formats( check_format_erasure_values(&formats, set_drive_count)?; if first_disk && should_init_erasure_disks(&errs) { - // UnformattedDisk, not format file create + // UnformattedDisk, try migrate from MinIO format first, else create new format info!("first_disk && should_init_erasure_disks"); - // new format and save + if let Ok(fm) = try_migrate_format(disks, set_count, set_drive_count).await { + info!("Migrated format from MinIO config"); + return Ok(fm); + } let fm = init_format_erasure(disks, set_count, set_drive_count, deployment_id).await?; - return Ok(fm); } @@ -149,6 +151,60 @@ async fn init_format_erasure( get_format_erasure_in_quorum(&fms) } +/// Tries to migrate format +/// Returns Ok(FormatV3) if migration succeeds, Err otherwise. +async fn try_migrate_format(disks: &[Option], set_count: usize, set_drive_count: usize) -> Result { + for disk in disks.iter().flatten() { + let data = match disk.read_all(MIGRATING_META_BUCKET, FORMAT_CONFIG_FILE).await { + Ok(d) if !d.is_empty() => d, + _ => continue, + }; + + let fm = FormatV3::try_from(data.as_ref()).map_err(|e| Error::other(format!("parse MinIO format: {e}")))?; + + let first_set = fm + .erasure + .sets + .first() + .ok_or_else(|| Error::other("MinIO format: erasure.sets is empty"))?; + if fm.erasure.sets.len() != set_count || first_set.len() != set_drive_count { + debug!( + "MinIO format set count mismatch: got {}x{}, expected {}x{}", + fm.erasure.sets.len(), + first_set.len(), + set_count, + set_drive_count + ); + continue; + } + + if fm.erasure.version != FormatErasureVersion::V3 { + debug!("MinIO format erasure version not V3: {:?}", fm.erasure.version); + continue; + } + + let mut fms = vec![None; disks.len()]; + for (idx, disk_opt) in disks.iter().enumerate() { + if disk_opt.is_none() { + continue; + } + let set_idx = idx / set_drive_count; + let disk_idx = idx % set_drive_count; + if set_idx >= fm.erasure.sets.len() || disk_idx >= fm.erasure.sets[set_idx].len() { + continue; + } + let mut newfm = fm.clone(); + newfm.erasure.this = fm.erasure.sets[set_idx][disk_idx]; + fms[idx] = Some(newfm); + } + + save_format_file_all(disks, &fms).await?; + return get_format_erasure_in_quorum(&fms); + } + + Err(Error::other("no MinIO format to migrate")) +} + pub fn get_format_erasure_in_quorum(formats: &[Option]) -> Result { let mut countmap = HashMap::new(); diff --git a/crates/ecstore/src/store_list_objects.rs b/crates/ecstore/src/store_list_objects.rs index 1e06f5235..915670dc7 100644 --- a/crates/ecstore/src/store_list_objects.rs +++ b/crates/ecstore/src/store_list_objects.rs @@ -492,7 +492,7 @@ impl ECStore { } pub async fn list_path(self: Arc, o: &ListPathOptions) -> Result { - // warn!("list_path opt {:?}", &o); + // tracing::warn!("list_path opt {:?}", &o); check_list_objs_args(&o.bucket, &o.prefix, &o.marker)?; // if opts.prefix.ends_with(SLASH_SEPARATOR) { diff --git a/crates/ecstore/src/store_utils.rs b/crates/ecstore/src/store_utils.rs index d9fb10007..a39bca8f3 100644 --- a/crates/ecstore/src/store_utils.rs +++ b/crates/ecstore/src/store_utils.rs @@ -13,7 +13,7 @@ // limitations under the License. use crate::config::storageclass::STANDARD; -use crate::disk::RUSTFS_META_BUCKET; +use crate::disk::{MIGRATING_META_BUCKET, RUSTFS_META_BUCKET}; use regex::Regex; use rustfs_utils::http::headers::{AMZ_OBJECT_TAGGING, AMZ_STORAGE_CLASS}; use std::collections::HashMap; @@ -45,7 +45,7 @@ pub fn clean_metadata_keys(metadata: &mut HashMap, key_names: &[ // Check whether the bucket is the metadata bucket fn is_meta_bucket(bucket_name: &str) -> bool { - bucket_name == RUSTFS_META_BUCKET + bucket_name == RUSTFS_META_BUCKET || bucket_name == MIGRATING_META_BUCKET } // Check whether the bucket is reserved @@ -164,6 +164,8 @@ mod tests { fn test_meta_bucket_is_invalid() { assert!(is_reserved_or_invalid_bucket(RUSTFS_META_BUCKET, false)); assert!(is_reserved_or_invalid_bucket(RUSTFS_META_BUCKET, true)); + assert!(is_reserved_or_invalid_bucket(MIGRATING_META_BUCKET, false)); + assert!(is_reserved_or_invalid_bucket(MIGRATING_META_BUCKET, true)); } #[test] diff --git a/crates/ecstore/src/tier/tier.rs b/crates/ecstore/src/tier/tier.rs index d9284d7c0..93012baaf 100644 --- a/crates/ecstore/src/tier/tier.rs +++ b/crates/ecstore/src/tier/tier.rs @@ -18,14 +18,16 @@ #![allow(unused_must_use)] #![allow(clippy::all)] +use byteorder::{ByteOrder, LittleEndian}; use bytes::Bytes; +use http::HeaderMap; use http::status::StatusCode; use lazy_static::lazy_static; use rand::{Rng, RngExt}; use serde::{Deserialize, Serialize}; use std::{ collections::{HashMap, hash_map::Entry}, - io::Cursor, + io::{self, Cursor}, sync::Arc, time::Duration, }; @@ -46,9 +48,9 @@ use crate::tier::{ use crate::{ StorageAPI, config::com::{CONFIG_PREFIX, read_config}, - disk::RUSTFS_META_BUCKET, + disk::{MIGRATING_META_BUCKET, RUSTFS_META_BUCKET}, store::ECStore, - store_api::{ObjectOptions, PutObjReader}, + store_api::{ObjectIO as _, ObjectOptions, PutObjReader}, }; use rustfs_rio::HashReader; use rustfs_utils::path::{SLASH_SEPARATOR, path_join}; @@ -61,10 +63,17 @@ use super::{ const TIER_CFG_REFRESH: Duration = Duration::from_secs(15 * 60); -pub const TIER_CONFIG_FILE: &str = "tier-config.json"; +const TIER_CONFIG_LEGACY_FILE: &str = "tier-config.json"; +pub const TIER_CONFIG_FILE: &str = "tier-config.bin"; pub const TIER_CONFIG_FORMAT: u16 = 1; pub const TIER_CONFIG_V1: u16 = 1; -pub const TIER_CONFIG_VERSION: u16 = 1; +pub const TIER_CONFIG_VERSION: u16 = 2; + +const EXTERNAL_TIER_TYPE_UNSUPPORTED: i32 = 0; +const EXTERNAL_TIER_TYPE_S3: i32 = 1; +const EXTERNAL_TIER_TYPE_AZURE: i32 = 2; +const EXTERNAL_TIER_TYPE_GCS: i32 = 3; +const EXTERNAL_TIER_TYPE_MINIO: i32 = 4; const _TIER_CFG_REFRESH_AT_HDR: &str = "X-RustFS-TierCfg-RefreshedAt"; @@ -104,6 +113,609 @@ pub struct TierConfigMgr { pub last_refreshed_at: OffsetDateTime, } +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +#[serde(default)] +struct ExternalTierConfigMgr { + #[serde(rename = "Tiers")] + tiers: HashMap, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +#[serde(default)] +struct ExternalTierConfig { + #[serde(rename = "Version")] + version: String, + #[serde(rename = "Type")] + tier_type: i32, + #[serde(rename = "Name")] + name: String, + #[serde(rename = "S3")] + s3: Option, + #[serde(rename = "Azure")] + azure: Option, + #[serde(rename = "GCS")] + gcs: Option, + #[serde(rename = "MinIO", alias = "Compatible")] + compatible_backend: Option, + #[serde(rename = "XTierType", skip_serializing_if = "Option::is_none")] + tier_type_hint: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +#[serde(default)] +struct ExternalTierS3 { + #[serde(rename = "Endpoint")] + endpoint: String, + #[serde(rename = "AccessKey")] + access_key: String, + #[serde(rename = "SecretKey")] + secret_key: String, + #[serde(rename = "Bucket")] + bucket: String, + #[serde(rename = "Prefix")] + prefix: String, + #[serde(rename = "Region")] + region: String, + #[serde(rename = "StorageClass")] + storage_class: String, + #[serde(rename = "AWSRole")] + aws_role: bool, + #[serde(rename = "AWSRoleWebIdentityTokenFile")] + aws_role_web_identity_token_file: String, + #[serde(rename = "AWSRoleARN")] + aws_role_arn: String, + #[serde(rename = "AWSRoleSessionName")] + aws_role_session_name: String, + #[serde(rename = "AWSRoleDurationSeconds")] + aws_role_duration_seconds: i32, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +#[serde(default)] +struct ExternalServicePrincipalAuth { + #[serde(rename = "TenantID")] + tenant_id: String, + #[serde(rename = "ClientID")] + client_id: String, + #[serde(rename = "ClientSecret")] + client_secret: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +#[serde(default)] +struct ExternalTierAzure { + #[serde(rename = "Endpoint")] + endpoint: String, + #[serde(rename = "AccountName")] + account_name: String, + #[serde(rename = "AccountKey")] + account_key: String, + #[serde(rename = "Bucket")] + bucket: String, + #[serde(rename = "Prefix")] + prefix: String, + #[serde(rename = "Region")] + region: String, + #[serde(rename = "StorageClass")] + storage_class: String, + #[serde(rename = "SPAuth")] + sp_auth: ExternalServicePrincipalAuth, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +#[serde(default)] +struct ExternalTierGcs { + #[serde(rename = "Endpoint")] + endpoint: String, + #[serde(rename = "Creds")] + creds: String, + #[serde(rename = "Bucket")] + bucket: String, + #[serde(rename = "Prefix")] + prefix: String, + #[serde(rename = "Region")] + region: String, + #[serde(rename = "StorageClass")] + storage_class: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +#[serde(default)] +struct ExternalTierCompatible { + #[serde(rename = "Endpoint")] + endpoint: String, + #[serde(rename = "AccessKey")] + access_key: String, + #[serde(rename = "SecretKey")] + secret_key: String, + #[serde(rename = "Bucket")] + bucket: String, + #[serde(rename = "Prefix")] + prefix: String, + #[serde(rename = "Region")] + region: String, +} + +fn tier_config_path(file: &str) -> String { + format!("{}{}{}", CONFIG_PREFIX, SLASH_SEPARATOR, file) +} + +fn tier_hint_for_type(tier_type: TierType) -> Option<&'static str> { + match tier_type { + TierType::RustFS => Some("rustfs"), + TierType::Aliyun => Some("aliyun"), + TierType::Tencent => Some("tencent"), + TierType::Huaweicloud => Some("huaweicloud"), + TierType::R2 => Some("r2"), + _ => None, + } +} + +fn tier_type_from_hint(hint: Option<&str>) -> Option { + match hint { + Some("rustfs") => Some(TierType::RustFS), + Some("aliyun") => Some(TierType::Aliyun), + Some("tencent") => Some(TierType::Tencent), + Some("huaweicloud") => Some(TierType::Huaweicloud), + Some("r2") => Some(TierType::R2), + _ => None, + } +} + +fn external_tier_s3_from_internal(s3: &crate::tier::tier_config::TierS3) -> ExternalTierS3 { + ExternalTierS3 { + endpoint: s3.endpoint.clone(), + access_key: s3.access_key.clone(), + secret_key: s3.secret_key.clone(), + bucket: s3.bucket.clone(), + prefix: s3.prefix.clone(), + region: s3.region.clone(), + storage_class: s3.storage_class.clone(), + aws_role: s3.aws_role, + aws_role_web_identity_token_file: s3.aws_role_web_identity_token_file.clone(), + aws_role_arn: s3.aws_role_arn.clone(), + aws_role_session_name: s3.aws_role_session_name.clone(), + aws_role_duration_seconds: s3.aws_role_duration_seconds, + } +} + +fn external_tier_s3_from_compatible_payload( + endpoint: String, + access_key: String, + secret_key: String, + bucket: String, + prefix: String, + region: String, +) -> ExternalTierS3 { + ExternalTierS3 { + endpoint, + access_key, + secret_key, + bucket, + prefix, + region, + storage_class: String::new(), + aws_role: false, + aws_role_web_identity_token_file: String::new(), + aws_role_arn: String::new(), + aws_role_session_name: String::new(), + aws_role_duration_seconds: 0, + } +} + +fn external_tier_alias_from_compatible_payload( + endpoint: String, + access_key: String, + secret_key: String, + bucket: String, + prefix: String, + region: String, +) -> ExternalTierCompatible { + ExternalTierCompatible { + endpoint, + access_key, + secret_key, + bucket, + prefix, + region, + } +} + +fn to_external_tier_config(name: &str, tier: &TierConfig) -> io::Result { + let mut out = ExternalTierConfig { + version: if tier.version.is_empty() { + "v1".to_string() + } else { + tier.version.clone() + }, + name: if tier.name.is_empty() { + name.to_string() + } else { + tier.name.clone() + }, + ..Default::default() + }; + + match tier.tier_type { + TierType::S3 => { + let s3 = tier + .s3 + .as_ref() + .ok_or_else(|| io::Error::other("tier config missing s3 backend payload"))?; + out.tier_type = EXTERNAL_TIER_TYPE_S3; + out.s3 = Some(external_tier_s3_from_internal(s3)); + } + TierType::Azure => { + let az = tier + .azure + .as_ref() + .ok_or_else(|| io::Error::other("tier config missing azure backend payload"))?; + out.tier_type = EXTERNAL_TIER_TYPE_AZURE; + out.azure = Some(ExternalTierAzure { + endpoint: az.endpoint.clone(), + account_name: az.access_key.clone(), + account_key: az.secret_key.clone(), + bucket: az.bucket.clone(), + prefix: az.prefix.clone(), + region: az.region.clone(), + storage_class: az.storage_class.clone(), + sp_auth: ExternalServicePrincipalAuth { + tenant_id: az.sp_auth.tenant_id.clone(), + client_id: az.sp_auth.client_id.clone(), + client_secret: az.sp_auth.client_secret.clone(), + }, + }); + } + TierType::GCS => { + let gcs = tier + .gcs + .as_ref() + .ok_or_else(|| io::Error::other("tier config missing gcs backend payload"))?; + out.tier_type = EXTERNAL_TIER_TYPE_GCS; + out.gcs = Some(ExternalTierGcs { + endpoint: gcs.endpoint.clone(), + creds: gcs.creds.clone(), + bucket: gcs.bucket.clone(), + prefix: gcs.prefix.clone(), + region: gcs.region.clone(), + storage_class: gcs.storage_class.clone(), + }); + } + TierType::MinIO => { + let backend = tier + .minio + .as_ref() + .ok_or_else(|| io::Error::other("tier config missing compatible backend payload"))?; + out.tier_type = EXTERNAL_TIER_TYPE_MINIO; + out.compatible_backend = Some(external_tier_alias_from_compatible_payload( + backend.endpoint.clone(), + backend.access_key.clone(), + backend.secret_key.clone(), + backend.bucket.clone(), + backend.prefix.clone(), + backend.region.clone(), + )); + } + TierType::RustFS => { + let backend = tier + .rustfs + .as_ref() + .ok_or_else(|| io::Error::other("tier config missing compatible backend payload"))?; + out.tier_type = EXTERNAL_TIER_TYPE_S3; + out.tier_type_hint = tier_hint_for_type(tier.tier_type.clone()).map(ToString::to_string); + out.s3 = Some(external_tier_s3_from_compatible_payload( + backend.endpoint.clone(), + backend.access_key.clone(), + backend.secret_key.clone(), + backend.bucket.clone(), + backend.prefix.clone(), + backend.region.clone(), + )); + } + TierType::Aliyun => { + let backend = tier + .aliyun + .as_ref() + .ok_or_else(|| io::Error::other("tier config missing compatible backend payload"))?; + out.tier_type = EXTERNAL_TIER_TYPE_S3; + out.tier_type_hint = tier_hint_for_type(tier.tier_type.clone()).map(ToString::to_string); + out.s3 = Some(external_tier_s3_from_compatible_payload( + backend.endpoint.clone(), + backend.access_key.clone(), + backend.secret_key.clone(), + backend.bucket.clone(), + backend.prefix.clone(), + backend.region.clone(), + )); + } + TierType::Tencent => { + let backend = tier + .tencent + .as_ref() + .ok_or_else(|| io::Error::other("tier config missing compatible backend payload"))?; + out.tier_type = EXTERNAL_TIER_TYPE_S3; + out.tier_type_hint = tier_hint_for_type(tier.tier_type.clone()).map(ToString::to_string); + out.s3 = Some(external_tier_s3_from_compatible_payload( + backend.endpoint.clone(), + backend.access_key.clone(), + backend.secret_key.clone(), + backend.bucket.clone(), + backend.prefix.clone(), + backend.region.clone(), + )); + } + TierType::Huaweicloud => { + let backend = tier + .huaweicloud + .as_ref() + .ok_or_else(|| io::Error::other("tier config missing compatible backend payload"))?; + out.tier_type = EXTERNAL_TIER_TYPE_S3; + out.tier_type_hint = tier_hint_for_type(tier.tier_type.clone()).map(ToString::to_string); + out.s3 = Some(external_tier_s3_from_compatible_payload( + backend.endpoint.clone(), + backend.access_key.clone(), + backend.secret_key.clone(), + backend.bucket.clone(), + backend.prefix.clone(), + backend.region.clone(), + )); + } + TierType::R2 => { + let backend = tier + .r2 + .as_ref() + .ok_or_else(|| io::Error::other("tier config missing compatible backend payload"))?; + out.tier_type = EXTERNAL_TIER_TYPE_S3; + out.tier_type_hint = tier_hint_for_type(tier.tier_type.clone()).map(ToString::to_string); + out.s3 = Some(external_tier_s3_from_compatible_payload( + backend.endpoint.clone(), + backend.access_key.clone(), + backend.secret_key.clone(), + backend.bucket.clone(), + backend.prefix.clone(), + backend.region.clone(), + )); + } + TierType::Unsupported => { + out.tier_type = EXTERNAL_TIER_TYPE_UNSUPPORTED; + } + } + Ok(out) +} + +fn decode_legacy_s3_like(name: &str, ext: &ExternalTierConfig) -> io::Result { + if let Some(s3) = ext.s3.as_ref() { + return Ok(s3.clone()); + } + if let Some(m) = ext.compatible_backend.as_ref() { + return Ok(external_tier_s3_from_compatible_payload( + m.endpoint.clone(), + m.access_key.clone(), + m.secret_key.clone(), + m.bucket.clone(), + m.prefix.clone(), + m.region.clone(), + )); + } + Err(io::Error::other(format!("tier config '{name}' missing compatible backend payload"))) +} + +fn from_external_tier_config(name: String, ext: ExternalTierConfig) -> io::Result { + let mut cfg = TierConfig { + version: if ext.version.is_empty() { + "v1".to_string() + } else { + ext.version.clone() + }, + name: if ext.name.is_empty() { name.clone() } else { ext.name.clone() }, + ..Default::default() + }; + + let hinted = tier_type_from_hint(ext.tier_type_hint.as_deref()); + let tier_type = if let Some(h) = hinted { + h + } else { + match ext.tier_type { + EXTERNAL_TIER_TYPE_S3 => TierType::S3, + EXTERNAL_TIER_TYPE_AZURE => TierType::Azure, + EXTERNAL_TIER_TYPE_GCS => TierType::GCS, + EXTERNAL_TIER_TYPE_MINIO => TierType::MinIO, + _ => TierType::Unsupported, + } + }; + + cfg.tier_type = tier_type.clone(); + + match tier_type { + TierType::S3 => { + let s3 = ext + .s3 + .as_ref() + .ok_or_else(|| io::Error::other(format!("tier config '{}' missing s3 backend payload", cfg.name)))?; + cfg.s3 = Some(crate::tier::tier_config::TierS3 { + name: cfg.name.clone(), + endpoint: s3.endpoint.clone(), + access_key: s3.access_key.clone(), + secret_key: s3.secret_key.clone(), + bucket: s3.bucket.clone(), + prefix: s3.prefix.clone(), + region: s3.region.clone(), + storage_class: s3.storage_class.clone(), + aws_role: s3.aws_role, + aws_role_web_identity_token_file: s3.aws_role_web_identity_token_file.clone(), + aws_role_arn: s3.aws_role_arn.clone(), + aws_role_session_name: s3.aws_role_session_name.clone(), + aws_role_duration_seconds: s3.aws_role_duration_seconds, + }); + } + TierType::Azure => { + let az = ext + .azure + .as_ref() + .ok_or_else(|| io::Error::other(format!("tier config '{}' missing azure backend payload", cfg.name)))?; + cfg.azure = Some(crate::tier::tier_config::TierAzure { + name: cfg.name.clone(), + endpoint: az.endpoint.clone(), + access_key: az.account_name.clone(), + secret_key: az.account_key.clone(), + bucket: az.bucket.clone(), + prefix: az.prefix.clone(), + region: az.region.clone(), + storage_class: az.storage_class.clone(), + sp_auth: crate::tier::tier_config::ServicePrincipalAuth { + tenant_id: az.sp_auth.tenant_id.clone(), + client_id: az.sp_auth.client_id.clone(), + client_secret: az.sp_auth.client_secret.clone(), + }, + }); + } + TierType::GCS => { + let gcs = ext + .gcs + .as_ref() + .ok_or_else(|| io::Error::other(format!("tier config '{}' missing gcs backend payload", cfg.name)))?; + cfg.gcs = Some(crate::tier::tier_config::TierGCS { + name: cfg.name.clone(), + endpoint: gcs.endpoint.clone(), + creds: gcs.creds.clone(), + bucket: gcs.bucket.clone(), + prefix: gcs.prefix.clone(), + region: gcs.region.clone(), + storage_class: gcs.storage_class.clone(), + }); + } + TierType::MinIO => { + let m = ext + .compatible_backend + .as_ref() + .ok_or_else(|| io::Error::other(format!("tier config '{}' missing compatible backend payload", cfg.name)))?; + cfg.minio = Some(crate::tier::tier_config::TierMinIO { + name: cfg.name.clone(), + endpoint: m.endpoint.clone(), + access_key: m.access_key.clone(), + secret_key: m.secret_key.clone(), + bucket: m.bucket.clone(), + prefix: m.prefix.clone(), + region: m.region.clone(), + }); + } + TierType::RustFS => { + let m = decode_legacy_s3_like(&cfg.name, &ext)?; + cfg.rustfs = Some(crate::tier::tier_config::TierRustFS { + name: cfg.name.clone(), + endpoint: m.endpoint, + access_key: m.access_key, + secret_key: m.secret_key, + bucket: m.bucket, + prefix: m.prefix, + region: m.region, + storage_class: m.storage_class, + }); + } + TierType::Aliyun => { + let m = decode_legacy_s3_like(&cfg.name, &ext)?; + cfg.aliyun = Some(crate::tier::tier_config::TierAliyun { + name: cfg.name.clone(), + endpoint: m.endpoint, + access_key: m.access_key, + secret_key: m.secret_key, + bucket: m.bucket, + prefix: m.prefix, + region: m.region, + }); + } + TierType::Tencent => { + let m = decode_legacy_s3_like(&cfg.name, &ext)?; + cfg.tencent = Some(crate::tier::tier_config::TierTencent { + name: cfg.name.clone(), + endpoint: m.endpoint, + access_key: m.access_key, + secret_key: m.secret_key, + bucket: m.bucket, + prefix: m.prefix, + region: m.region, + }); + } + TierType::Huaweicloud => { + let m = decode_legacy_s3_like(&cfg.name, &ext)?; + cfg.huaweicloud = Some(crate::tier::tier_config::TierHuaweicloud { + name: cfg.name.clone(), + endpoint: m.endpoint, + access_key: m.access_key, + secret_key: m.secret_key, + bucket: m.bucket, + prefix: m.prefix, + region: m.region, + }); + } + TierType::R2 => { + let m = decode_legacy_s3_like(&cfg.name, &ext)?; + cfg.r2 = Some(crate::tier::tier_config::TierR2 { + name: cfg.name.clone(), + endpoint: m.endpoint, + access_key: m.access_key, + secret_key: m.secret_key, + bucket: m.bucket, + prefix: m.prefix, + region: m.region, + }); + } + TierType::Unsupported => {} + } + Ok(cfg) +} + +fn encode_external_tiering_config_blob(cfg: &TierConfigMgr) -> io::Result { + let mut tiers = HashMap::with_capacity(cfg.tiers.len()); + for (name, tier_cfg) in &cfg.tiers { + tiers.insert(name.clone(), to_external_tier_config(name, tier_cfg)?); + } + let payload = rmp_serde::to_vec(&ExternalTierConfigMgr { tiers }) + .map_err(|err| io::Error::other(format!("serialize tier config payload failed: {err}")))?; + let mut data = Vec::with_capacity(4 + payload.len()); + let mut format = [0u8; 2]; + LittleEndian::write_u16(&mut format, TIER_CONFIG_FORMAT); + data.extend_from_slice(&format); + let mut version = [0u8; 2]; + LittleEndian::write_u16(&mut version, TIER_CONFIG_VERSION); + data.extend_from_slice(&version); + data.extend_from_slice(&payload); + Ok(Bytes::from(data)) +} + +fn decode_external_tiering_config_blob(data: &[u8]) -> io::Result { + if data.len() <= 4 { + return Err(io::Error::other("tierConfigInit: no data")); + } + let format = LittleEndian::read_u16(&data[0..2]); + if format != TIER_CONFIG_FORMAT { + return Err(io::Error::other(format!("tierConfigInit: unknown format: {format}"))); + } + let version = LittleEndian::read_u16(&data[2..4]); + if version != TIER_CONFIG_V1 && version != TIER_CONFIG_VERSION { + return Err(io::Error::other(format!("tierConfigInit: unknown version: {version}"))); + } + + let external: ExternalTierConfigMgr = + rmp_serde::from_slice(&data[4..]).map_err(|err| io::Error::other(format!("decode tier config payload failed: {err}")))?; + let mut tiers = HashMap::with_capacity(external.tiers.len()); + for (name, ext_cfg) in external.tiers { + tiers.insert(name.clone(), from_external_tier_config(name, ext_cfg)?); + } + Ok(TierConfigMgr { + driver_cache: HashMap::new(), + tiers, + last_refreshed_at: OffsetDateTime::now_utc(), + }) +} + +fn decode_tiering_config_blob(data: &[u8]) -> io::Result { + if let Ok(cfg) = TierConfigMgr::unmarshal(data) { + return Ok(cfg); + } + decode_external_tiering_config_blob(data) +} + impl TierConfigMgr { pub fn new() -> Arc> { Arc::new(RwLock::new(Self { @@ -285,12 +897,12 @@ impl TierConfigMgr { rustfs.secret_key = creds.secret_key; } TierType::MinIO => { - let mut minio = tier_config.minio.as_mut().expect("err"); + let compatible_backend = tier_config.minio.as_mut().expect("err"); if creds.access_key == "" || creds.secret_key == "" { return Err(ERR_TIER_MISSING_CREDENTIALS.clone()); } - minio.access_key = creds.access_key; - minio.secret_key = creds.secret_key; + compatible_backend.access_key = creds.access_key; + compatible_backend.secret_key = creds.secret_key; } TierType::Aliyun => { let mut aliyun = tier_config.aliyun.as_mut().expect("err"); @@ -401,9 +1013,8 @@ impl TierConfigMgr { } pub async fn save_tiering_config(&self, api: Arc) -> std::result::Result<(), std::io::Error> { - let data = self.marshal()?; - - let config_file = format!("{}{}{}", CONFIG_PREFIX, SLASH_SEPARATOR, TIER_CONFIG_FILE); + let data = encode_external_tiering_config_blob(self)?; + let config_file = tier_config_path(TIER_CONFIG_FILE); self.save_config(api, &config_file, data).await } @@ -483,38 +1094,229 @@ async fn new_and_save_tiering_config(api: Arc) -> Result) -> std::result::Result { - let config_file = format!("{}{}{}", CONFIG_PREFIX, SLASH_SEPARATOR, TIER_CONFIG_FILE); - let data = read_config(api.clone(), config_file.as_str()).await; - if let Err(err) = data { - if is_err_config_not_found(&err) { - warn!("config not found, start to init"); - let cfg = new_and_save_tiering_config(api).await?; - return Ok(cfg); - } else { - error!("read config err {:?}", &err); - return Err(std::io::Error::other(err)); - } - } - - let cfg; - let version = 1; //LittleEndian::read_u16(&data[2..4]); - match version { - TIER_CONFIG_V1/* | TIER_CONFIG_VERSION */ => { - cfg = match TierConfigMgr::unmarshal(&data.unwrap()) { - Ok(cfg) => cfg, - Err(err) => { - return Err(std::io::Error::other(err.to_string())); + let config_file = tier_config_path(TIER_CONFIG_FILE); + match read_config(api.clone(), config_file.as_str()).await { + Ok(data) => decode_tiering_config_blob(&data), + Err(err) if is_err_config_not_found(&err) => { + let legacy_file = tier_config_path(TIER_CONFIG_LEGACY_FILE); + match read_config(api.clone(), legacy_file.as_str()).await { + Ok(data) => { + let cfg = TierConfigMgr::unmarshal(&data)?; + let normalized = encode_external_tiering_config_blob(&cfg)?; + let _ = api + .put_object( + RUSTFS_META_BUCKET, + &config_file, + &mut PutObjReader::from_vec(normalized.to_vec()), + &ObjectOptions { + max_parity: true, + ..Default::default() + }, + ) + .await; + Ok(cfg) } - }; + Err(legacy_err) if is_err_config_not_found(&legacy_err) => { + warn!("config not found, start to init"); + new_and_save_tiering_config(api).await.map_err(io::Error::other) + } + Err(legacy_err) => Err(io::Error::other(legacy_err)), + } } - _ => { - return Err(std::io::Error::other(format!("tierConfigInit: unknown version: {}", version))); + Err(err) => { + error!("read config err {:?}", &err); + Err(io::Error::other(err)) } } +} - Ok(cfg) +async fn read_tier_config_from_bucket( + api: Arc, + bucket: &str, + path: &str, + opts: &ObjectOptions, +) -> io::Result>> { + let mut rd = match api.get_object_reader(bucket, path, None, HeaderMap::new(), opts).await { + Ok(v) => v, + Err(err) if is_err_config_not_found(&err) => return Ok(None), + Err(err) => return Err(io::Error::other(err)), + }; + let data = rd.read_all().await.map_err(io::Error::other)?; + if data.is_empty() { + return Ok(None); + } + Ok(Some(data)) +} + +async fn write_tier_config_to_rustfs(api: Arc, path: &str, data: Bytes) -> io::Result<()> { + api.put_object( + RUSTFS_META_BUCKET, + path, + &mut PutObjReader::from_vec(data.to_vec()), + &ObjectOptions { + max_parity: true, + ..Default::default() + }, + ) + .await + .map_err(io::Error::other)?; + Ok(()) +} + +pub async fn try_migrate_tiering_config(api: Arc) { + let target_path = tier_config_path(TIER_CONFIG_FILE); + if api + .get_object_info(RUSTFS_META_BUCKET, &target_path, &ObjectOptions::default()) + .await + .is_ok() + { + debug!("tier config already exists in RustFS metadata bucket, skip migration"); + return; + } + + let opts = ObjectOptions { + max_parity: true, + no_lock: true, + ..Default::default() + }; + + let legacy_path = tier_config_path(TIER_CONFIG_LEGACY_FILE); + match read_tier_config_from_bucket(api.clone(), RUSTFS_META_BUCKET, &legacy_path, &opts).await { + Ok(Some(data)) => match TierConfigMgr::unmarshal(&data) + .and_then(|cfg| encode_external_tiering_config_blob(&cfg).map_err(io::Error::other)) + { + Ok(out) => { + if write_tier_config_to_rustfs(api.clone(), &target_path, out).await.is_ok() { + info!("Migrated tier config from legacy RustFS metadata format"); + return; + } + } + Err(err) => warn!("legacy tier config is incompatible, skip local migration: {}", err), + }, + Ok(None) => {} + Err(err) => warn!("read legacy local tier config failed: {}", err), + } + + match read_tier_config_from_bucket(api.clone(), MIGRATING_META_BUCKET, &target_path, &opts).await { + Ok(Some(data)) => match decode_tiering_config_blob(&data).and_then(|cfg| encode_external_tiering_config_blob(&cfg)) { + Ok(out) => { + if write_tier_config_to_rustfs(api.clone(), &target_path, out).await.is_ok() { + info!("Migrated compatible tier config from migrating metadata bucket"); + } + } + Err(err) => warn!("migrating tier config is incompatible, skip migration: {}", err), + }, + Ok(None) => {} + Err(err) => warn!("read migrating tier config failed: {}", err), + } } pub fn is_err_config_not_found(err: &StorageError) -> bool { - matches!(err, StorageError::ObjectNotFound(_, _)) || err == &StorageError::ConfigNotFound + matches!(err, StorageError::ObjectNotFound(_, _) | StorageError::BucketNotFound(_)) || err == &StorageError::ConfigNotFound +} + +#[cfg(test)] +mod tests { + use super::*; + + fn build_s3_tier(name: &str) -> TierConfig { + TierConfig { + version: "v1".to_string(), + tier_type: TierType::S3, + name: name.to_string(), + s3: Some(crate::tier::tier_config::TierS3 { + name: name.to_string(), + endpoint: "https://example-s3.invalid".to_string(), + access_key: "ak".to_string(), + secret_key: "sk".to_string(), + bucket: "bucket-a".to_string(), + prefix: "prefix-a".to_string(), + region: "us-east-1".to_string(), + storage_class: "STANDARD".to_string(), + aws_role: false, + aws_role_web_identity_token_file: String::new(), + aws_role_arn: String::new(), + aws_role_session_name: String::new(), + aws_role_duration_seconds: 0, + }), + ..Default::default() + } + } + + #[test] + fn test_tiering_external_blob_roundtrip_for_standard_type() { + let mut cfg = TierConfigMgr { + driver_cache: HashMap::new(), + tiers: HashMap::new(), + last_refreshed_at: OffsetDateTime::now_utc(), + }; + cfg.tiers.insert("COLD-A".to_string(), build_s3_tier("COLD-A")); + + let bytes = encode_external_tiering_config_blob(&cfg).expect("encode should succeed"); + assert_eq!(&bytes[0..2], &TIER_CONFIG_FORMAT.to_le_bytes()); + assert_eq!(&bytes[2..4], &TIER_CONFIG_VERSION.to_le_bytes()); + + let decoded = decode_external_tiering_config_blob(&bytes).expect("decode should succeed"); + let tier = decoded.tiers.get("COLD-A").expect("tier should exist"); + assert_eq!(tier.tier_type.as_lowercase(), "s3"); + assert_eq!(tier.s3.as_ref().expect("s3 should exist").endpoint, "https://example-s3.invalid"); + } + + #[test] + fn test_tiering_external_blob_roundtrip_for_extended_type_hint() { + let mut cfg = TierConfigMgr { + driver_cache: HashMap::new(), + tiers: HashMap::new(), + last_refreshed_at: OffsetDateTime::now_utc(), + }; + cfg.tiers.insert( + "COLD-B".to_string(), + TierConfig { + version: "v1".to_string(), + tier_type: TierType::RustFS, + name: "COLD-B".to_string(), + rustfs: Some(crate::tier::tier_config::TierRustFS { + name: "COLD-B".to_string(), + endpoint: "https://example-compat.invalid".to_string(), + access_key: "ak".to_string(), + secret_key: "sk".to_string(), + bucket: "bucket-b".to_string(), + prefix: "prefix-b".to_string(), + region: "us-east-1".to_string(), + storage_class: "STANDARD".to_string(), + }), + ..Default::default() + }, + ); + + let bytes = encode_external_tiering_config_blob(&cfg).expect("encode should succeed"); + let decoded = decode_external_tiering_config_blob(&bytes).expect("decode should succeed"); + let tier = decoded.tiers.get("COLD-B").expect("tier should exist"); + assert_eq!(tier.tier_type.as_lowercase(), "rustfs"); + assert_eq!( + tier.rustfs.as_ref().expect("backend should exist").endpoint, + "https://example-compat.invalid" + ); + } + + #[test] + fn test_decode_tiering_config_blob_accepts_legacy_json() { + let mut cfg = TierConfigMgr { + driver_cache: HashMap::new(), + tiers: HashMap::new(), + last_refreshed_at: OffsetDateTime::now_utc(), + }; + cfg.tiers.insert("COLD-A".to_string(), build_s3_tier("COLD-A")); + + let data = serde_json::to_vec(&cfg).expect("legacy json should encode"); + let decoded = decode_tiering_config_blob(&data).expect("legacy json should decode"); + assert_eq!( + decoded + .tiers + .get("COLD-A") + .and_then(|tier| tier.s3.as_ref()) + .map(|s3| s3.bucket.as_str()), + Some("bucket-a") + ); + } } diff --git a/crates/ecstore/src/tier/tier_config.rs b/crates/ecstore/src/tier/tier_config.rs index 2646d7d6b..74cc38e27 100644 --- a/crates/ecstore/src/tier/tier_config.rs +++ b/crates/ecstore/src/tier/tier_config.rs @@ -146,7 +146,7 @@ impl Clone for TierConfig { fn clone(&self) -> TierConfig { let mut s3 = None; let mut r = None; - let mut m = None; + let mut compatible_backend = None; let mut aliyun = None; let mut tencent = None; let mut huaweicloud = None; @@ -165,9 +165,9 @@ impl Clone for TierConfig { r = Some(r_); } TierType::MinIO => { - let mut m_ = self.minio.as_ref().expect("err").clone(); - m_.secret_key = "REDACTED".to_string(); - m = Some(m_); + let mut compatible_backend_ = self.minio.as_ref().expect("err").clone(); + compatible_backend_.secret_key = "REDACTED".to_string(); + compatible_backend = Some(compatible_backend_); } TierType::Aliyun => { let mut aliyun_ = self.aliyun.as_ref().expect("err").clone(); @@ -207,7 +207,7 @@ impl Clone for TierConfig { name: self.name.clone(), s3, rustfs: r, - minio: m, + minio: compatible_backend, aliyun, tencent, huaweicloud, @@ -408,7 +408,7 @@ impl TierMinIO { if name.is_empty() { return Err(std::io::Error::other(ERR_TIER_NAME_EMPTY)); } - let m = TierMinIO { + let backend = TierMinIO { access_key: access_key.to_string(), secret_key: secret_key.to_string(), bucket: bucket.to_string(), @@ -417,7 +417,7 @@ impl TierMinIO { }; for option in options { - let option = option(m.clone()); + let option = option(backend.clone()); let option = *option; option?; } @@ -426,7 +426,7 @@ impl TierMinIO { version: C_TIER_CONFIG_VER.to_string(), tier_type: TierType::MinIO, name: name.to_string(), - minio: Some(m), + minio: Some(backend), ..Default::default() }) } diff --git a/crates/ecstore/tests/legacy_bitrot_read_test.rs b/crates/ecstore/tests/legacy_bitrot_read_test.rs new file mode 100644 index 000000000..04d62121e --- /dev/null +++ b/crates/ecstore/tests/legacy_bitrot_read_test.rs @@ -0,0 +1,238 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Integration test: verify create_bitrot_reader with HighwayHash256SLegacy reads main-version objects. +//! +//! Supports both EC (part files) and inline objects. Reads xl.meta to compute params. +//! Uses create_bitrot_reader only (no ECStore/get_object_reader) to isolate bitrot compatibility. +//! +//! Run from workspace root: +//! cargo test -p rustfs-ecstore test_legacy_bitrot_read -- --nocapture +//! +//! Environment: +//! RUSTFS_LEGACY_TEST_ROOT - Test data root (default: workspace root) +//! RUSTFS_LEGACY_TEST_DISK - Disk name under root (default: "test1" for rustfs, "test" for minio) +//! RUSTFS_SKIP_LEGACY_TEST - Set to 1 to skip +//! +//! For MinIO data: RUSTFS_LEGACY_TEST_ROOT=/path/to/minio (disk "test" and .minio.sys auto-detected). + +use rustfs_ecstore::bitrot::create_bitrot_reader; +use rustfs_ecstore::disk::endpoint::Endpoint; +use rustfs_ecstore::disk::{DiskOption, STORAGE_FORMAT_FILE, new_disk}; +use rustfs_filemeta::{FileInfoOpts, get_file_info}; +use rustfs_utils::HashAlgorithm; +use std::path::PathBuf; +use tokio::fs; + +fn workspace_root() -> PathBuf { + let manifest = std::env::var("CARGO_MANIFEST_DIR").unwrap_or_else(|_| ".".into()); + PathBuf::from(&manifest) + .ancestors() + .nth(2) + .unwrap_or(std::path::Path::new(".")) + .to_path_buf() +} + +fn legacy_test_data_exists() -> bool { + if std::env::var("RUSTFS_SKIP_LEGACY_TEST").unwrap_or_default() == "1" { + return false; + } + let root = workspace_root(); + let format_path = root.join("test1/.rustfs.sys/format.json"); + let ktvzip_meta = root.join("test1/vvvv/ktvzip.tar.gz").join(STORAGE_FORMAT_FILE); + let path_traversal_meta = root.join("test1/vvvv/path_traversal.md").join(STORAGE_FORMAT_FILE); + format_path.exists() && (ktvzip_meta.exists() || path_traversal_meta.exists()) +} + +async fn run_legacy_bitrot_test_for_object(root: &std::path::Path, disk_name: &str, bucket: &str, object: &str) -> bool { + let xl_meta_path = root.join(disk_name).join(bucket).join(object).join(STORAGE_FORMAT_FILE); + if !xl_meta_path.exists() { + eprintln!("xl_meta_path not found: {:?}", xl_meta_path); + return false; + } + + let buf = match fs::read(&xl_meta_path).await { + Ok(b) => b, + Err(_) => { + eprintln!("Failed to read xl_meta_path: {:?}", xl_meta_path); + return false; + } + }; + + let fi = match get_file_info( + &buf, + bucket, + object, + "", + FileInfoOpts { + data: true, // need inline data for inline objects + include_free_versions: false, + }, + ) { + Ok(f) => f, + Err(_) => { + eprintln!("Failed to get file info: {:?}", xl_meta_path); + return false; + } + }; + + if fi.deleted || fi.parts.is_empty() { + eprintln!("File is deleted or has no parts: {:?}", xl_meta_path); + return false; + } + + let shard_size = fi.erasure.shard_size(); + let part_number = 1; + let checksum_info = fi.erasure.get_checksum_info(part_number); + let checksum_algo = if fi.uses_legacy_checksum && checksum_info.algorithm == HashAlgorithm::HighwayHash256S { + HashAlgorithm::HighwayHash256SLegacy + } else { + checksum_info.algorithm + }; + + eprintln!("fi.inline_data(): {:?}", fi.inline_data()); + eprintln!("fi.is_remote(): {:?}", fi.metadata); + + // Inline path: use fi.data + if fi.inline_data() { + let inline_bytes = match &fi.data { + Some(b) if !b.is_empty() => b.as_ref(), + _ => { + eprintln!("Inline data is empty: {:?}", xl_meta_path); + return false; + } + }; + let read_length = fi.size as usize; + if read_length == 0 { + eprintln!("Read length is 0: {:?}", xl_meta_path); + return false; + } + + let mut reader = match create_bitrot_reader( + Some(inline_bytes), + None, + bucket, + "", + 0, + read_length, + shard_size, + checksum_algo.clone(), + false, + ) + .await + { + Ok(Some(r)) => r, + _ => { + eprintln!("Failed to create bitrot reader for inline data: {:?}", xl_meta_path); + return false; + } + }; + + let mut buf = vec![0u8; shard_size]; + match reader.read(&mut buf).await { + Ok(n) if n > 0 => { + eprintln!("Successfully read {} bytes (inline) via create_bitrot_reader with {:?}", n, checksum_algo); + return true; + } + _ => { + eprintln!("Failed to read inline data: {:?}", xl_meta_path); + return false; + } + } + } + + // EC path: use disk + part file + let data_dir = match fi.data_dir { + Some(d) => d, + None => { + eprintln!("Data dir is empty: {:?}", xl_meta_path); + return false; + } + }; + let path = format!("{object}/{data_dir}/part.{}", part_number); + let part_path = root.join(disk_name).join(bucket).join(&path); + if !part_path.exists() { + eprintln!("Part file not found: {:?}", part_path); + return false; + } + + let disk_path = root.join(disk_name); + let path_str = disk_path.to_str().expect("path"); + let mut ep = Endpoint::try_from(path_str).expect("endpoint"); + ep.set_pool_index(0); + ep.set_set_index(0); + ep.set_disk_index(0); + let opt = DiskOption { + cleanup: false, + health_check: false, + }; + let disk = match new_disk(&ep, &opt).await { + Ok(d) => d, + Err(_) => { + eprintln!("Failed to create disk: {:?}", disk_path); + return false; + } + }; + + let read_length = shard_size; + let mut reader = + match create_bitrot_reader(None, Some(&disk), bucket, &path, 0, read_length, shard_size, checksum_algo.clone(), false) + .await + { + Ok(Some(r)) => r, + _ => { + eprintln!("Failed to create bitrot reader for EC part: {:?}", part_path); + return false; + } + }; + + let mut buf = vec![0u8; shard_size]; + match reader.read(&mut buf).await { + Ok(n) if n > 0 => { + eprintln!( + "Successfully read {} bytes (EC part) via create_bitrot_reader with {:?}", + n, checksum_algo + ); + true + } + _ => { + eprintln!("Failed to read EC part: {:?}", part_path); + false + } + } +} + +#[tokio::test] +async fn test_legacy_bitrot_read() { + if !legacy_test_data_exists() { + eprintln!("Skipping legacy bitrot test: test1/vvvv xl.meta or RUSTFS_SKIP_LEGACY_TEST"); + return; + } + + // let root = workspace_root(); + let root = PathBuf::from("/Users/weisd/project/minio"); + + let disk_name = "test"; + let bucket = "vvvv"; + + // Try both EC (part files) and inline objects + let ok = run_legacy_bitrot_test_for_object(&root, disk_name, bucket, "ktvzip.tar.gz").await; + + eprintln!("ok: {:?}", ok); + + assert!(ok, "create_bitrot_reader failed for both ktvzip.tar.gz and path_traversal.md"); + + let ok = run_legacy_bitrot_test_for_object(&root, disk_name, bucket, "path_traversal.md").await; + assert!(ok, "create_bitrot_reader failed for path_traversal.md"); +} diff --git a/crates/filemeta/examples/dump_versions.rs b/crates/filemeta/examples/dump_versions.rs new file mode 100644 index 000000000..f9dd8e39c --- /dev/null +++ b/crates/filemeta/examples/dump_versions.rs @@ -0,0 +1,24 @@ +use rustfs_filemeta::FileMeta; +use std::{env, fs, path::PathBuf}; + +fn main() { + let path = env::args() + .nth(1) + .map(PathBuf::from) + .expect("usage: dump_versions "); + + let data = fs::read(&path).expect("read xl.meta"); + let meta = FileMeta::load(&data).expect("load xl.meta"); + let versions = meta + .into_file_info_versions("debug-bucket", "debug-object", true) + .expect("decode versions"); + + println!("path: {}", path.display()); + println!("versions: {}", versions.versions.len()); + for (idx, version) in versions.versions.iter().enumerate() { + println!( + "#{idx}: version_id={:?} deleted={} mark_deleted={} is_latest={} size={} mod_time={:?}", + version.version_id, version.deleted, version.mark_deleted, version.is_latest, version.size, version.mod_time + ); + } +} diff --git a/crates/filemeta/src/fileinfo.rs b/crates/filemeta/src/fileinfo.rs index 8e6285cc7..822ec8a55 100644 --- a/crates/filemeta/src/fileinfo.rs +++ b/crates/filemeta/src/fileinfo.rs @@ -16,7 +16,10 @@ use crate::{Error, ReplicationState, ReplicationStatusType, Result, TRANSITION_C use bytes::Bytes; use rmp_serde::Serializer; use rustfs_utils::HashAlgorithm; -use rustfs_utils::http::headers::{RESERVED_METADATA_PREFIX_LOWER, RUSTFS_HEALING}; +use rustfs_utils::http::{ + SUFFIX_COMPRESSION, SUFFIX_DATA_MOVED, SUFFIX_HEALING, SUFFIX_INLINE_DATA, SUFFIX_TIER_FV_ID, SUFFIX_TIER_FV_MARKER, + SUFFIX_TIER_SKIP_FV_ID, contains_key_str, get_str, insert_str, +}; use s3s::dto::{RestoreStatus, Timestamp}; use s3s::header::X_AMZ_RESTORE; use serde::{Deserialize, Serialize}; @@ -236,6 +239,8 @@ pub struct FileInfo { // Combined checksum when object was uploaded pub checksum: Option, pub versioned: bool, + /// True when version meta was parsed via rmp_serde fallback (legacy format). + pub uses_legacy_checksum: bool, } impl FileInfo { @@ -367,58 +372,48 @@ impl FileInfo { } pub fn set_healing(&mut self) { - self.metadata.insert(RUSTFS_HEALING.to_string(), "true".to_string()); + insert_str(&mut self.metadata, SUFFIX_HEALING, "true".to_string()); } pub fn set_tier_free_version_id(&mut self, version_id: &str) { - self.metadata - .insert(format!("{RESERVED_METADATA_PREFIX_LOWER}{TIER_FV_ID}"), version_id.to_string()); + insert_str(&mut self.metadata, SUFFIX_TIER_FV_ID, version_id.to_string()); } pub fn tier_free_version_id(&self) -> String { - self.metadata[&format!("{RESERVED_METADATA_PREFIX_LOWER}{TIER_FV_ID}")].clone() + get_str(&self.metadata, SUFFIX_TIER_FV_ID).unwrap_or_default() } pub fn set_tier_free_version(&mut self) { - self.metadata - .insert(format!("{RESERVED_METADATA_PREFIX_LOWER}{TIER_FV_MARKER}"), "".to_string()); + insert_str(&mut self.metadata, SUFFIX_TIER_FV_MARKER, "".to_string()); } pub fn set_skip_tier_free_version(&mut self) { - self.metadata - .insert(format!("{RESERVED_METADATA_PREFIX_LOWER}{TIER_SKIP_FV_ID}"), "".to_string()); + insert_str(&mut self.metadata, SUFFIX_TIER_SKIP_FV_ID, "".to_string()); } pub fn skip_tier_free_version(&self) -> bool { - self.metadata - .contains_key(&format!("{RESERVED_METADATA_PREFIX_LOWER}{TIER_SKIP_FV_ID}")) + contains_key_str(&self.metadata, SUFFIX_TIER_SKIP_FV_ID) } pub fn tier_free_version(&self) -> bool { - self.metadata - .contains_key(&format!("{RESERVED_METADATA_PREFIX_LOWER}{TIER_FV_MARKER}")) + contains_key_str(&self.metadata, SUFFIX_TIER_FV_MARKER) } pub fn set_inline_data(&mut self) { - self.metadata - .insert(format!("{RESERVED_METADATA_PREFIX_LOWER}inline-data").to_owned(), "true".to_owned()); + insert_str(&mut self.metadata, SUFFIX_INLINE_DATA, "true".to_string()); } pub fn set_data_moved(&mut self) { - self.metadata - .insert(format!("{RESERVED_METADATA_PREFIX_LOWER}data-moved").to_owned(), "true".to_owned()); + insert_str(&mut self.metadata, SUFFIX_DATA_MOVED, "true".to_string()); } pub fn inline_data(&self) -> bool { - self.metadata - .contains_key(format!("{RESERVED_METADATA_PREFIX_LOWER}inline-data").as_str()) - && !self.is_remote() + contains_key_str(&self.metadata, SUFFIX_INLINE_DATA) && !self.is_remote() } /// Check if the object is compressed pub fn is_compressed(&self) -> bool { - self.metadata - .contains_key(&format!("{RESERVED_METADATA_PREFIX_LOWER}compression")) + contains_key_str(&self.metadata, SUFFIX_COMPRESSION) } /// Check if the object is remote (transitioned to another tier) diff --git a/crates/filemeta/src/filemeta.rs b/crates/filemeta/src/filemeta.rs index 7e89ef91f..0ef91d991 100644 --- a/crates/filemeta/src/filemeta.rs +++ b/crates/filemeta/src/filemeta.rs @@ -13,17 +13,20 @@ // limitations under the License. use crate::{ - ErasureAlgo, ErasureInfo, Error, FileInfo, FileInfoVersions, InlineData, ObjectPartInfo, RawFileInfo, ReplicationState, - ReplicationStatusType, Result, TIER_FV_ID, TIER_FV_MARKER, VersionPurgeStatusType, is_restored_object_on_disk, + ErasureAlgo, ErasureInfo, Error, FileInfo, FileInfoVersions, InlineData, NULL_VERSION_ID, ObjectPartInfo, RawFileInfo, + ReplicationState, ReplicationStatusType, Result, VersionPurgeStatusType, is_restored_object_on_disk, replication_statuses_map, version_purge_statuses_map, }; use byteorder::ByteOrder; use bytes::Bytes; -use rustfs_utils::http::AMZ_BUCKET_REPLICATION_STATUS; use rustfs_utils::http::headers::{ - self, AMZ_META_UNENCRYPTED_CONTENT_LENGTH, AMZ_META_UNENCRYPTED_CONTENT_MD5, AMZ_RESTORE_EXPIRY_DAYS, - AMZ_RESTORE_REQUEST_DATE, AMZ_STORAGE_CLASS, RESERVED_METADATA_PREFIX, RESERVED_METADATA_PREFIX_LOWER, - VERSION_PURGE_STATUS_KEY, + AMZ_META_UNENCRYPTED_CONTENT_LENGTH, AMZ_META_UNENCRYPTED_CONTENT_MD5, AMZ_RESTORE_EXPIRY_DAYS, AMZ_RESTORE_REQUEST_DATE, + AMZ_STORAGE_CLASS, +}; +use rustfs_utils::http::{ + AMZ_BUCKET_REPLICATION_STATUS, SUFFIX_DATA_MOV, SUFFIX_HEALING, SUFFIX_PURGESTATUS, SUFFIX_REPLICA_STATUS, + SUFFIX_REPLICA_TIMESTAMP, SUFFIX_REPLICATION_STATUS, SUFFIX_REPLICATION_TIMESTAMP, has_internal_suffix, insert_bytes, + is_internal_key, }; use s3s::header::X_AMZ_RESTORE; use serde::{Deserialize, Serialize}; @@ -48,7 +51,8 @@ pub static XL_FILE_HEADER: [u8; 4] = [b'X', b'L', b'2', b' ']; static XL_FILE_VERSION_MAJOR: u16 = 1; static XL_FILE_VERSION_MINOR: u16 = 3; static XL_HEADER_VERSION: u8 = 3; -pub static XL_META_VERSION: u8 = 2; +pub static XL_META_VERSION: u8 = 3; +/// Legacy format (main branch): meta_ver=2 with file/header versions 1.3.3. static XXHASH_SEED: u64 = 0; const XL_FLAG_FREE_VERSION: u8 = 1 << 0; @@ -58,6 +62,18 @@ const _XL_FLAG_INLINE_DATA: u8 = 1 << 2; const META_DATA_READ_DEFAULT: usize = 4 << 10; const MSGP_UINT32_SIZE: usize = 5; +/// Max object versions per object, default is 10000 +const DEFAULT_OBJECT_MAX_VERSIONS: usize = 10000; + +/// Returns the inline data map key for a version_id. "null" for null version. +pub(crate) fn data_key_for_version(version_id: Option) -> String { + if version_id.is_none() || version_id == Some(Uuid::nil()) { + NULL_VERSION_ID.to_string() + } else { + version_id.unwrap_or_default().to_string() + } +} + pub const TRANSITION_COMPLETE: &str = "complete"; pub const TRANSITION_PENDING: &str = "pending"; @@ -68,8 +84,14 @@ pub const TRANSITIONED_OBJECTNAME: &str = "transitioned-object"; pub const TRANSITIONED_VERSION_ID: &str = "transitioned-versionID"; pub const TRANSITION_TIER: &str = "transition-tier"; +/// Returns true if the key is a transient internal flag that should not be persisted to meta_sys. +pub fn is_skip_meta_key(key: &str) -> bool { + has_internal_suffix(key, SUFFIX_HEALING) || has_internal_suffix(key, SUFFIX_DATA_MOV) +} + mod codec; mod inline_data; +mod msgp_decode; mod validation; mod version; @@ -171,11 +193,10 @@ impl FileMeta { for (k, v) in fi.metadata.iter() { // Split metadata into meta_user and meta_sys based on prefix // This logic must match From for MetaObject - if k.len() > RESERVED_METADATA_PREFIX.len() - && (k.starts_with(RESERVED_METADATA_PREFIX) || k.starts_with(RESERVED_METADATA_PREFIX_LOWER)) - { + let is_system = is_internal_key(k); + if is_system { // Skip internal flags that shouldn't be persisted - if k == headers::X_RUSTFS_HEALING || k == headers::X_RUSTFS_DATA_MOV { + if is_skip_meta_key(k) { continue; } // Insert into meta_sys @@ -221,18 +242,26 @@ impl FileMeta { } pub fn add_version(&mut self, mut fi: FileInfo) -> Result<()> { + // empty version_id means "null" (versioning disabled/suspended) if fi.version_id.is_none() { fi.version_id = Some(Uuid::nil()); } + let version_key = data_key_for_version(fi.version_id); + let mut next_data = self.data.clone(); + if let Some(ref data) = fi.data { - let key = fi.version_id.unwrap_or_default().to_string(); - self.data.replace(&key, data.to_vec())?; + next_data.replace(&version_key, data.to_vec())?; + } else { + let _ = next_data.remove_key(&version_key)?; } let version = FileMetaVersion::from(fi); - self.add_version_filemata(version) + self.add_version_filemata(version)?; + self.data = next_data; + + Ok(()) } pub fn add_version_filemata(&mut self, version: FileMetaVersion) -> Result<()> { @@ -240,13 +269,12 @@ impl FileMeta { return Err(Error::other("file meta version invalid")); } - // TODO: make it configurable - // 1000 is the limit of versions - // if self.versions.len() + 1 > 1000 { - // return Err(Error::other( - // "You've exceeded the limit on the number of versions you can create on this object", - // )); - // } + // check max versions limit + if self.versions.len() + 1 > DEFAULT_OBJECT_MAX_VERSIONS { + return Err(Error::other( + "You've exceeded the limit on the number of versions you can create on this object", + )); + } if self.versions.is_empty() { self.versions.push(FileMetaShallowVersion::try_from(version)?); @@ -255,21 +283,44 @@ impl FileMeta { let vid = version.get_version_id(); - if let Some(fidx) = self.versions.iter().position(|v| v.header.version_id == vid) { + // Match existing version for replace; null version: None and Some(nil) are equivalent + let matches = |h: &Option| { + let v_null = vid.is_none() || vid == Some(Uuid::nil()); + let h_null = h.is_none() || *h == Some(Uuid::nil()); + (v_null && h_null) || (vid == *h) + }; + + if let Some(fidx) = self.versions.iter().position(|v| matches(&v.header.version_id)) { return self.set_idx(fidx, version); } + // append placeholder to find insert position + let placeholder = FileMetaShallowVersion { + header: FileMetaVersionHeader { + mod_time: None, // None sorts before any real mod_time + ..Default::default() + }, + meta: Vec::new(), + }; + self.versions.push(placeholder); + let mod_time = version.get_mod_time(); + let new_shallow = FileMetaShallowVersion::try_from(version)?; for (idx, exist) in self.versions.iter().enumerate() { - if let Some(ref ex_mt) = exist.header.mod_time - && let Some(ref in_md) = mod_time - && ex_mt <= in_md - { - self.versions.insert(idx, FileMetaShallowVersion::try_from(version)?); + let ex_mt = exist.header.mod_time; + let insert_here = match (ex_mt, mod_time) { + (None, _) => true, // placeholder: always insert before + (Some(em), Some(nm)) => em <= nm, + (Some(_), None) => false, + }; + if insert_here { + self.versions.insert(idx, new_shallow); + self.versions.pop(); // remove placeholder return Ok(()); } } + self.versions.pop(); // remove placeholder on fallback Err(Error::other("add_version failed")) // if !ver.valid() { @@ -343,8 +394,9 @@ impl FileMeta { && let Some(delete_marker) = ventry.delete_marker.as_mut() { if fi.delete_marker_replication_status() == ReplicationStatusType::Replica { - delete_marker.meta_sys.insert( - format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, "replica-status"), + insert_bytes( + &mut delete_marker.meta_sys, + SUFFIX_REPLICA_STATUS, fi.replication_state_internal .as_ref() .map(|v| v.replica_status.clone()) @@ -353,8 +405,9 @@ impl FileMeta { .as_bytes() .to_vec(), ); - delete_marker.meta_sys.insert( - format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, "replica-timestamp"), + insert_bytes( + &mut delete_marker.meta_sys, + SUFFIX_REPLICA_TIMESTAMP, fi.replication_state_internal .as_ref() .map(|v| v.replica_timestamp.unwrap_or(OffsetDateTime::UNIX_EPOCH).to_string()) @@ -363,8 +416,9 @@ impl FileMeta { .to_vec(), ); } else { - delete_marker.meta_sys.insert( - format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, "replication-status"), + insert_bytes( + &mut delete_marker.meta_sys, + SUFFIX_REPLICATION_STATUS, fi.replication_state_internal .as_ref() .map(|v| v.replication_status_internal.clone().unwrap_or_default()) @@ -372,8 +426,9 @@ impl FileMeta { .as_bytes() .to_vec(), ); - delete_marker.meta_sys.insert( - format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, "replication-timestamp"), + insert_bytes( + &mut delete_marker.meta_sys, + SUFFIX_REPLICATION_TIMESTAMP, fi.replication_state_internal .as_ref() .map(|v| v.replication_timestamp.unwrap_or(OffsetDateTime::UNIX_EPOCH).to_string()) @@ -387,15 +442,14 @@ impl FileMeta { if !fi.version_purge_status().is_empty() && let Some(delete_marker) = ventry.delete_marker.as_mut() { - delete_marker.meta_sys.insert( - VERSION_PURGE_STATUS_KEY.to_string(), - fi.replication_state_internal - .as_ref() - .map(|v| v.version_purge_status_internal.clone().unwrap_or_default()) - .unwrap_or_default() - .as_bytes() - .to_vec(), - ); + let value = fi + .replication_state_internal + .as_ref() + .map(|v| v.version_purge_status_internal.clone().unwrap_or_default()) + .unwrap_or_default() + .as_bytes() + .to_vec(); + insert_bytes(&mut delete_marker.meta_sys, SUFFIX_PURGESTATUS, value); } if let Some(delete_marker) = ventry.delete_marker.as_mut() { @@ -433,8 +487,9 @@ impl FileMeta { if let Some(delete_marker) = v.delete_marker.as_mut() { if !fi.delete_marker_replication_status().is_empty() { if fi.delete_marker_replication_status() == ReplicationStatusType::Replica { - delete_marker.meta_sys.insert( - format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, "replica-status"), + insert_bytes( + &mut delete_marker.meta_sys, + SUFFIX_REPLICA_STATUS, fi.replication_state_internal .as_ref() .map(|v| v.replica_status.clone()) @@ -443,8 +498,9 @@ impl FileMeta { .as_bytes() .to_vec(), ); - delete_marker.meta_sys.insert( - format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, "replica-timestamp"), + insert_bytes( + &mut delete_marker.meta_sys, + SUFFIX_REPLICA_TIMESTAMP, fi.replication_state_internal .as_ref() .map(|v| v.replica_timestamp.unwrap_or(OffsetDateTime::UNIX_EPOCH).to_string()) @@ -453,8 +509,9 @@ impl FileMeta { .to_vec(), ); } else { - delete_marker.meta_sys.insert( - format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, "replication-status"), + insert_bytes( + &mut delete_marker.meta_sys, + SUFFIX_REPLICATION_STATUS, fi.replication_state_internal .as_ref() .map(|v| v.replication_status_internal.clone().unwrap_or_default()) @@ -462,8 +519,9 @@ impl FileMeta { .as_bytes() .to_vec(), ); - delete_marker.meta_sys.insert( - format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, "replication-timestamp"), + insert_bytes( + &mut delete_marker.meta_sys, + SUFFIX_REPLICATION_TIMESTAMP, fi.replication_state_internal .as_ref() .map(|v| v.replication_timestamp.unwrap_or(OffsetDateTime::UNIX_EPOCH).to_string()) @@ -502,15 +560,14 @@ impl FileMeta { let mut v = self.get_idx(i)?; if let Some(obj) = v.object.as_mut() { - obj.meta_sys.insert( - VERSION_PURGE_STATUS_KEY.to_string(), - fi.replication_state_internal - .as_ref() - .map(|v| v.version_purge_status_internal.clone().unwrap_or_default()) - .unwrap_or_default() - .as_bytes() - .to_vec(), - ); + let value = fi + .replication_state_internal + .as_ref() + .map(|v| v.version_purge_status_internal.clone().unwrap_or_default()) + .unwrap_or_default() + .as_bytes() + .to_vec(); + insert_bytes(&mut obj.meta_sys, SUFFIX_PURGESTATUS, value); for (k, v) in fi .replication_state_internal .as_ref() @@ -617,13 +674,14 @@ impl FileMeta { // TODO: freeVersion if header.free_version() { non_free_versions -= 1; - if include_free_versions && found_free_version.is_none() { - let mut found_free_fi = FileMetaVersion::default(); - if found_free_fi.unmarshal_msg(&ver.meta).is_ok() && found_free_fi.version_type != VersionType::Invalid { - let mut free_fi = found_free_fi.into_fileinfo(volume, path, all_parts); - free_fi.is_latest = true; - found_free_version = Some(free_fi); - } + if include_free_versions + && found_free_version.is_none() + && let Ok(found_free_fi) = ver.parse_version_meta() + && found_free_fi.version_type != VersionType::Invalid + { + let mut free_fi = found_free_fi.into_fileinfo(volume, path, all_parts); + free_fi.is_latest = true; + found_free_version = Some(free_fi); } if header.version_id != Some(vid) { @@ -650,10 +708,10 @@ impl FileMeta { fi.successor_mod_time = succ_mod_time; } - if read_data { + if read_data && fi.inline_data() { fi.data = self .data - .find(fi.version_id.unwrap_or_default().to_string().as_str())? + .find(data_key_for_version(fi.version_id).as_str())? .map(bytes::Bytes::from); } @@ -725,9 +783,7 @@ impl FileMeta { pub fn into_file_info_versions(&self, volume: &str, path: &str, all_parts: bool) -> Result { let mut versions = Vec::new(); for version in self.versions.iter() { - let mut file_version = FileMetaVersion::default(); - file_version.unmarshal_msg(&version.meta)?; - let fi = file_version.into_fileinfo(volume, path, all_parts); + let fi = version.into_fileinfo(volume, path, all_parts)?; versions.push(fi); } @@ -770,29 +826,9 @@ impl FileMeta { self.versions.first().unwrap().header.mod_time } - /// Load or convert from buffer + /// Load or convert from buffer. Handles both current (meta_ver=3) and legacy (meta_ver=2) formats. pub fn load_or_convert(buf: &[u8]) -> Result { - // Try to load as current format first - match Self::load(buf) { - Ok(meta) => Ok(meta), - Err(_) => { - // Try to convert from legacy format - Self::load_legacy(buf) - } - } - } - - /// Load legacy format - pub fn load_legacy(_buf: &[u8]) -> Result { - // Implementation for loading legacy xl.meta formats - // This would handle conversion from older formats - Err(Error::other("Legacy format not yet implemented")) - } - - /// Add legacy version - pub fn add_legacy(&mut self, _legacy_obj: &str) -> Result<()> { - // Implementation for adding legacy xl.meta v1 objects - Err(Error::other("Legacy version addition not yet implemented")) + Self::load(buf) } /// List all versions as FileInfo @@ -1061,6 +1097,7 @@ mod test { object: None, delete_marker: None, write_version: 1, + uses_legacy_checksum: false, }; assert!(legacy_version.is_legacy(), "Should be recognized as a Legacy version"); @@ -1178,6 +1215,7 @@ mod test { }), delete_marker: None, write_version: 1, + uses_legacy_checksum: false, }; let shallow_version = FileMetaShallowVersion::try_from(version).expect("Conversion failed"); @@ -1391,6 +1429,7 @@ mod test { object: None, delete_marker: Some(delete_marker), write_version: (i + 100) as u64, + uses_legacy_checksum: false, }; let shallow_version = FileMetaShallowVersion::try_from(delete_version).unwrap(); @@ -1458,6 +1497,57 @@ mod test { assert!(fm.validate_integrity().is_ok()); } + #[test] + fn test_add_version_clears_stale_inline_data_for_null_version() { + let mut fm = FileMeta::new(); + + let mut inline_fi = crate::fileinfo::FileInfo::new("test", 2, 1); + inline_fi.mod_time = Some(OffsetDateTime::now_utc()); + inline_fi.data = Some(Bytes::new()); + inline_fi.set_inline_data(); + fm.add_version(inline_fi).unwrap(); + + let inline_version = fm.into_fileinfo("bucket", "test", "", true, false, true).unwrap(); + assert!(inline_version.inline_data()); + assert_eq!(inline_version.data, Some(Bytes::new())); + + let mut disk_fi = crate::fileinfo::FileInfo::new("test", 2, 1); + disk_fi.mod_time = Some(OffsetDateTime::now_utc() + time::Duration::seconds(1)); + disk_fi.size = 1024; + fm.add_version(disk_fi).unwrap(); + + let latest = fm.into_fileinfo("bucket", "test", "", true, false, true).unwrap(); + assert!(!latest.inline_data()); + assert!(latest.data.is_none()); + assert!( + fm.data + .find(data_key_for_version(latest.version_id).as_str()) + .unwrap() + .is_none() + ); + } + + #[test] + fn test_add_version_keeps_inline_data_when_version_insert_fails() { + let mut fm = FileMeta::new(); + + let mut inline_fi = crate::fileinfo::FileInfo::new("test", 2, 1); + inline_fi.mod_time = Some(OffsetDateTime::now_utc()); + inline_fi.data = Some(Bytes::from_static(b"inline")); + inline_fi.set_inline_data(); + fm.add_version(inline_fi).unwrap(); + + let before = fm.data.find(data_key_for_version(Some(Uuid::nil())).as_str()).unwrap(); + assert_eq!(before, Some(Bytes::from_static(b"inline").to_vec())); + + let invalid_disk_fi = crate::fileinfo::FileInfo::new("test", 2, 1); + let err = fm.add_version(invalid_disk_fi).unwrap_err(); + assert!(err.to_string().contains("file meta version invalid")); + + let after = fm.data.find(data_key_for_version(Some(Uuid::nil())).as_str()).unwrap(); + assert_eq!(after, Some(Bytes::from_static(b"inline").to_vec())); + } + #[test] fn test_version_merge_scenarios() { // Test various version merge scenarios diff --git a/crates/filemeta/src/filemeta/codec.rs b/crates/filemeta/src/filemeta/codec.rs index 610f95ac4..164d67fc4 100644 --- a/crates/filemeta/src/filemeta/codec.rs +++ b/crates/filemeta/src/filemeta/codec.rs @@ -22,10 +22,26 @@ impl FileMeta { pub fn load(buf: &[u8]) -> Result { let mut xl = FileMeta::default(); xl.unmarshal_msg(buf)?; - Ok(xl) } + /// Read (major, minor, header_ver, meta_ver) from xl.meta without full parse. + /// Returns Err if format is not feat (e.g. legacy uses rmp_serde in meta block). + pub fn read_format_versions(buf: &[u8]) -> Result<(u16, u16, u8, u8)> { + let (buf, major, minor) = Self::check_xl2_v1(buf)?; + if buf.len() < 5 { + return Err(Error::other("insufficient data for metadata length prefix")); + } + let (mut size_buf, buf) = buf.split_at(5); + let bin_len = rmp::decode::read_bin_len(&mut size_buf)?; + if buf.len() < bin_len as usize { + return Err(Error::other("insufficient data for metadata")); + } + let (meta, _) = buf.split_at(bin_len as usize); + let (_, header_ver, meta_ver, _) = Self::decode_xl_headers(meta)?; + Ok((major, minor, header_ver, meta_ver)) + } + pub fn check_xl2_v1(buf: &[u8]) -> Result<(&[u8], u16, u16)> { if buf.len() < 8 { return Err(Error::other("xl file header not exists")); @@ -294,11 +310,9 @@ impl FileMeta { let offset = wr.len(); - // xl header - rmp::encode::write_uint8(&mut wr, XL_HEADER_VERSION)?; - rmp::encode::write_uint8(&mut wr, XL_META_VERSION)?; + rmp::encode::write_uint(&mut wr, XL_HEADER_VERSION as u64)?; + rmp::encode::write_uint(&mut wr, XL_META_VERSION as u64)?; - // versions rmp::encode::write_sint(&mut wr, self.versions.len() as i64)?; for ver in self.versions.iter() { diff --git a/crates/filemeta/src/filemeta/inline_data.rs b/crates/filemeta/src/filemeta/inline_data.rs index 11ff6ccb2..e8dca577f 100644 --- a/crates/filemeta/src/filemeta/inline_data.rs +++ b/crates/filemeta/src/filemeta/inline_data.rs @@ -40,10 +40,14 @@ impl FileMeta { /// Count shared data directories pub fn shared_data_dir_count(&self, version_id: Option, data_dir: Option) -> usize { - let version_id = version_id.unwrap_or_default(); + let vid = version_id.unwrap_or_default(); if self.data.entries().unwrap_or_default() > 0 - && self.data.find(version_id.to_string().as_str()).unwrap_or_default().is_some() + && self + .data + .find(super::data_key_for_version(version_id).as_str()) + .unwrap_or_default() + .is_some() { return 0; } @@ -51,9 +55,7 @@ impl FileMeta { self.versions .iter() .filter(|v| { - v.header.version_type == VersionType::Object - && v.header.version_id != Some(version_id) - && v.header.uses_data_dir() + v.header.version_type == VersionType::Object && v.header.version_id != Some(vid) && v.header.uses_data_dir() }) .filter_map(|v| FileMetaVersion::decode_data_dir_from_meta(&v.meta).ok()) .filter(|&dir| dir.is_some() && dir == data_dir) diff --git a/crates/filemeta/src/filemeta/msgp_decode.rs b/crates/filemeta/src/filemeta/msgp_decode.rs new file mode 100644 index 000000000..68e44ca02 --- /dev/null +++ b/crates/filemeta/src/filemeta/msgp_decode.rs @@ -0,0 +1,212 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::{Error, Result}; +use rmp::Marker; +use std::io::Read; + +/// Reader that prepends a single byte to the stream. Used when we've read the marker +/// and need to pass it to a decoder that expects to read the marker itself. +pub(crate) struct PrependByteReader<'a, R> { + pub(crate) byte: Option, + pub(crate) inner: &'a mut R, +} + +impl Read for PrependByteReader<'_, R> { + #[allow(clippy::collapsible_if)] + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + if let Some(b) = self.byte.take() { + if !buf.is_empty() { + buf[0] = b; + return Ok(1); + } + self.byte = Some(b); + } + self.inner.read(buf) + } +} + +/// Read MessagePack nil or array length. Returns None for nil, Some(len) for array. +pub(crate) fn read_nil_or_array_len(rd: &mut R) -> Result> { + let mut buf = [0u8; 1]; + rd.read_exact(&mut buf).map_err(Error::from)?; + let marker = buf[0]; + match marker { + 0xc0 => Ok(None), // nil + 0x90..=0x9f => Ok(Some((marker & 0x0f) as usize)), + 0xdc => { + let mut b = [0u8; 2]; + rd.read_exact(&mut b).map_err(Error::from)?; + Ok(Some(u16::from_be_bytes(b) as usize)) + } + 0xdd => { + let mut b = [0u8; 4]; + rd.read_exact(&mut b).map_err(Error::from)?; + Ok(Some(u32::from_be_bytes(b) as usize)) + } + _ => Err(Error::other(format!("expected nil or array, got marker 0x{marker:02x}"))), + } +} + +/// Read MessagePack nil or map length. Returns None for nil, Some(len) for map. +pub(crate) fn read_nil_or_map_len(rd: &mut R) -> Result> { + let mut buf = [0u8; 1]; + rd.read_exact(&mut buf).map_err(Error::from)?; + let marker = buf[0]; + match marker { + 0xc0 => Ok(None), // nil + 0x80..=0x8f => Ok(Some((marker & 0x0f) as usize)), + 0xde => { + let mut b = [0u8; 2]; + rd.read_exact(&mut b).map_err(Error::from)?; + Ok(Some(u16::from_be_bytes(b) as usize)) + } + 0xdf => { + let mut b = [0u8; 4]; + rd.read_exact(&mut b).map_err(Error::from)?; + Ok(Some(u32::from_be_bytes(b) as usize)) + } + _ => Err(Error::other(format!("expected nil or map, got marker 0x{marker:02x}"))), + } +} + +/// Skip a single MessagePack value. Used for unknown map keys. +pub(crate) fn skip_msgp_value(rd: &mut R) -> Result<()> { + let marker = rmp::decode::read_marker(rd).map_err(Error::from)?; + let skip_len: usize = match marker { + Marker::Null | Marker::False | Marker::True => 0, + Marker::FixPos(_) | Marker::FixNeg(_) => 0, + Marker::U8 => 1, + Marker::U16 => 2, + Marker::U32 => 4, + Marker::U64 => 8, + Marker::I8 => 1, + Marker::I16 => 2, + Marker::I32 => 4, + Marker::I64 => 8, + Marker::F32 => 4, + Marker::F64 => 8, + Marker::FixStr(n) => n as usize, + Marker::Str8 => { + let mut b = [0u8; 1]; + rd.read_exact(&mut b).map_err(Error::from)?; + b[0] as usize + } + Marker::Str16 => { + let mut b = [0u8; 2]; + rd.read_exact(&mut b).map_err(Error::from)?; + u16::from_be_bytes(b) as usize + } + Marker::Str32 => { + let mut b = [0u8; 4]; + rd.read_exact(&mut b).map_err(Error::from)?; + u32::from_be_bytes(b) as usize + } + Marker::Bin8 => { + let mut b = [0u8; 1]; + rd.read_exact(&mut b).map_err(Error::from)?; + b[0] as usize + } + Marker::Bin16 => { + let mut b = [0u8; 2]; + rd.read_exact(&mut b).map_err(Error::from)?; + u16::from_be_bytes(b) as usize + } + Marker::Bin32 => { + let mut b = [0u8; 4]; + rd.read_exact(&mut b).map_err(Error::from)?; + u32::from_be_bytes(b) as usize + } + Marker::FixArray(n) => { + for _ in 0..n { + skip_msgp_value(rd)?; + } + return Ok(()); + } + Marker::Array16 => { + let mut b = [0u8; 2]; + rd.read_exact(&mut b).map_err(Error::from)?; + let n = u16::from_be_bytes(b); + for _ in 0..n { + skip_msgp_value(rd)?; + } + return Ok(()); + } + Marker::Array32 => { + let mut b = [0u8; 4]; + rd.read_exact(&mut b).map_err(Error::from)?; + let n = u32::from_be_bytes(b); + for _ in 0..n { + skip_msgp_value(rd)?; + } + return Ok(()); + } + Marker::FixMap(n) => { + for _ in 0..n { + skip_msgp_value(rd)?; + skip_msgp_value(rd)?; + } + return Ok(()); + } + Marker::Map16 => { + let mut b = [0u8; 2]; + rd.read_exact(&mut b).map_err(Error::from)?; + let n = u16::from_be_bytes(b); + for _ in 0..n { + skip_msgp_value(rd)?; + skip_msgp_value(rd)?; + } + return Ok(()); + } + Marker::Map32 => { + let mut b = [0u8; 4]; + rd.read_exact(&mut b).map_err(Error::from)?; + let n = u32::from_be_bytes(b); + for _ in 0..n { + skip_msgp_value(rd)?; + skip_msgp_value(rd)?; + } + return Ok(()); + } + Marker::FixExt1 => 1, + Marker::FixExt2 => 2, + Marker::FixExt4 => 4, + Marker::FixExt8 => 8, + Marker::FixExt16 => 16, + Marker::Ext8 => { + let mut b = [0u8; 1]; + rd.read_exact(&mut b).map_err(Error::from)?; + let len = b[0] as usize; + 1 + len // type byte + data + } + Marker::Ext16 => { + let mut b = [0u8; 2]; + rd.read_exact(&mut b).map_err(Error::from)?; + let len = u16::from_be_bytes(b) as usize; + 2 + len // type bytes + data + } + Marker::Ext32 => { + let mut b = [0u8; 4]; + rd.read_exact(&mut b).map_err(Error::from)?; + let len = u32::from_be_bytes(b) as usize; + 4 + len // type bytes + data + } + Marker::Reserved => 0, + }; + if skip_len > 0 { + let mut buf = vec![0u8; skip_len]; + rd.read_exact(&mut buf).map_err(Error::from)?; + } + Ok(()) +} diff --git a/crates/filemeta/src/filemeta/version.rs b/crates/filemeta/src/filemeta/version.rs index 048502447..0cd344d45 100644 --- a/crates/filemeta/src/filemeta/version.rs +++ b/crates/filemeta/src/filemeta/version.rs @@ -12,7 +12,23 @@ // See the License for the specific language governing permissions and // limitations under the License. +//! Version meta parsing with legacy compatibility. +//! +//! **Rule**: To parse version meta bytes (from `FileMetaShallowVersion.meta` or raw `&[u8]`), +//! always use one of: +//! - `FileMetaShallowVersion::parse_version_meta()` or `into_fileinfo()` +//! - `FileMetaVersion::try_from(buf)` +//! +//! Do NOT use `FileMetaVersion::default()` + `unmarshal_msg()` directly, as that fails on +//! legacy (rmp_serde) format. `try_from` falls back to rmp_serde when hand-written decode fails. + +use super::msgp_decode::{PrependByteReader, read_nil_or_array_len, read_nil_or_map_len, skip_msgp_value}; use super::*; +use rustfs_utils::http::{ + SUFFIX_CRC, SUFFIX_FREE_VERSION, SUFFIX_INLINE_DATA, SUFFIX_PURGESTATUS, SUFFIX_TIER_FV_ID, SUFFIX_TIER_FV_MARKER, + SUFFIX_TRANSITION_STATUS, SUFFIX_TRANSITION_TIER, SUFFIX_TRANSITIONED_OBJECTNAME, SUFFIX_TRANSITIONED_VERSION_ID, + contains_key_bytes, get_bytes, has_internal_suffix, insert_bytes, is_internal_key, remove_bytes, strip_internal_prefix, +}; #[derive(Serialize, Deserialize, Debug, Default, PartialEq, Clone, Eq, PartialOrd, Ord)] pub struct FileMetaShallowVersion { @@ -21,9 +37,14 @@ pub struct FileMetaShallowVersion { } impl FileMetaShallowVersion { - pub fn into_fileinfo(&self, volume: &str, path: &str, all_parts: bool) -> Result { - let file_version = FileMetaVersion::try_from(self.meta.as_slice())?; + /// Parse version meta with legacy format compatibility. + /// Use this instead of `FileMetaVersion::default()` + `unmarshal_msg()` to handle old-version xl.meta. + pub fn parse_version_meta(&self) -> Result { + FileMetaVersion::try_from(self.meta.as_slice()) + } + pub fn into_fileinfo(&self, volume: &str, path: &str, all_parts: bool) -> Result { + let file_version = self.parse_version_meta()?; Ok(file_version.into_fileinfo(volume, path, all_parts)) } } @@ -48,6 +69,9 @@ pub struct FileMetaVersion { pub delete_marker: Option, #[serde(rename = "v")] pub write_version: u64, // rustfs version + /// True when parsed via rmp_serde fallback (legacy format). Used for checksum algorithm selection. + #[serde(skip)] + pub uses_legacy_checksum: bool, } impl FileMetaVersion { @@ -104,23 +128,132 @@ impl FileMetaVersion { // decode_data_dir_from_meta reads data_dir from meta TODO: directly parse only data_dir from meta buf, msg.skip pub fn decode_data_dir_from_meta(buf: &[u8]) -> Result> { let mut ver = Self::default(); - ver.unmarshal_msg(buf)?; - + ver.decode_from(&mut std::io::Cursor::new(buf))?; let data_dir = ver.object.map(|v| v.data_dir).unwrap_or_default(); Ok(data_dir) } + pub fn decode_from(&mut self, rd: &mut R) -> Result<()> { + let mut fields = rmp::decode::read_map_len(rd)?; + *self = FileMetaVersion::default(); + + while fields > 0 { + fields -= 1; + + let key_len = rmp::decode::read_str_len(rd)?; + let mut key_buf = vec![0u8; key_len as usize]; + rd.read_exact(&mut key_buf)?; + let key = String::from_utf8(key_buf)?; + + match key.as_str() { + "Type" => { + let v: i64 = rmp::decode::read_int(rd)?; + self.version_type = VersionType::from_u8(v as u8); + } + "V1Obj" => { + // Skip V1Obj (legacy), not supported + let mut buf = [0u8; 1]; + rd.read_exact(&mut buf).map_err(Error::from)?; + if buf[0] != 0xc0 { + let mut prepend = PrependByteReader { + byte: Some(buf[0]), + inner: rd, + }; + skip_msgp_value(&mut prepend)?; + } + } + "V2Obj" => { + let mut buf = [0u8; 1]; + rd.read_exact(&mut buf).map_err(Error::from)?; + if buf[0] == 0xc0 { + self.object = None; + } else { + let mut prepend = PrependByteReader { + byte: Some(buf[0]), + inner: rd, + }; + let mut obj = MetaObject::default(); + obj.decode_from(&mut prepend)?; + self.object = Some(obj); + } + } + "DelObj" => { + let mut buf = [0u8; 1]; + rd.read_exact(&mut buf).map_err(Error::from)?; + if buf[0] == 0xc0 { + self.delete_marker = None; + } else { + let mut prepend = PrependByteReader { + byte: Some(buf[0]), + inner: rd, + }; + let mut dm = MetaDeleteMarker::default(); + dm.decode_from(&mut prepend)?; + self.delete_marker = Some(dm); + } + } + "v" => { + let v: i64 = rmp::decode::read_int(rd)?; + if v < 0 { + return Err(Error::other("negative write_version not supported")); + } + self.write_version = v as u64; + } + other => { + tracing::debug!(field = %other, "decode_from: skipping unknown field"); + skip_msgp_value(rd)?; + } + } + } + + Ok(()) + } + + pub fn encode_to(&self, wr: &mut W) -> Result<()> { + // Variable map size: omit V2Obj/DelObj when None + let mut map_len: u32 = 2; // "Type" + "v" + if self.object.is_some() { + map_len += 1; + } + if self.delete_marker.is_some() { + map_len += 1; + } + + rmp::encode::write_map_len(wr, map_len)?; + + // Type + rmp::encode::write_str(wr, "Type")?; + rmp::encode::write_uint(wr, self.version_type.to_u8() as u64)?; + + // V2Obj + if let Some(ref obj) = self.object { + rmp::encode::write_str(wr, "V2Obj")?; + obj.encode_to(wr)?; + } + + // DelObj + if let Some(ref dm) = self.delete_marker { + rmp::encode::write_str(wr, "DelObj")?; + dm.encode_to(wr)?; + } + + // v + rmp::encode::write_str(wr, "v")?; + rmp::encode::write_uint(wr, self.write_version)?; + + Ok(()) + } + pub fn unmarshal_msg(&mut self, buf: &[u8]) -> Result { - let ret: Self = rmp_serde::from_slice(buf)?; - - *self = ret; - - Ok(buf.len() as u64) + let mut cur = std::io::Cursor::new(buf); + self.decode_from(&mut cur)?; + Ok(cur.position()) } pub fn marshal_msg(&self) -> Result> { - let buf = rmp_serde::to_vec(self)?; - Ok(buf) + let mut wr = Vec::new(); + self.encode_to(&mut wr)?; + Ok(wr) } pub fn free_version(&self) -> bool { @@ -132,7 +265,7 @@ impl FileMetaVersion { } pub fn into_fileinfo(&self, volume: &str, path: &str, all_parts: bool) -> FileInfo { - match self.version_type { + let mut fi = match self.version_type { VersionType::Invalid | VersionType::Legacy => FileInfo { name: path.to_string(), volume: volume.to_string(), @@ -148,7 +281,9 @@ impl FileMetaVersion { .as_ref() .unwrap_or(&MetaDeleteMarker::default()) .into_fileinfo(volume, path, all_parts), - } + }; + fi.uses_legacy_checksum = self.uses_legacy_checksum; + fi } /// Support for Legacy version type @@ -215,28 +350,34 @@ impl TryFrom<&[u8]> for FileMetaVersion { fn try_from(value: &[u8]) -> std::result::Result { let mut ver = FileMetaVersion::default(); - ver.unmarshal_msg(value)?; + if ver.unmarshal_msg(value).is_ok() { + ver.uses_legacy_checksum = false; + return Ok(ver); + } + // Fallback for legacy ver_meta: rmp_serde format + let mut ver: Self = rmp_serde::from_slice(value).map_err(Error::other)?; + ver.uses_legacy_checksum = true; Ok(ver) } } impl From for FileMetaVersion { fn from(value: FileInfo) -> Self { - { - if value.deleted { - FileMetaVersion { - version_type: VersionType::Delete, - delete_marker: Some(MetaDeleteMarker::from(value)), - object: None, - write_version: 0, - } - } else { - FileMetaVersion { - version_type: VersionType::Object, - delete_marker: None, - object: Some(MetaObject::from(value)), - write_version: 0, - } + if value.deleted { + FileMetaVersion { + version_type: VersionType::Delete, + delete_marker: Some(MetaDeleteMarker::from(value)), + object: None, + write_version: 0, + uses_legacy_checksum: false, + } + } else { + FileMetaVersion { + version_type: VersionType::Object, + delete_marker: None, + object: Some(MetaObject::from(value)), + write_version: 0, + uses_legacy_checksum: false, } } } @@ -527,16 +668,472 @@ pub struct MetaObject { impl MetaObject { pub fn unmarshal_msg(&mut self, buf: &[u8]) -> Result { - let ret: Self = rmp_serde::from_slice(buf)?; - - *self = ret; - - Ok(buf.len() as u64) + let mut cur = std::io::Cursor::new(buf); + self.decode_from(&mut cur)?; + Ok(cur.position()) } + // marshal_msg custom messagepack naming consistent with go pub fn marshal_msg(&self) -> Result> { - let buf = rmp_serde::to_vec(self)?; - Ok(buf) + let mut wr = Vec::new(); + self.encode_to(&mut wr)?; + Ok(wr) + } + + pub fn encode_to(&self, wr: &mut W) -> Result<()> { + // Variable map size: omit PartIdx when empty + let mut map_len = 18u32; + if self.part_indices.is_empty() { + map_len -= 1; + } + rmp::encode::write_map_len(wr, map_len)?; + + // ID + rmp::encode::write_str(wr, "ID")?; + rmp::encode::write_bin(wr, self.version_id.unwrap_or_default().as_bytes())?; + + // DDir + rmp::encode::write_str(wr, "DDir")?; + rmp::encode::write_bin(wr, self.data_dir.unwrap_or_default().as_bytes())?; + + // EcAlgo + rmp::encode::write_str(wr, "EcAlgo")?; + rmp::encode::write_uint(wr, self.erasure_algorithm.to_u8() as u64)?; + + // EcM + rmp::encode::write_str(wr, "EcM")?; + rmp::encode::write_sint(wr, self.erasure_m as i64)?; + + // EcN + rmp::encode::write_str(wr, "EcN")?; + rmp::encode::write_sint(wr, self.erasure_n as i64)?; + + // EcBSize + rmp::encode::write_str(wr, "EcBSize")?; + rmp::encode::write_sint(wr, self.erasure_block_size as i64)?; + + // EcIndex + rmp::encode::write_str(wr, "EcIndex")?; + rmp::encode::write_sint(wr, self.erasure_index as i64)?; + + // EcDist + rmp::encode::write_str(wr, "EcDist")?; + rmp::encode::write_array_len(wr, self.erasure_dist.len() as u32)?; + for v in &self.erasure_dist { + rmp::encode::write_uint(wr, *v as u64)?; + } + + // CSumAlgo + rmp::encode::write_str(wr, "CSumAlgo")?; + rmp::encode::write_uint(wr, self.bitrot_checksum_algo.to_u8() as u64)?; + + // PartNums + rmp::encode::write_str(wr, "PartNums")?; + rmp::encode::write_array_len(wr, self.part_numbers.len() as u32)?; + for n in &self.part_numbers { + rmp::encode::write_sint(wr, *n as i64)?; + } + + // PartETags (write nil when empty) + rmp::encode::write_str(wr, "PartETags")?; + if self.part_etags.is_empty() { + rmp::encode::write_nil(wr)?; + } else { + rmp::encode::write_array_len(wr, self.part_etags.len() as u32)?; + for et in &self.part_etags { + rmp::encode::write_str(wr, et)?; + } + } + + // PartSizes + rmp::encode::write_str(wr, "PartSizes")?; + rmp::encode::write_array_len(wr, self.part_sizes.len() as u32)?; + for s in &self.part_sizes { + rmp::encode::write_sint(wr, *s as i64)?; + } + + // PartASizes (write nil when empty) + rmp::encode::write_str(wr, "PartASizes")?; + if self.part_actual_sizes.is_empty() { + rmp::encode::write_nil(wr)?; + } else { + rmp::encode::write_array_len(wr, self.part_actual_sizes.len() as u32)?; + for s in &self.part_actual_sizes { + rmp::encode::write_sint(wr, *s)?; + } + } + + // PartIdx (omit when empty) + if !self.part_indices.is_empty() { + rmp::encode::write_str(wr, "PartIdx")?; + rmp::encode::write_array_len(wr, self.part_indices.len() as u32)?; + for idx in &self.part_indices { + rmp::encode::write_bin(wr, idx)?; + } + } + + // Size + rmp::encode::write_str(wr, "Size")?; + rmp::encode::write_sint(wr, self.size)?; + + // MTime Unix timestamp nanos + rmp::encode::write_str(wr, "MTime")?; + let nanos = self.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH).unix_timestamp_nanos(); + rmp::encode::write_sint(wr, nanos as i64)?; + + // MetaSys (write nil when empty) + rmp::encode::write_str(wr, "MetaSys")?; + if self.meta_sys.is_empty() { + rmp::encode::write_nil(wr)?; + } else { + rmp::encode::write_map_len(wr, self.meta_sys.len() as u32)?; + for (k, v) in &self.meta_sys { + rmp::encode::write_str(wr, k)?; + rmp::encode::write_bin(wr, v)?; + } + } + + // MetaUsr (write nil when empty) + rmp::encode::write_str(wr, "MetaUsr")?; + if self.meta_user.is_empty() { + rmp::encode::write_nil(wr)?; + } else { + rmp::encode::write_map_len(wr, self.meta_user.len() as u32)?; + for (k, v) in &self.meta_user { + rmp::encode::write_str(wr, k)?; + rmp::encode::write_str(wr, v)?; + } + } + + Ok(()) + } + + pub fn decode_from(&mut self, rd: &mut R) -> Result<()> { + let mut fields = rmp::decode::read_map_len(rd).map_err(|e| { + tracing::error!(error = %e, "decode_from: read_map_len failed"); + e + })?; + *self = MetaObject::default(); + + while fields > 0 { + fields -= 1; + + let key_len = rmp::decode::read_str_len(rd).map_err(|e| { + tracing::error!(error = %e, "decode_from: read_str_len key failed"); + e + })?; + let mut key_buf = vec![0u8; key_len as usize]; + rd.read_exact(&mut key_buf).map_err(|e| { + tracing::error!(error = %e, "decode_from: read_exact key_buf failed"); + e + })?; + let key = String::from_utf8(key_buf).map_err(|e| { + tracing::error!(error = %e, "decode_from: from_utf8 key failed"); + e + })?; + + match key.as_str() { + "ID" => { + let _ = rmp::decode::read_bin_len(rd).map_err(|e| { + tracing::error!(error = %e, "decode_from: read_bin_len ID failed"); + e + })?; + let mut buf = [0u8; 16]; + rd.read_exact(&mut buf).map_err(|e| { + tracing::error!(error = %e, "decode_from: read_exact ID buf failed"); + e + })?; + let id = Uuid::from_bytes(buf); + self.version_id = if id.is_nil() { None } else { Some(id) }; + } + "DDir" => { + let _ = rmp::decode::read_bin_len(rd).map_err(|e| { + tracing::error!(error = %e, "decode_from: read_bin_len DDir failed"); + e + })?; + let mut buf = [0u8; 16]; + rd.read_exact(&mut buf).map_err(|e| { + tracing::error!(error = %e, "decode_from: read_exact DDir buf failed"); + e + })?; + let id = Uuid::from_bytes(buf); + self.data_dir = if id.is_nil() { None } else { Some(id) }; + } + "EcAlgo" => { + let v: i64 = rmp::decode::read_int(rd).map_err(|e| { + tracing::error!(error = %e, "decode_from: read_int EcAlgo failed"); + e + })?; + self.erasure_algorithm = ErasureAlgo::from_u8(v as u8); + } + "EcM" => { + let v: i64 = rmp::decode::read_int(rd).map_err(|e| { + tracing::error!(error = %e, "decode_from: read_int EcM failed"); + e + })?; + self.erasure_m = v as usize; + } + "EcN" => { + let v: i64 = rmp::decode::read_int(rd).map_err(|e| { + tracing::error!(error = %e, "decode_from: read_int EcN failed"); + e + })?; + self.erasure_n = v as usize; + } + "EcBSize" => { + let v: i64 = rmp::decode::read_int(rd).map_err(|e| { + tracing::error!(error = %e, "decode_from: read_int EcBSize failed"); + e + })?; + self.erasure_block_size = v as usize; + } + "EcIndex" => { + let v: i64 = rmp::decode::read_int(rd).map_err(|e| { + tracing::error!(error = %e, "decode_from: read_int EcIndex failed"); + e + })?; + self.erasure_index = v as usize; + } + "EcDist" => { + let len = rmp::decode::read_array_len(rd).map_err(|e| { + tracing::error!(error = %e, "decode_from: read_array_len EcDist failed"); + e + })? as usize; + self.erasure_dist.clear(); + self.erasure_dist.reserve(len); + for _ in 0..len { + let v: i64 = rmp::decode::read_int(rd).map_err(|e| { + tracing::error!(error = %e, "decode_from: read_int EcDist item failed"); + e + })?; + self.erasure_dist.push(v as u8); + } + } + "CSumAlgo" => { + let v: i64 = rmp::decode::read_int(rd).map_err(|e| { + tracing::error!(error = %e, "decode_from: read_int CSumAlgo failed"); + e + })?; + self.bitrot_checksum_algo = ChecksumAlgo::from_u8(v as u8); + } + "PartNums" => { + let len = rmp::decode::read_array_len(rd).map_err(|e| { + tracing::error!(error = %e, "decode_from: read_array_len PartNums failed"); + e + })? as usize; + self.part_numbers.clear(); + self.part_numbers.reserve(len); + for _ in 0..len { + let v: i64 = rmp::decode::read_int(rd).map_err(|e| { + tracing::error!(error = %e, "decode_from: read_int PartNums item failed"); + e + })?; + self.part_numbers.push(v as usize); + } + } + "PartETags" => { + let len = match read_nil_or_array_len(rd).map_err(|e| { + tracing::error!(error = %e, "decode_from: read PartETags failed"); + e + })? { + None => { + self.part_etags.clear(); + continue; + } + Some(n) => n, + }; + self.part_etags.clear(); + self.part_etags.reserve(len); + for _ in 0..len { + let s_len = rmp::decode::read_str_len(rd).map_err(|e| { + tracing::error!(error = %e, "decode_from: read_str_len PartETags item failed"); + e + })?; + let mut sbuf = vec![0u8; s_len as usize]; + rd.read_exact(&mut sbuf).map_err(|e| { + tracing::error!(error = %e, "decode_from: read_exact PartETags sbuf failed"); + e + })?; + let s = String::from_utf8(sbuf).map_err(|e| { + tracing::error!(error = %e, "decode_from: from_utf8 PartETags item failed"); + e + })?; + self.part_etags.push(s); + } + } + "PartSizes" => { + let len = rmp::decode::read_array_len(rd).map_err(|e| { + tracing::error!(error = %e, "decode_from: read_array_len PartSizes failed"); + e + })? as usize; + self.part_sizes.clear(); + self.part_sizes.reserve(len); + for _ in 0..len { + let v: i64 = rmp::decode::read_int(rd).map_err(|e| { + tracing::error!(error = %e, "decode_from: read_int PartSizes item failed"); + e + })?; + self.part_sizes.push(v as usize); + } + } + "PartASizes" => { + let len = match read_nil_or_array_len(rd).map_err(|e| { + tracing::error!(error = %e, "decode_from: read PartASizes failed"); + e + })? { + None => { + self.part_actual_sizes.clear(); + continue; + } + Some(n) => n, + }; + self.part_actual_sizes.clear(); + self.part_actual_sizes.reserve(len); + for _ in 0..len { + let v: i64 = rmp::decode::read_int(rd).map_err(|e| { + tracing::error!(error = %e, "decode_from: read_int PartASizes item failed"); + e + })?; + self.part_actual_sizes.push(v); + } + } + "PartIdx" => { + let len = rmp::decode::read_array_len(rd).map_err(|e| { + tracing::error!(error = %e, "decode_from: read_array_len PartIdx failed"); + e + })? as usize; + self.part_indices.clear(); + self.part_indices.reserve(len); + for _ in 0..len { + let blen = rmp::decode::read_bin_len(rd).map_err(|e| { + tracing::error!(error = %e, "decode_from: read_bin_len PartIdx item failed"); + e + })? as usize; + let mut buf = vec![0u8; blen]; + rd.read_exact(&mut buf).map_err(|e| { + tracing::error!(error = %e, "decode_from: read_exact PartIdx buf failed"); + e + })?; + self.part_indices.push(Bytes::from(buf)); + } + } + "Size" => { + let v: i64 = rmp::decode::read_int(rd).map_err(|e| { + tracing::error!(error = %e, "decode_from: read_int Size failed"); + e + })?; + self.size = v; + } + "MTime" => { + let nanos: i64 = rmp::decode::read_int(rd).map_err(|e| { + tracing::error!(error = %e, "decode_from: read_int MTime failed"); + e + })?; + let time = OffsetDateTime::from_unix_timestamp_nanos(nanos as i128).inspect_err(|&e| { + tracing::error!(error = %e, "decode_from: from_unix_timestamp_nanos MTime failed"); + })?; + self.mod_time = if time == OffsetDateTime::UNIX_EPOCH { + None + } else { + Some(time) + }; + } + "MetaSys" => { + let len = match read_nil_or_map_len(rd).map_err(|e| { + tracing::error!(error = %e, "decode_from: read MetaSys failed"); + e + })? { + None => { + self.meta_sys.clear(); + continue; + } + Some(n) => n, + }; + self.meta_sys.clear(); + for _ in 0..len { + let k_len = rmp::decode::read_str_len(rd).map_err(|e| { + tracing::error!(error = %e, "decode_from: read_str_len MetaSys key failed"); + e + })?; + let mut kbuf = vec![0u8; k_len as usize]; + rd.read_exact(&mut kbuf).map_err(|e| { + tracing::error!(error = %e, "decode_from: read_exact MetaSys kbuf failed"); + e + })?; + let k = String::from_utf8(kbuf).map_err(|e| { + tracing::error!(error = %e, "decode_from: from_utf8 MetaSys key failed"); + e + })?; + + let blen = rmp::decode::read_bin_len(rd).map_err(|e| { + tracing::error!(error = %e, "decode_from: read_bin_len MetaSys value failed"); + e + })? as usize; + let mut v = vec![0u8; blen]; + rd.read_exact(&mut v).map_err(|e| { + tracing::error!(error = %e, "decode_from: read_exact MetaSys value failed"); + e + })?; + + self.meta_sys.insert(k, v); + } + } + "MetaUsr" => { + let len = match read_nil_or_map_len(rd).map_err(|e| { + tracing::error!(error = %e, "decode_from: read MetaUsr failed"); + e + })? { + None => { + self.meta_user.clear(); + continue; + } + Some(n) => n, + }; + self.meta_user.clear(); + for _ in 0..len { + let k_len = rmp::decode::read_str_len(rd).map_err(|e| { + tracing::error!(error = %e, "decode_from: read_str_len MetaUsr key failed"); + e + })?; + let mut kbuf = vec![0u8; k_len as usize]; + rd.read_exact(&mut kbuf).map_err(|e| { + tracing::error!(error = %e, "decode_from: read_exact MetaUsr kbuf failed"); + e + })?; + let k = String::from_utf8(kbuf).map_err(|e| { + tracing::error!(error = %e, "decode_from: from_utf8 MetaUsr key failed"); + e + })?; + + let v_len = rmp::decode::read_str_len(rd).map_err(|e| { + tracing::error!(error = %e, "decode_from: read_str_len MetaUsr value failed"); + e + })?; + let mut vbuf = vec![0u8; v_len as usize]; + rd.read_exact(&mut vbuf).map_err(|e| { + tracing::error!(error = %e, "decode_from: read_exact MetaUsr vbuf failed"); + e + })?; + let v = String::from_utf8(vbuf).map_err(|e| { + tracing::error!(error = %e, "decode_from: from_utf8 MetaUsr value failed"); + e + })?; + + self.meta_user.insert(k, v); + } + } + other => { + // Skip unknown fields for forward compatibility with Go (dc.Skip()) + tracing::debug!(field = %other, "decode_from: skipping unknown field"); + skip_msgp_value(rd).map_err(|e| { + tracing::error!(error = %e, "decode_from: skip unknown field failed"); + e + })?; + } + } + } + + Ok(()) } pub fn into_fileinfo(&self, volume: &str, path: &str, all_parts: bool) -> FileInfo { @@ -580,13 +1177,10 @@ impl MetaObject { metadata.insert(k.to_owned(), v.to_owned()); } - let tier_fvidkey = format!("{RESERVED_METADATA_PREFIX_LOWER}{TIER_FV_ID}").to_lowercase(); - let tier_fvmarker_key = format!("{RESERVED_METADATA_PREFIX_LOWER}{TIER_FV_MARKER}").to_lowercase(); - for (k, v) in &self.meta_sys { let lower_k = k.to_lowercase(); - if lower_k == tier_fvidkey || lower_k == tier_fvmarker_key { + if has_internal_suffix(&lower_k, SUFFIX_TIER_FV_ID) || has_internal_suffix(&lower_k, SUFFIX_TIER_FV_MARKER) { continue; } @@ -594,10 +1188,7 @@ impl MetaObject { continue; } - if k.starts_with(RESERVED_METADATA_PREFIX) - || k.starts_with(RESERVED_METADATA_PREFIX_LOWER) - || lower_k == VERSION_PURGE_STATUS_KEY.to_lowercase() - { + if is_internal_key(k) { metadata.insert(k.to_owned(), String::from_utf8(v.to_owned()).unwrap_or_default()); } } @@ -617,10 +1208,7 @@ impl MetaObject { } } - let checksum = self - .meta_sys - .get(format!("{RESERVED_METADATA_PREFIX_LOWER}crc").as_str()) - .map(|v| Bytes::from(v.clone())); + let checksum = get_bytes(&self.meta_sys, SUFFIX_CRC).map(Bytes::from); let erasure = ErasureInfo { algorithm: self.erasure_algorithm.to_string(), @@ -632,24 +1220,16 @@ impl MetaObject { ..Default::default() }; - let transition_status = self - .meta_sys - .get(format!("{RESERVED_METADATA_PREFIX_LOWER}{TRANSITION_STATUS}").as_str()) - .map(|v| String::from_utf8_lossy(v).to_string()) + let transition_status = get_bytes(&self.meta_sys, SUFFIX_TRANSITION_STATUS) + .map(|v| String::from_utf8_lossy(&v).to_string()) .unwrap_or_default(); - let transitioned_objname = self - .meta_sys - .get(format!("{RESERVED_METADATA_PREFIX_LOWER}{TRANSITIONED_OBJECTNAME}").as_str()) - .map(|v| String::from_utf8_lossy(v).to_string()) + let transitioned_objname = get_bytes(&self.meta_sys, SUFFIX_TRANSITIONED_OBJECTNAME) + .map(|v| String::from_utf8_lossy(&v).to_string()) .unwrap_or_default(); - let transition_version_id = self - .meta_sys - .get(format!("{RESERVED_METADATA_PREFIX_LOWER}{TRANSITIONED_VERSION_ID}").as_str()) - .map(|v| Uuid::from_slice(v.as_slice()).unwrap_or_default()); - let transition_tier = self - .meta_sys - .get(format!("{RESERVED_METADATA_PREFIX_LOWER}{TRANSITION_TIER}").as_str()) - .map(|v| String::from_utf8_lossy(v).to_string()) + let transition_version_id = + get_bytes(&self.meta_sys, SUFFIX_TRANSITIONED_VERSION_ID).map(|v| Uuid::from_slice(v.as_slice()).unwrap_or_default()); + let transition_tier = get_bytes(&self.meta_sys, SUFFIX_TRANSITION_TIER) + .map(|v| String::from_utf8_lossy(&v).to_string()) .unwrap_or_default(); FileInfo { @@ -674,24 +1254,20 @@ impl MetaObject { } pub fn set_transition(&mut self, fi: &FileInfo) { - self.meta_sys.insert( - format!("{RESERVED_METADATA_PREFIX_LOWER}{TRANSITION_STATUS}"), - fi.transition_status.as_bytes().to_vec(), - ); - self.meta_sys.insert( - format!("{RESERVED_METADATA_PREFIX_LOWER}{TRANSITIONED_OBJECTNAME}"), + insert_bytes(&mut self.meta_sys, SUFFIX_TRANSITION_STATUS, fi.transition_status.as_bytes().to_vec()); + insert_bytes( + &mut self.meta_sys, + SUFFIX_TRANSITIONED_OBJECTNAME, fi.transitioned_objname.as_bytes().to_vec(), ); if let Some(transition_version_id) = fi.transition_version_id.as_ref() { - self.meta_sys.insert( - format!("{RESERVED_METADATA_PREFIX_LOWER}{TRANSITIONED_VERSION_ID}"), + insert_bytes( + &mut self.meta_sys, + SUFFIX_TRANSITIONED_VERSION_ID, transition_version_id.as_bytes().to_vec(), ); } - self.meta_sys.insert( - format!("{RESERVED_METADATA_PREFIX_LOWER}{TRANSITION_TIER}"), - fi.transition_tier.as_bytes().to_vec(), - ); + insert_bytes(&mut self.meta_sys, SUFFIX_TRANSITION_TIER, fi.transition_tier.as_bytes().to_vec()); } pub fn remove_restore_hdrs(&mut self) { @@ -701,10 +1277,8 @@ impl MetaObject { } pub fn uses_data_dir(&self) -> bool { - if let Some(status) = self - .meta_sys - .get(&format!("{RESERVED_METADATA_PREFIX_LOWER}{TRANSITION_STATUS}")) - && *status == TRANSITION_COMPLETE.as_bytes().to_vec() + if let Some(status) = get_bytes(&self.meta_sys, SUFFIX_TRANSITION_STATUS) + && status == TRANSITION_COMPLETE.as_bytes().to_vec() { return false; } @@ -713,13 +1287,11 @@ impl MetaObject { } pub fn inlinedata(&self) -> bool { - self.meta_sys - .contains_key(format!("{RESERVED_METADATA_PREFIX_LOWER}inline-data").as_str()) + contains_key_bytes(&self.meta_sys, SUFFIX_INLINE_DATA) } pub fn reset_inline_data(&mut self) { - self.meta_sys - .remove(format!("{RESERVED_METADATA_PREFIX_LOWER}inline-data").as_str()); + remove_bytes(&mut self.meta_sys, SUFFIX_INLINE_DATA); } /// Remove restore headers @@ -745,10 +1317,8 @@ impl MetaObject { if fi.skip_tier_free_version() { return (FileMetaVersion::default(), false); } - if let Some(status) = self - .meta_sys - .get(&format!("{RESERVED_METADATA_PREFIX_LOWER}{TRANSITION_STATUS}")) - && *status == TRANSITION_COMPLETE.as_bytes().to_vec() + if let Some(status) = get_bytes(&self.meta_sys, SUFFIX_TRANSITION_STATUS) + && status == TRANSITION_COMPLETE.as_bytes().to_vec() { let vid = Uuid::parse_str(&fi.tier_free_version_id()); if let Err(err) = vid { @@ -768,18 +1338,15 @@ impl MetaObject { let delete_marker = free_entry.delete_marker.as_mut().unwrap(); - delete_marker - .meta_sys - .insert(format!("{RESERVED_METADATA_PREFIX_LOWER}{FREE_VERSION}"), vec![]); + insert_bytes(&mut delete_marker.meta_sys, SUFFIX_FREE_VERSION, vec![]); - let tier_key = format!("{RESERVED_METADATA_PREFIX_LOWER}{TRANSITION_TIER}"); - let tier_obj_key = format!("{RESERVED_METADATA_PREFIX_LOWER}{TRANSITIONED_OBJECTNAME}"); - let tier_obj_vid_key = format!("{RESERVED_METADATA_PREFIX_LOWER}{TRANSITIONED_VERSION_ID}"); - - let aa = [tier_key, tier_obj_key, tier_obj_vid_key]; - for (k, v) in &self.meta_sys { - if aa.contains(k) { - delete_marker.meta_sys.insert(k.clone(), v.clone()); + for suffix in [ + SUFFIX_TRANSITION_TIER, + SUFFIX_TRANSITIONED_OBJECTNAME, + SUFFIX_TRANSITIONED_VERSION_ID, + ] { + if let Some(v) = get_bytes(&self.meta_sys, suffix) { + insert_bytes(&mut delete_marker.meta_sys, suffix, v); } } return (free_entry, true); @@ -805,10 +1372,8 @@ impl From for MetaObject { let mut meta_sys = HashMap::new(); let mut meta_user = HashMap::new(); for (k, v) in value.metadata.iter() { - if k.len() > RESERVED_METADATA_PREFIX.len() - && (k.starts_with(RESERVED_METADATA_PREFIX) || k.starts_with(RESERVED_METADATA_PREFIX_LOWER)) - { - if k == headers::X_RUSTFS_HEALING || k == headers::X_RUSTFS_DATA_MOV { + if is_internal_key(k) { + if is_skip_meta_key(k) { continue; } @@ -819,35 +1384,27 @@ impl From for MetaObject { } if !value.transition_status.is_empty() { - meta_sys.insert( - format!("{RESERVED_METADATA_PREFIX_LOWER}{TRANSITION_STATUS}"), - value.transition_status.as_bytes().to_vec(), - ); + insert_bytes(&mut meta_sys, SUFFIX_TRANSITION_STATUS, value.transition_status.as_bytes().to_vec()); } if !value.transitioned_objname.is_empty() { - meta_sys.insert( - format!("{RESERVED_METADATA_PREFIX_LOWER}{TRANSITIONED_OBJECTNAME}"), + insert_bytes( + &mut meta_sys, + SUFFIX_TRANSITIONED_OBJECTNAME, value.transitioned_objname.as_bytes().to_vec(), ); } if let Some(vid) = &value.transition_version_id { - meta_sys.insert( - format!("{RESERVED_METADATA_PREFIX_LOWER}{TRANSITIONED_VERSION_ID}"), - vid.as_bytes().to_vec(), - ); + insert_bytes(&mut meta_sys, SUFFIX_TRANSITIONED_VERSION_ID, vid.as_bytes().to_vec()); } if !value.transition_tier.is_empty() { - meta_sys.insert( - format!("{RESERVED_METADATA_PREFIX_LOWER}{TRANSITION_TIER}"), - value.transition_tier.as_bytes().to_vec(), - ); + insert_bytes(&mut meta_sys, SUFFIX_TRANSITION_TIER, value.transition_tier.as_bytes().to_vec()); } if let Some(content_hash) = value.checksum { - meta_sys.insert(format!("{RESERVED_METADATA_PREFIX_LOWER}crc"), content_hash.to_vec()); + insert_bytes(&mut meta_sys, SUFFIX_CRC, content_hash.to_vec()); } Self { @@ -878,15 +1435,16 @@ fn get_internal_replication_state(metadata: &HashMap) -> Option< let mut has = false; for (k, v) in metadata.iter() { - if k == VERSION_PURGE_STATUS_KEY { + if has_internal_suffix(k, SUFFIX_PURGESTATUS) { rs.version_purge_status_internal = Some(v.clone()); rs.purge_targets = version_purge_statuses_map(v.as_str()); has = true; continue; } - if let Some(sub_key) = k.strip_prefix(RESERVED_METADATA_PREFIX_LOWER) { - match sub_key { + let sub_key_opt = strip_internal_prefix(k); + if let Some(ref sub_key) = sub_key_opt { + match sub_key.as_str() { "replica-timestamp" => { has = true; rs.replica_timestamp = Some(OffsetDateTime::parse(v, &Rfc3339).unwrap_or(OffsetDateTime::UNIX_EPOCH)); @@ -929,8 +1487,7 @@ pub struct MetaDeleteMarker { impl MetaDeleteMarker { pub fn free_version(&self) -> bool { - self.meta_sys - .contains_key(format!("{RESERVED_METADATA_PREFIX_LOWER}{FREE_VERSION}").as_str()) + contains_key_bytes(&self.meta_sys, SUFFIX_FREE_VERSION) } pub fn into_fileinfo(&self, volume: &str, path: &str, _all_parts: bool) -> FileInfo { @@ -955,137 +1512,108 @@ impl MetaDeleteMarker { if self.free_version() { fi.set_tier_free_version(); - fi.transition_tier = self - .meta_sys - .get(format!("{RESERVED_METADATA_PREFIX_LOWER}{TRANSITION_TIER}").as_str()) - .map(|v| String::from_utf8_lossy(v).to_string()) + fi.transition_tier = get_bytes(&self.meta_sys, SUFFIX_TRANSITION_TIER) + .map(|v| String::from_utf8_lossy(&v).to_string()) .unwrap_or_default(); - fi.transitioned_objname = self - .meta_sys - .get(format!("{RESERVED_METADATA_PREFIX_LOWER}{TRANSITIONED_OBJECTNAME}").as_str()) - .map(|v| String::from_utf8_lossy(v).to_string()) + fi.transitioned_objname = get_bytes(&self.meta_sys, SUFFIX_TRANSITIONED_OBJECTNAME) + .map(|v| String::from_utf8_lossy(&v).to_string()) .unwrap_or_default(); - fi.transition_version_id = self - .meta_sys - .get(format!("{RESERVED_METADATA_PREFIX_LOWER}{TRANSITIONED_VERSION_ID}").as_str()) + fi.transition_version_id = get_bytes(&self.meta_sys, SUFFIX_TRANSITIONED_VERSION_ID) .map(|v| Uuid::from_slice(v.as_slice()).unwrap_or_default()); } fi } + pub fn encode_to(&self, wr: &mut W) -> Result<()> { + rmp::encode::write_map_len(wr, 3)?; + + // ID + rmp::encode::write_str(wr, "ID")?; + rmp::encode::write_bin(wr, self.version_id.unwrap_or_default().as_bytes())?; + + // MTime Unix timestamp nanos + rmp::encode::write_str(wr, "MTime")?; + let nanos = self.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH).unix_timestamp_nanos(); + rmp::encode::write_sint(wr, nanos as i64)?; + + // MetaSys + rmp::encode::write_str(wr, "MetaSys")?; + rmp::encode::write_map_len(wr, self.meta_sys.len() as u32)?; + for (k, v) in &self.meta_sys { + rmp::encode::write_str(wr, k)?; + rmp::encode::write_bin(wr, v)?; + } + + Ok(()) + } + + pub fn decode_from(&mut self, rd: &mut R) -> Result<()> { + let mut fields = rmp::decode::read_map_len(rd)?; + *self = MetaDeleteMarker::default(); + + while fields > 0 { + fields -= 1; + + let key_len = rmp::decode::read_str_len(rd)?; + let mut key_buf = vec![0u8; key_len as usize]; + rd.read_exact(&mut key_buf)?; + let key = String::from_utf8(key_buf)?; + + match key.as_str() { + "ID" => { + let _ = rmp::decode::read_bin_len(rd)?; + let mut buf = [0u8; 16]; + rd.read_exact(&mut buf)?; + let id = Uuid::from_bytes(buf); + self.version_id = if id.is_nil() { None } else { Some(id) }; + } + "MTime" => { + let nanos: i64 = rmp::decode::read_int(rd)?; + let time = OffsetDateTime::from_unix_timestamp_nanos(nanos as i128)?; + self.mod_time = if time == OffsetDateTime::UNIX_EPOCH { + None + } else { + Some(time) + }; + } + "MetaSys" => { + let len = rmp::decode::read_map_len(rd)? as usize; + self.meta_sys.clear(); + for _ in 0..len { + let k_len = rmp::decode::read_str_len(rd)?; + let mut kbuf = vec![0u8; k_len as usize]; + rd.read_exact(&mut kbuf)?; + let k = String::from_utf8(kbuf)?; + + let blen = rmp::decode::read_bin_len(rd)? as usize; + let mut v = vec![0u8; blen]; + rd.read_exact(&mut v)?; + + self.meta_sys.insert(k, v); + } + } + other => { + return Err(Error::other(format!("unsupported field in MetaDeleteMarker: {other}"))); + } + } + } + + Ok(()) + } + pub fn unmarshal_msg(&mut self, buf: &[u8]) -> Result { - let ret: Self = rmp_serde::from_slice(buf)?; - - *self = ret; - - Ok(buf.len() as u64) - - // let mut cur = Cursor::new(buf); - - // let mut fields_len = rmp::decode::read_map_len(&mut cur)?; - - // while fields_len > 0 { - // fields_len -= 1; - - // let str_len = rmp::decode::read_str_len(&mut cur)?; - - // // !!! Vec::with_capacity(str_len) fails, vec! works normally - // let mut field_buff = vec![0u8; str_len as usize]; - - // cur.read_exact(&mut field_buff)?; - - // let field = String::from_utf8(field_buff)?; - - // match field.as_str() { - // "ID" => { - // rmp::decode::read_bin_len(&mut cur)?; - // let mut buf = [0u8; 16]; - // cur.read_exact(&mut buf)?; - // self.version_id = { - // let id = Uuid::from_bytes(buf); - // if id.is_nil() { None } else { Some(id) } - // }; - // } - - // "MTime" => { - // let unix: i64 = rmp::decode::read_int(&mut cur)?; - // let time = OffsetDateTime::from_unix_timestamp(unix)?; - // if time == OffsetDateTime::UNIX_EPOCH { - // self.mod_time = None; - // } else { - // self.mod_time = Some(time); - // } - // } - // "MetaSys" => { - // let l = rmp::decode::read_map_len(&mut cur)?; - // let mut map = HashMap::new(); - // for _ in 0..l { - // let str_len = rmp::decode::read_str_len(&mut cur)?; - // let mut field_buff = vec![0u8; str_len as usize]; - // cur.read_exact(&mut field_buff)?; - // let key = String::from_utf8(field_buff)?; - - // let blen = rmp::decode::read_bin_len(&mut cur)?; - // let mut val = vec![0u8; blen as usize]; - // cur.read_exact(&mut val)?; - - // map.insert(key, val); - // } - - // self.meta_sys = Some(map); - // } - // name => return Err(Error::other(format!("not support field name {name}"))), - // } - // } - - // Ok(cur.position()) + let mut cur = std::io::Cursor::new(buf); + self.decode_from(&mut cur)?; + Ok(cur.position()) } pub fn marshal_msg(&self) -> Result> { - let buf = rmp_serde::to_vec(self)?; - Ok(buf) - - // let mut len: u32 = 3; - // let mut mask: u8 = 0; - - // if self.meta_sys.is_none() { - // len -= 1; - // mask |= 0x4; - // } - - // let mut wr = Vec::new(); - - // // Field count - // rmp::encode::write_map_len(&mut wr, len)?; - - // // string "ID" - // rmp::encode::write_str(&mut wr, "ID")?; - // rmp::encode::write_bin(&mut wr, self.version_id.unwrap_or_default().as_bytes())?; - - // // string "MTime" - // rmp::encode::write_str(&mut wr, "MTime")?; - // rmp::encode::write_uint( - // &mut wr, - // self.mod_time - // .unwrap_or(OffsetDateTime::UNIX_EPOCH) - // .unix_timestamp() - // .try_into() - // .unwrap(), - // )?; - - // if (mask & 0x4) == 0 { - // let metas = self.meta_sys.as_ref().unwrap(); - // rmp::encode::write_map_len(&mut wr, metas.len() as u32)?; - // for (k, v) in metas { - // rmp::encode::write_str(&mut wr, k.as_str())?; - // rmp::encode::write_bin(&mut wr, v)?; - // } - // } - - // Ok(wr) + let mut wr = Vec::new(); + self.encode_to(&mut wr)?; + Ok(wr) } /// Get delete marker signature diff --git a/crates/filemeta/src/filemeta_inline.rs b/crates/filemeta/src/filemeta_inline.rs index 8d5559a92..fe52055cc 100644 --- a/crates/filemeta/src/filemeta_inline.rs +++ b/crates/filemeta/src/filemeta_inline.rs @@ -181,6 +181,58 @@ impl InlineData { self.serialize(keys, values) } + + pub fn remove_key(&mut self, key: &str) -> Result { + let buf = self.after_version(); + if buf.is_empty() { + return Ok(false); + } + + let mut cur = Cursor::new(buf); + + let mut fields_len = rmp::decode::read_map_len(&mut cur)? as usize; + let mut keys = Vec::with_capacity(fields_len); + let mut values = Vec::with_capacity(fields_len); + let mut found = false; + + while fields_len > 0 { + fields_len -= 1; + + let str_len = rmp::decode::read_str_len(&mut cur)?; + + let mut field_buff = vec![0u8; str_len as usize]; + + cur.read_exact(&mut field_buff)?; + + let find_key = String::from_utf8(field_buff)?; + + let bin_len = rmp::decode::read_bin_len(&mut cur)? as usize; + let start = cur.position() as usize; + let end = start + bin_len; + cur.set_position(end as u64); + + if find_key == key { + found = true; + continue; + } + + keys.push(find_key); + values.push(buf[start..end].to_vec()); + } + + if !found { + return Ok(false); + } + + if keys.is_empty() { + self.0 = Vec::new(); + return Ok(true); + } + + self.serialize(keys, values)?; + Ok(true) + } + pub fn remove(&mut self, remove_keys: Vec) -> Result { let buf = self.after_version(); if buf.is_empty() { diff --git a/crates/filemeta/src/replication.rs b/crates/filemeta/src/replication.rs index 5675ca587..236e820ad 100644 --- a/crates/filemeta/src/replication.rs +++ b/crates/filemeta/src/replication.rs @@ -15,7 +15,7 @@ use bytes::Bytes; use core::fmt; use regex::Regex; -use rustfs_utils::http::RESERVED_METADATA_PREFIX_LOWER; +use rustfs_utils::http::internal_key_rustfs; use serde::{Deserialize, Serialize}; use std::any::Any; use std::collections::HashMap; @@ -859,7 +859,7 @@ pub fn get_replication_state(rinfos: &ReplicatedInfos, prev_state: &ReplicationS } pub fn target_reset_header(arn: &str) -> String { - format!("{RESERVED_METADATA_PREFIX_LOWER}{REPLICATION_RESET}-{arn}") + internal_key_rustfs(&format!("{REPLICATION_RESET}-{arn}")) } #[derive(Debug, Clone, Serialize, Deserialize, Default)] diff --git a/crates/filemeta/src/test_data.rs b/crates/filemeta/src/test_data.rs index e6448c69a..8e3f43861 100644 --- a/crates/filemeta/src/test_data.rs +++ b/crates/filemeta/src/test_data.rs @@ -56,6 +56,7 @@ pub fn create_real_xlmeta() -> Result> { object: Some(object_version), delete_marker: None, write_version: 1, + uses_legacy_checksum: false, }; let shallow_version = FileMetaShallowVersion::try_from(file_version)?; @@ -74,6 +75,7 @@ pub fn create_real_xlmeta() -> Result> { object: None, delete_marker: Some(delete_marker), write_version: 2, + uses_legacy_checksum: false, }; let delete_shallow_version = FileMetaShallowVersion::try_from(delete_file_version)?; @@ -86,6 +88,7 @@ pub fn create_real_xlmeta() -> Result> { object: None, delete_marker: None, write_version: 3, + uses_legacy_checksum: false, }; let mut legacy_shallow = FileMetaShallowVersion::try_from(legacy_version)?; @@ -139,6 +142,7 @@ pub fn create_complex_xlmeta() -> Result> { object: Some(object_version), delete_marker: None, write_version: (i + 1) as u64, + uses_legacy_checksum: false, }; let shallow_version = FileMetaShallowVersion::try_from(file_version)?; @@ -158,6 +162,7 @@ pub fn create_complex_xlmeta() -> Result> { object: None, delete_marker: Some(delete_marker), write_version: (i + 100) as u64, + uses_legacy_checksum: false, }; let delete_shallow_version = FileMetaShallowVersion::try_from(delete_file_version)?; @@ -245,6 +250,7 @@ pub fn create_xlmeta_with_inline_data() -> Result> { object: Some(object_version), delete_marker: None, write_version: 1, + uses_legacy_checksum: false, }; let shallow_version = FileMetaShallowVersion::try_from(file_version)?; diff --git a/crates/heal/tests/heal_integration_test.rs b/crates/heal/tests/heal_integration_test.rs index 509c5cd93..e7ba323ef 100644 --- a/crates/heal/tests/heal_integration_test.rs +++ b/crates/heal/tests/heal_integration_test.rs @@ -38,6 +38,11 @@ use walkdir::WalkDir; const HEAL_FORMAT_WAIT_TIMEOUT: Duration = Duration::from_secs(25); const HEAL_FORMAT_WAIT_INTERVAL: Duration = Duration::from_millis(250); +const NON_INLINE_TEST_DATA_SIZE: usize = 256 * 1024 + 137; + +fn non_inline_test_data() -> Vec { + (0..NON_INLINE_TEST_DATA_SIZE).map(|idx| (idx % 251) as u8).collect() +} async fn wait_for_path_exists(path: &Path, timeout: Duration, interval: Duration) -> bool { let deadline = tokio::time::Instant::now() + timeout; @@ -177,10 +182,10 @@ mod serial_tests { // Create test bucket and object let bucket_name = "test-heal-object-basic"; let object_name = "test-object.txt"; - let test_data = b"Hello, this is test data for healing!"; + let test_data = non_inline_test_data(); create_test_bucket(&ecstore, bucket_name).await; - upload_test_object(&ecstore, bucket_name, object_name, test_data).await; + upload_test_object(&ecstore, bucket_name, object_name, &test_data).await; let _obj_dir = disk_paths[0].join(bucket_name).join(object_name); // ─── 1️⃣ delete single data shard file ───────────────────────────────────── let obj_dir = disk_paths[0].join(bucket_name).join(object_name); @@ -198,55 +203,22 @@ mod serial_tests { assert!(!target_part.exists()); println!("✅ Deleted shard part file: {target_part:?}"); - // Create heal manager with faster interval - let cfg = HealConfig { - heal_interval: Duration::from_millis(1), + let heal_opts = HealOpts { + recreate: true, + remove: false, + update_parity: true, ..Default::default() }; - let heal_manager = HealManager::new(heal_storage.clone(), Some(cfg)); - heal_manager.start().await.unwrap(); - - // Submit heal request for the object - let heal_request = HealRequest::new( - HealType::Object { - bucket: bucket_name.to_string(), - object: object_name.to_string(), - version_id: None, - }, - HealOptions { - dry_run: false, - recursive: false, - remove_corrupted: false, - recreate_missing: true, - scan_mode: HealScanMode::Normal, - update_parity: true, - timeout: Some(Duration::from_secs(300)), - pool_index: None, - set_index: None, - }, - HealPriority::Normal, - ); - - let task_id = heal_manager - .submit_heal_request(heal_request) + let (object_result, object_error) = heal_storage + .heal_object(bucket_name, object_name, None, &heal_opts) .await - .expect("Failed to submit heal request"); + .expect("failed to heal object"); + info!("heal_object result: {:?}, error: {:?}", object_result, object_error); + assert!(object_error.is_none(), "heal_object returned error: {object_error:?}"); - info!("Submitted heal request with task ID: {}", task_id); - - // Wait for task completion - tokio::time::sleep(tokio::time::Duration::from_secs(8)).await; - - // Attempt to fetch task status (might be removed if finished) - match heal_manager.get_task_status(&task_id).await { - Ok(status) => info!("Task status: {:?}", status), - Err(e) => info!("Task status not found (likely completed): {}", e), - } - - // ─── 2️⃣ verify each part file is restored ─────── - assert!(target_part.exists()); - - // ─── 3️⃣ verify object data integrity by actually reading it ─────── + // `test_heal_format_with_data` covers on-disk shard restoration. Here we + // focus on the object-level healing contract: the object must remain + // readable with intact contents after healing. let mut reader = ecstore .get_object_reader(bucket_name, object_name, None, HeaderMap::new(), &ObjectOptions::default()) .await @@ -364,10 +336,10 @@ mod serial_tests { // Create test bucket and object let bucket_name = "test-heal-format-with-data"; let object_name = "test-object.txt"; - let test_data = b"Hello, this is test data for healing!"; + let test_data = non_inline_test_data(); create_test_bucket(&ecstore, bucket_name).await; - upload_test_object(&ecstore, bucket_name, object_name, test_data).await; + upload_test_object(&ecstore, bucket_name, object_name, &test_data).await; let obj_dir = disk_paths[0].join(bucket_name).join(object_name); let target_part = WalkDir::new(&obj_dir) .min_depth(2) diff --git a/crates/iam/src/manager.rs b/crates/iam/src/manager.rs index 2b4363dfb..0e95639f2 100644 --- a/crates/iam/src/manager.rs +++ b/crates/iam/src/manager.rs @@ -32,7 +32,7 @@ use rustfs_policy::{ format::Format, policy::{Policy, PolicyDoc, default::DEFAULT_POLICIES, iam_policy_claim_name_sa}, }; -use rustfs_utils::path::path_join_buf; +use rustfs_utils::{get_env_opt_str, path::path_join_buf}; use serde::{Deserialize, Serialize}; use serde_json::Value; use std::sync::atomic::AtomicU8; @@ -147,7 +147,7 @@ where // Background ticker for synchronization // Check if environment variable is set - let skip_background_task = std::env::var("RUSTFS_SKIP_BACKGROUND_TASK").is_ok(); + let skip_background_task = get_env_opt_str("RUSTFS_SKIP_BACKGROUND_TASK").is_some(); if !skip_background_task { // Background thread starts periodic updates or receives signal updates @@ -595,7 +595,11 @@ where Ok(users .values() .filter_map(|x| { - if !access_key.is_empty() && x.credentials.parent_user.as_str() == access_key && x.credentials.is_temp() { + if !access_key.is_empty() + && x.credentials.parent_user.as_str() == access_key + && x.credentials.is_temp() + && !x.credentials.is_service_account() + { let mut c = x.credentials.clone(); c.secret_key = String::new(); c.session_token = String::new(); @@ -1419,9 +1423,6 @@ where } pub async fn get_group_description(&self, name: &str) -> Result { - let (ps, updated_at) = self.policy_db_get_internal(name, true, false).await?; - let policy = ps.join(","); - let gi = self .cache .groups @@ -1430,13 +1431,25 @@ where .cloned() .ok_or(Error::NoSuchGroup(name.to_string()))?; - Ok(GroupDesc { - name: name.to_string(), - policy, - members: gi.members, - updated_at: Some(updated_at), - status: gi.status, - }) + let mapped_policy = if let Some(policy) = self.cache.group_policies.load().get(name).cloned() { + Some(policy) + } else { + let mut policies = HashMap::new(); + if let Err(err) = self.api.load_mapped_policy(name, UserType::Reg, true, &mut policies).await + && !is_err_no_such_policy(&err) + { + return Err(err); + } + + if let Some(policy) = policies.get(name).cloned() { + Cache::add_or_update(&self.cache.group_policies, name, &policy, OffsetDateTime::now_utc()); + Some(policy) + } else { + None + } + }; + + Ok(build_group_desc(name, gi, mapped_policy)) } pub async fn list_groups(&self) -> Result> { @@ -1882,6 +1895,20 @@ fn filter_policies(cache: &Cache, policy_name: &str, bucket_name: &str) -> (Stri (policies.join(","), Policy::merge_policies(to_merge)) } +fn build_group_desc(name: &str, group_info: GroupInfo, mapped_policy: Option) -> GroupDesc { + let (policy, updated_at) = mapped_policy + .map(|policy| (policy.policies, Some(policy.update_at))) + .unwrap_or_else(|| (String::new(), Some(OffsetDateTime::now_utc()))); + + GroupDesc { + name: name.to_string(), + policy, + members: group_info.members, + updated_at, + status: group_info.status, + } +} + #[cfg(test)] mod tests { use super::*; @@ -2226,4 +2253,22 @@ mod tests { assert!(merged.statements.is_empty()); assert!(merged.is_empty()); } + + #[test] + fn test_build_group_desc_preserves_policy_for_disabled_group() { + let group_info = GroupInfo { + status: STATUS_DISABLED.to_string(), + members: vec!["alice".to_string()], + ..Default::default() + }; + let mapped_policy = MappedPolicy::new("readonly"); + + let desc = build_group_desc("ops", group_info, Some(mapped_policy)); + + assert_eq!(desc.name, "ops"); + assert_eq!(desc.status, STATUS_DISABLED); + assert_eq!(desc.policy, "readonly"); + assert_eq!(desc.members, vec!["alice".to_string()]); + assert!(desc.updated_at.is_some()); + } } diff --git a/crates/iam/src/store.rs b/crates/iam/src/store.rs index 9d26bde6f..9e189b99b 100644 --- a/crates/iam/src/store.rs +++ b/crates/iam/src/store.rs @@ -113,7 +113,11 @@ impl UserType { #[derive(Serialize, Deserialize, Clone)] pub struct MappedPolicy { pub version: i64, + /// policy, legacy: policies. Serialize as policy. + #[serde(rename = "policy", alias = "policies")] pub policies: String, + /// updatedAt (RFC3339), legacy: update_at. Serialize as updatedAt. + #[serde(rename = "updatedAt", alias = "update_at", with = "rustfs_policy::serde_datetime")] pub update_at: OffsetDateTime, } @@ -158,6 +162,13 @@ pub struct GroupInfo { pub version: i64, pub status: String, pub members: Vec, + /// updatedAt (RFC3339), legacy: update_at. Serialize as updatedAt. + #[serde( + rename = "updatedAt", + alias = "update_at", + default, + with = "rustfs_policy::serde_datetime::option" + )] pub update_at: Option, } @@ -171,3 +182,46 @@ impl GroupInfo { } } } + +#[cfg(test)] +mod tests { + use super::{GroupInfo, MappedPolicy}; + + /// uses RFC3339 for updatedAt. MappedPolicy must serialize as RFC3339. + #[test] + fn test_mapped_policy_timestamps_serialize_as_rfc3339() { + let mp = MappedPolicy::new("readwrite"); + let json = serde_json::to_string(&mp).expect("serialize"); + assert!(json.contains('T'), "MappedPolicy updatedAt should be RFC3339; got: {}", json); + assert!( + json.contains('Z') || json.contains("+00:00"), + "MappedPolicy updatedAt should be RFC3339; got: {}", + json + ); + } + + /// Deserialize MappedPolicy from JSON (RFC3339 updatedAt). + #[test] + fn test_mapped_policy_deserialize_minio_style_rfc3339() { + let minio_style = r#"{"version":1,"policy":"readwrite","updatedAt":"2025-03-07T12:00:00Z"}"#; + let mp: MappedPolicy = serde_json::from_str(minio_style).expect("deserialize"); + assert_eq!(mp.policies, "readwrite"); + } + + /// GroupInfo updatedAt: uses RFC3339. + #[test] + fn test_group_info_timestamps_serialize_as_rfc3339() { + let g = GroupInfo::new(vec!["u1".to_string()]); + let json = serde_json::to_string(&g).expect("serialize"); + assert!(json.contains('T'), "GroupInfo updatedAt should be RFC3339; got: {}", json); + } + + /// Deserialize GroupInfo from JSON (RFC3339 updatedAt). + #[test] + fn test_group_info_deserialize_minio_style_rfc3339() { + let minio_style = r#"{"version":1,"status":"enabled","members":["u1"],"updatedAt":"2025-03-07T12:00:00Z"}"#; + let g: GroupInfo = serde_json::from_str(minio_style).expect("deserialize"); + assert_eq!(g.members, ["u1"]); + assert!(g.update_at.is_some()); + } +} diff --git a/crates/iam/src/store/object.rs b/crates/iam/src/store/object.rs index ad4704943..4f8327ee1 100644 --- a/crates/iam/src/store/object.rs +++ b/crates/iam/src/store/object.rs @@ -129,15 +129,55 @@ impl ObjectStore { } fn decrypt_data(data: &[u8]) -> Result> { - let de = rustfs_crypto::decrypt_data(get_global_action_cred().unwrap_or_default().secret_key.as_bytes(), data)?; - Ok(de) + if Self::is_plaintext_json(data) { + return Ok(data.to_vec()); + } + + let cred = get_global_action_cred().unwrap_or_default(); + let secret_key = cred.secret_key; + let mut keys: Vec<(Vec, bool)> = vec![(secret_key.clone().into_bytes(), false)]; + if !cred.access_key.is_empty() && !secret_key.is_empty() { + keys.push((format!("{}:{secret_key}", cred.access_key).into_bytes(), true)); + } + + const STREAM_IO_HEADER_LEN: usize = 41; + let mut last_err = None; + for (key, is_access_secret) in keys { + if is_access_secret + && data.len() >= STREAM_IO_HEADER_LEN + && let Ok(plain) = rustfs_crypto::decrypt_stream_io(&key, data) + { + return Ok(plain); + } + match rustfs_crypto::decrypt_data(&key, data) { + Ok(plain) => return Ok(plain), + Err(err) => last_err = Some(err), + } + } + + Err(last_err.unwrap_or(rustfs_crypto::Error::ErrUnexpectedHeader).into()) } fn encrypt_data(data: &[u8]) -> Result> { - let en = rustfs_crypto::encrypt_data(get_global_action_cred().unwrap_or_default().secret_key.as_bytes(), data)?; + let cred = get_global_action_cred().unwrap_or_default(); + let password = if !cred.access_key.is_empty() && !cred.secret_key.is_empty() { + format!("{}:{}", cred.access_key, cred.secret_key).into_bytes() + } else { + cred.secret_key.clone().into_bytes() + }; + let en = rustfs_crypto::encrypt_stream_io(&password, data)?; Ok(en) } + fn is_plaintext_json(data: &[u8]) -> bool { + std::str::from_utf8(data).is_ok() && serde_json::from_slice::(data).is_ok() + } + + #[cfg(test)] + fn encrypt_data_for_test(data: &[u8]) -> Result> { + Self::encrypt_data(data) + } + async fn load_iamconfig_bytes_with_metadata(&self, path: impl AsRef + Send) -> Result<(Vec, ObjectInfo)> { let (data, obj) = read_config_with_metadata(self.object_api.clone(), path.as_ref(), &ObjectOptions::default()).await?; @@ -1164,3 +1204,83 @@ impl Store for ObjectStore { // Ok(()) // } } + +#[cfg(test)] +mod tests { + use super::ObjectStore; + use rustfs_credentials::{Credentials, get_global_action_cred, init_global_action_credentials}; + + fn test_cred() -> Credentials { + if let Some(cred) = get_global_action_cred() { + return cred; + } + let _ = init_global_action_credentials(Some("COMPATTESTAK".to_string()), Some("COMPATTESTSK1234567890".to_string())); + get_global_action_cred().unwrap_or_default() + } + + #[test] + fn test_decrypt_data_accepts_plaintext_json() { + let raw = br#"{"Version":1,"policy":"readonly"}"#; + let out = ObjectStore::decrypt_data(raw).expect("plaintext json should pass through"); + assert_eq!(out, raw); + } + + #[test] + fn test_decrypt_data_accepts_rustfs_legacy_secret_encryption() { + let cred = test_cred(); + let plain = br#"{"accessKey":"ak","secretKey":"sk"}"#; + let encrypted = rustfs_crypto::encrypt_data(cred.secret_key.as_bytes(), plain).expect("encrypt with rustfs secret"); + let out = ObjectStore::decrypt_data(&encrypted).expect("decrypt rustfs legacy encryption"); + assert_eq!(out, plain); + } + + #[test] + fn test_decrypt_data_accepts_access_secret_encryption() { + let cred = test_cred(); + let plain = br#"{"Version":1,"updatedAt":"2025-03-07T12:00:00Z"}"#; + let root_cred = format!("{}:{}", cred.access_key, cred.secret_key); + let encrypted = rustfs_crypto::encrypt_stream_io(root_cred.as_bytes(), plain).expect("encrypt with stream_io"); + let out = ObjectStore::decrypt_data(&encrypted).expect("decrypt stream_io"); + assert_eq!(out, plain); + } + + #[test] + fn test_decrypt_data_corrupt_stream_io_fails() { + let cred = test_cred(); + let plain = br#"{"Version":1}"#; + let root_cred = format!("{}:{}", cred.access_key, cred.secret_key); + let mut encrypted = rustfs_crypto::encrypt_stream_io(root_cred.as_bytes(), plain).expect("encrypt with stream_io"); + if encrypted.len() > 50 { + encrypted[50] ^= 0xFF; // corrupt one byte + } + let result = ObjectStore::decrypt_data(&encrypted); + assert!(result.is_err(), "corrupt stream_io data should fail decrypt"); + } + + #[test] + fn test_decrypt_data_short_data_fails() { + let short = &[0x00u8; 40]; // less than 41-byte stream_io header, not valid JSON + let result = ObjectStore::decrypt_data(short); + assert!(result.is_err(), "short non-JSON data should fail decrypt"); + } + + #[test] + fn test_encrypt_data_produces_stream_io_format() { + let _ = test_cred(); + let plain = br#"{"Version":1,"policy":"readonly"}"#; + let encrypted = ObjectStore::encrypt_data_for_test(plain).expect("encrypt should succeed"); + // stream_io header: salt(32) + alg_id(1) + nonce_prefix(8) = 41 bytes + const STREAM_IO_HEADER_LEN: usize = 41; + assert!( + encrypted.len() >= STREAM_IO_HEADER_LEN, + "encrypted should have at least 41-byte stream_io header" + ); + assert!( + encrypted[32] == 0x00 || encrypted[32] == 0x01 || encrypted[32] == 0x02, + "alg_id should be 0x00, 0x01, or 0x02" + ); + // Round-trip: encrypt then decrypt + let decrypted = ObjectStore::decrypt_data(&encrypted).expect("decrypt should succeed"); + assert_eq!(plain, decrypted.as_slice()); + } +} diff --git a/crates/iam/src/sys.rs b/crates/iam/src/sys.rs index 1924971f5..1fa74e0ed 100644 --- a/crates/iam/src/sys.rs +++ b/crates/iam/src/sys.rs @@ -361,10 +361,6 @@ impl IamSys { return Err(IamError::IAMActionNotAllowed); } - if opts.expiration.is_none() { - return Err(IamError::InvalidExpiration); - } - // TODO: check allow_site_replicator_account let policy_buf = if let Some(policy) = opts.session_policy { @@ -619,6 +615,7 @@ impl IamSys { } let updated_at = self.store.add_user(access_key, args).await?; + self.load_user(access_key, UserType::Reg).await?; self.notify_for_user(access_key, false).await; diff --git a/crates/keystone/Cargo.toml b/crates/keystone/Cargo.toml index ef1b4659b..e43b26777 100644 --- a/crates/keystone/Cargo.toml +++ b/crates/keystone/Cargo.toml @@ -36,6 +36,7 @@ time = { workspace = true } moka = { workspace = true } rustfs-credentials = { workspace = true } rustfs-policy = { workspace = true } +rustfs-utils = { workspace = true } # Middleware dependencies tower = { workspace = true } http = { workspace = true } @@ -50,6 +51,7 @@ tokio = { workspace = true, features = ["test-util"] } tower = { workspace = true, features = ["util"] } hyper = { workspace = true, features = ["server"] } serde_json = { workspace = true } +temp-env = { workspace = true } [[test]] name = "integration" diff --git a/crates/keystone/src/config.rs b/crates/keystone/src/config.rs index 61c80b08d..9f69532cb 100644 --- a/crates/keystone/src/config.rs +++ b/crates/keystone/src/config.rs @@ -13,6 +13,7 @@ // limitations under the License. use crate::{KeystoneError, KeystoneVersion, Result}; +use rustfs_utils::{get_env_bool, get_env_opt_str, get_env_str, get_env_u64}; use serde::{Deserialize, Serialize}; use std::time::Duration; @@ -80,59 +81,35 @@ pub struct RoleMapping { impl KeystoneConfig { /// Load configuration from environment variables pub fn from_env() -> Result { - let enable = std::env::var("RUSTFS_KEYSTONE_ENABLE") - .unwrap_or_else(|_| "false".to_string()) - .parse() - .unwrap_or(false); + let enable = get_env_bool("RUSTFS_KEYSTONE_ENABLE", false); if !enable { return Ok(Self::default()); } - let auth_url = std::env::var("RUSTFS_KEYSTONE_AUTH_URL") - .map_err(|_| KeystoneError::ConfigError("RUSTFS_KEYSTONE_AUTH_URL not set".to_string()))?; + let auth_url = get_env_opt_str("RUSTFS_KEYSTONE_AUTH_URL") + .ok_or_else(|| KeystoneError::ConfigError("RUSTFS_KEYSTONE_AUTH_URL not set".to_string()))?; - let version = std::env::var("RUSTFS_KEYSTONE_VERSION").unwrap_or_else(|_| "v3".to_string()); + let version = get_env_str("RUSTFS_KEYSTONE_VERSION", "v3"); - let admin_user = std::env::var("RUSTFS_KEYSTONE_ADMIN_USER").ok(); - let admin_password = std::env::var("RUSTFS_KEYSTONE_ADMIN_PASSWORD").ok(); - let admin_project = std::env::var("RUSTFS_KEYSTONE_ADMIN_PROJECT").ok(); - let admin_domain = std::env::var("RUSTFS_KEYSTONE_ADMIN_DOMAIN").ok(); + let admin_user = get_env_opt_str("RUSTFS_KEYSTONE_ADMIN_USER"); + let admin_password = get_env_opt_str("RUSTFS_KEYSTONE_ADMIN_PASSWORD"); + let admin_project = get_env_opt_str("RUSTFS_KEYSTONE_ADMIN_PROJECT"); + let admin_domain = get_env_opt_str("RUSTFS_KEYSTONE_ADMIN_DOMAIN"); - let verify_ssl = std::env::var("RUSTFS_KEYSTONE_VERIFY_SSL") - .unwrap_or_else(|_| "true".to_string()) - .parse() - .unwrap_or(true); + let verify_ssl = get_env_bool("RUSTFS_KEYSTONE_VERIFY_SSL", true); - let enable_cache = std::env::var("RUSTFS_KEYSTONE_ENABLE_CACHE") - .unwrap_or_else(|_| "true".to_string()) - .parse() - .unwrap_or(true); + let enable_cache = get_env_bool("RUSTFS_KEYSTONE_ENABLE_CACHE", true); - let cache_size = std::env::var("RUSTFS_KEYSTONE_CACHE_SIZE") - .unwrap_or_else(|_| "10000".to_string()) - .parse() - .unwrap_or(10000); + let cache_size = get_env_u64("RUSTFS_KEYSTONE_CACHE_SIZE", 10000); - let cache_ttl_seconds = std::env::var("RUSTFS_KEYSTONE_CACHE_TTL") - .unwrap_or_else(|_| "300".to_string()) - .parse() - .unwrap_or(300); + let cache_ttl_seconds = get_env_u64("RUSTFS_KEYSTONE_CACHE_TTL", 300); - let enable_tenant_prefix = std::env::var("RUSTFS_KEYSTONE_TENANT_PREFIX") - .unwrap_or_else(|_| "true".to_string()) - .parse() - .unwrap_or(true); + let enable_tenant_prefix = get_env_bool("RUSTFS_KEYSTONE_TENANT_PREFIX", true); - let implicit_tenants = std::env::var("RUSTFS_KEYSTONE_IMPLICIT_TENANTS") - .unwrap_or_else(|_| "true".to_string()) - .parse() - .unwrap_or(true); + let implicit_tenants = get_env_bool("RUSTFS_KEYSTONE_IMPLICIT_TENANTS", true); - let timeout_seconds = std::env::var("RUSTFS_KEYSTONE_TIMEOUT") - .unwrap_or_else(|_| "30".to_string()) - .parse() - .unwrap_or(30); + let timeout_seconds = get_env_u64("RUSTFS_KEYSTONE_TIMEOUT", 30); Ok(Self { enable, @@ -224,6 +201,7 @@ impl Default for KeystoneConfig { #[cfg(test)] mod tests { use super::*; + use temp_env::with_vars; #[test] fn test_default_config() { @@ -248,4 +226,44 @@ mod tests { config.version = "invalid".to_string(); assert!(config.get_version().is_err()); } + + #[test] + fn test_from_env_reads_configuration() { + with_vars( + vec![ + ("RUSTFS_KEYSTONE_ENABLE", Some("true")), + ("RUSTFS_KEYSTONE_AUTH_URL", Some("https://keystone.example.com")), + ("RUSTFS_KEYSTONE_VERSION", Some("v2.0")), + ("RUSTFS_KEYSTONE_ADMIN_USER", Some("admin")), + ("RUSTFS_KEYSTONE_ADMIN_PASSWORD", Some("secret")), + ("RUSTFS_KEYSTONE_ADMIN_PROJECT", Some("service")), + ("RUSTFS_KEYSTONE_ADMIN_DOMAIN", Some("Default")), + ("RUSTFS_KEYSTONE_VERIFY_SSL", Some("false")), + ("RUSTFS_KEYSTONE_ENABLE_CACHE", Some("false")), + ("RUSTFS_KEYSTONE_CACHE_SIZE", Some("2048")), + ("RUSTFS_KEYSTONE_CACHE_TTL", Some("900")), + ("RUSTFS_KEYSTONE_TENANT_PREFIX", Some("false")), + ("RUSTFS_KEYSTONE_IMPLICIT_TENANTS", Some("false")), + ("RUSTFS_KEYSTONE_TIMEOUT", Some("99")), + ], + || { + let config = KeystoneConfig::from_env().expect("keystone config should load from env"); + + assert!(config.enable); + assert_eq!(config.auth_url, "https://keystone.example.com"); + assert_eq!(config.version, "v2.0"); + assert_eq!(config.admin_user.as_deref(), Some("admin")); + assert_eq!(config.admin_password.as_deref(), Some("secret")); + assert_eq!(config.admin_project.as_deref(), Some("service")); + assert_eq!(config.admin_domain.as_deref(), Some("Default")); + assert!(!config.verify_ssl); + assert!(!config.enable_cache); + assert_eq!(config.cache_size, 2048); + assert_eq!(config.cache_ttl_seconds, 900); + assert!(!config.enable_tenant_prefix); + assert!(!config.implicit_tenants); + assert_eq!(config.timeout_seconds, 99); + }, + ); + } } diff --git a/crates/kms/Cargo.toml b/crates/kms/Cargo.toml index b22f12718..f3afe9f33 100644 --- a/crates/kms/Cargo.toml +++ b/crates/kms/Cargo.toml @@ -56,6 +56,7 @@ moka = { workspace = true, features = ["future"] } # Additional dependencies md5 = { workspace = true } arc-swap = { workspace = true } +rustfs-utils = { workspace = true } # HTTP client for Vault reqwest = { workspace = true } @@ -63,6 +64,7 @@ vaultrs = { workspace = true } [dev-dependencies] tempfile = { workspace = true } +temp-env = { workspace = true } [features] default = [] diff --git a/crates/kms/src/config.rs b/crates/kms/src/config.rs index c7b28416c..177ccb473 100644 --- a/crates/kms/src/config.rs +++ b/crates/kms/src/config.rs @@ -15,6 +15,7 @@ //! KMS configuration management use crate::error::{KmsError, Result}; +use rustfs_utils::{get_env_bool, get_env_opt_str, get_env_str}; use serde::{Deserialize, Serialize}; use std::path::PathBuf; use std::time::Duration; @@ -304,7 +305,7 @@ impl KmsConfig { let mut config = Self::default(); // Backend type - if let Ok(backend_type) = std::env::var("RUSTFS_KMS_BACKEND") { + if let Some(backend_type) = get_env_opt_str("RUSTFS_KMS_BACKEND") { config.backend = match backend_type.to_lowercase().as_str() { "local" => KmsBackend::Local, "vault" => KmsBackend::Vault, @@ -313,12 +314,12 @@ impl KmsConfig { } // Default key ID - if let Ok(key_id) = std::env::var("RUSTFS_KMS_DEFAULT_KEY_ID") { + if let Some(key_id) = get_env_opt_str("RUSTFS_KMS_DEFAULT_KEY_ID") { config.default_key_id = Some(key_id); } // Timeout - if let Ok(timeout_str) = std::env::var("RUSTFS_KMS_TIMEOUT_SECS") { + if let Some(timeout_str) = get_env_opt_str("RUSTFS_KMS_TIMEOUT_SECS") { let timeout_secs = timeout_str .parse::() .map_err(|_| KmsError::configuration_error("Invalid timeout value"))?; @@ -326,22 +327,20 @@ impl KmsConfig { } // Retry attempts - if let Ok(retries_str) = std::env::var("RUSTFS_KMS_RETRY_ATTEMPTS") { + if let Some(retries_str) = get_env_opt_str("RUSTFS_KMS_RETRY_ATTEMPTS") { config.retry_attempts = retries_str .parse() .map_err(|_| KmsError::configuration_error("Invalid retry attempts value"))?; } // Enable cache - if let Ok(cache_str) = std::env::var("RUSTFS_KMS_ENABLE_CACHE") { - config.enable_cache = cache_str.parse().unwrap_or(true); - } + config.enable_cache = get_env_bool("RUSTFS_KMS_ENABLE_CACHE", config.enable_cache); // Backend-specific configuration match config.backend { KmsBackend::Local => { - let key_dir = std::env::var("RUSTFS_KMS_LOCAL_KEY_DIR").unwrap_or_else(|_| "./kms_keys".to_string()); - let master_key = std::env::var("RUSTFS_KMS_LOCAL_MASTER_KEY").ok(); + let key_dir = get_env_str("RUSTFS_KMS_LOCAL_KEY_DIR", "./kms_keys"); + let master_key = get_env_opt_str("RUSTFS_KMS_LOCAL_MASTER_KEY"); config.backend_config = BackendConfig::Local(LocalConfig { key_dir: PathBuf::from(key_dir), @@ -350,17 +349,16 @@ impl KmsConfig { }); } KmsBackend::Vault => { - let address = std::env::var("RUSTFS_KMS_VAULT_ADDRESS").unwrap_or_else(|_| "http://localhost:8200".to_string()); - let token = std::env::var("RUSTFS_KMS_VAULT_TOKEN").unwrap_or_else(|_| "dev-token".to_string()); + let address = get_env_str("RUSTFS_KMS_VAULT_ADDRESS", "http://localhost:8200"); + let token = get_env_str("RUSTFS_KMS_VAULT_TOKEN", "dev-token"); config.backend_config = BackendConfig::Vault(Box::new(VaultConfig { address, auth_method: VaultAuthMethod::Token { token }, - namespace: std::env::var("RUSTFS_KMS_VAULT_NAMESPACE").ok(), - mount_path: std::env::var("RUSTFS_KMS_VAULT_MOUNT_PATH").unwrap_or_else(|_| "transit".to_string()), - kv_mount: std::env::var("RUSTFS_KMS_VAULT_KV_MOUNT").unwrap_or_else(|_| "secret".to_string()), - key_path_prefix: std::env::var("RUSTFS_KMS_VAULT_KEY_PREFIX") - .unwrap_or_else(|_| "rustfs/kms/keys".to_string()), + namespace: get_env_opt_str("RUSTFS_KMS_VAULT_NAMESPACE"), + mount_path: get_env_str("RUSTFS_KMS_VAULT_MOUNT_PATH", "transit"), + kv_mount: get_env_str("RUSTFS_KMS_VAULT_KV_MOUNT", "secret"), + key_path_prefix: get_env_str("RUSTFS_KMS_VAULT_KEY_PREFIX", "rustfs/kms/keys"), tls: None, })); } @@ -374,6 +372,7 @@ impl KmsConfig { #[cfg(test)] mod tests { use super::*; + use temp_env::with_vars; use tempfile::TempDir; #[test] @@ -423,4 +422,39 @@ mod tests { config.retry_attempts = 0; assert!(config.validate().is_err()); } + + #[test] + fn test_from_env_reads_vault_settings() { + with_vars( + vec![ + ("RUSTFS_KMS_BACKEND", Some("vault")), + ("RUSTFS_KMS_DEFAULT_KEY_ID", Some("tenant-key")), + ("RUSTFS_KMS_TIMEOUT_SECS", Some("42")), + ("RUSTFS_KMS_RETRY_ATTEMPTS", Some("7")), + ("RUSTFS_KMS_ENABLE_CACHE", Some("false")), + ("RUSTFS_KMS_VAULT_ADDRESS", Some("https://vault.example.com")), + ("RUSTFS_KMS_VAULT_TOKEN", Some("vault-token")), + ("RUSTFS_KMS_VAULT_NAMESPACE", Some("tenant-a")), + ("RUSTFS_KMS_VAULT_MOUNT_PATH", Some("transit-alt")), + ("RUSTFS_KMS_VAULT_KV_MOUNT", Some("secret-alt")), + ("RUSTFS_KMS_VAULT_KEY_PREFIX", Some("tenant/keys")), + ], + || { + let config = KmsConfig::from_env().expect("kms config should load from env"); + + assert_eq!(config.backend, KmsBackend::Vault); + assert_eq!(config.default_key_id.as_deref(), Some("tenant-key")); + assert_eq!(config.timeout, Duration::from_secs(42)); + assert_eq!(config.retry_attempts, 7); + assert!(!config.enable_cache); + + let vault = config.vault_config().expect("vault backend config"); + assert_eq!(vault.address, "https://vault.example.com"); + assert_eq!(vault.namespace.as_deref(), Some("tenant-a")); + assert_eq!(vault.mount_path, "transit-alt"); + assert_eq!(vault.kv_mount, "secret-alt"); + assert_eq!(vault.key_path_prefix, "tenant/keys"); + }, + ); + } } diff --git a/crates/madmin/src/group.rs b/crates/madmin/src/group.rs index 64aad6813..3821af16b 100644 --- a/crates/madmin/src/group.rs +++ b/crates/madmin/src/group.rs @@ -13,10 +13,11 @@ // limitations under the License. use serde::Deserialize; +use serde::Deserializer; use serde::Serialize; use time::OffsetDateTime; -#[derive(Debug, Serialize, Deserialize, Default)] +#[derive(Debug, Serialize, Default, PartialEq, Eq)] #[serde(rename_all = "lowercase")] pub enum GroupStatus { #[default] @@ -24,6 +25,20 @@ pub enum GroupStatus { Disabled, } +impl<'de> Deserialize<'de> for GroupStatus { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let value = String::deserialize(deserializer)?; + match value.as_str() { + "" | "enabled" => Ok(Self::Enabled), + "disabled" => Ok(Self::Disabled), + _ => Err(serde::de::Error::unknown_variant(&value, &["enabled", "disabled"])), + } + } +} + #[derive(Debug, Serialize, Deserialize, Default)] pub struct GroupAddRemove { pub group: String, @@ -40,6 +55,40 @@ pub struct GroupDesc { pub status: String, pub members: Vec, pub policy: String, - #[serde(rename = "updatedAt", skip_serializing_if = "Option::is_none")] + #[serde( + rename = "updatedAt", + skip_serializing_if = "Option::is_none", + with = "time::serde::rfc3339::option" + )] pub updated_at: Option, } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn group_desc_updated_at_serializes_as_rfc3339() { + let now = OffsetDateTime::now_utc().replace_nanosecond(0).unwrap(); + let group = GroupDesc { + name: "group-a".to_string(), + status: "enabled".to_string(), + members: vec!["user-a".to_string()], + policy: "readwrite".to_string(), + updated_at: Some(now), + }; + + let json = serde_json::to_string(&group).unwrap(); + let decoded: GroupDesc = serde_json::from_str(&json).unwrap(); + + assert!(json.contains("\"updatedAt\":\"")); + assert!(json.contains('T')); + assert_eq!(decoded.updated_at, Some(now)); + } + + #[test] + fn group_status_accepts_empty_string_as_enabled() { + let status: GroupStatus = serde_json::from_str(r#""""#).unwrap(); + assert_eq!(status, GroupStatus::Enabled); + } +} diff --git a/crates/madmin/src/user.rs b/crates/madmin/src/user.rs index 323f3d7d0..e98c2c23b 100644 --- a/crates/madmin/src/user.rs +++ b/crates/madmin/src/user.rs @@ -13,6 +13,7 @@ // limitations under the License. use serde::{Deserialize, Deserializer, Serialize, Serializer}; +use serde_json::Value; use serde_json::value::RawValue; use std::collections::HashMap; use time::OffsetDateTime; @@ -88,7 +89,7 @@ pub struct UserInfo { #[serde(rename = "memberOf", skip_serializing_if = "Option::is_none")] pub member_of: Option>, - #[serde(rename = "updatedAt")] + #[serde(rename = "updatedAt", with = "time::serde::rfc3339::option")] pub updated_at: Option, } @@ -134,47 +135,61 @@ pub struct ListServiceAccountsResp { pub accounts: Vec, } +#[derive(Debug, Serialize, Deserialize, Default)] +pub struct ListAccessKeysResp { + #[serde(rename = "serviceAccounts", default)] + pub service_accounts: Vec, + #[serde(rename = "stsKeys", default)] + pub sts_keys: Vec, +} + +pub const ACCESS_KEY_LIST_USERS_ONLY: &str = "users-only"; +pub const ACCESS_KEY_LIST_STS_ONLY: &str = "sts-only"; +pub const ACCESS_KEY_LIST_SVCACC_ONLY: &str = "svcacc-only"; +pub const ACCESS_KEY_LIST_ALL: &str = "all"; + #[derive(Debug, Serialize, Deserialize)] pub struct AddServiceAccountReq { - #[serde(rename = "policy", skip_serializing_if = "Option::is_none")] - pub policy: Option, + #[serde( + rename = "policy", + skip_serializing_if = "Option::is_none", + default, + deserialize_with = "deserialize_optional_policy_value" + )] + pub policy: Option, #[serde(rename = "targetUser", skip_serializing_if = "Option::is_none")] pub target_user: Option, - #[serde(rename = "accessKey")] + #[serde(rename = "accessKey", default)] pub access_key: String, - #[serde(rename = "secretKey")] + #[serde(rename = "secretKey", default)] pub secret_key: String, - #[serde(rename = "name")] + #[serde(rename = "name", skip_serializing_if = "Option::is_none")] pub name: Option, #[serde(rename = "description", skip_serializing_if = "Option::is_none")] pub description: Option, - #[serde(rename = "expiration", with = "time::serde::rfc3339::option")] + #[serde( + rename = "expiration", + skip_serializing_if = "Option::is_none", + default, + with = "time::serde::rfc3339::option" + )] pub expiration: Option, + + #[serde(rename = "comment", skip_serializing_if = "Option::is_none")] + pub comment: Option, } impl AddServiceAccountReq { pub fn validate(&self) -> Result<(), String> { - if self.access_key.is_empty() { - return Err("accessKey is empty".to_string()); - } - - if self.secret_key.is_empty() { - return Err("secretKey is empty".to_string()); - } - - if self.name.is_none() { - return Err("name is empty".to_string()); - } - - // TODO: validate - - Ok(()) + validate_service_account_name(self.name.as_deref())?; + validate_service_account_description(self.description.as_deref().or(self.comment.as_deref()))?; + validate_service_account_expiration(self.expiration) } } @@ -195,7 +210,7 @@ pub struct AddServiceAccountResp<'a> { pub credentials: Credentials<'a>, } -#[derive(Serialize)] +#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)] #[serde(rename_all = "camelCase")] pub struct InfoServiceAccountResp { pub parent_user: String, @@ -213,10 +228,70 @@ pub struct InfoServiceAccountResp { pub expiration: Option, } +pub type TemporaryAccountInfoResp = InfoServiceAccountResp; + +#[derive(Debug, Serialize, Deserialize, Default, PartialEq, Eq)] +pub struct LDAPSpecificAccessKeyInfo { + #[serde(rename = "username", skip_serializing_if = "Option::is_none")] + pub username: Option, +} + +impl LDAPSpecificAccessKeyInfo { + pub fn is_empty(&self) -> bool { + self.username.is_none() + } +} + +#[derive(Debug, Serialize, Deserialize, Default, PartialEq, Eq)] +pub struct OpenIDSpecificAccessKeyInfo { + #[serde(rename = "configName", skip_serializing_if = "Option::is_none")] + pub config_name: Option, + #[serde(rename = "userID", skip_serializing_if = "Option::is_none")] + pub user_id: Option, + #[serde(rename = "userIDClaim", skip_serializing_if = "Option::is_none")] + pub user_id_claim: Option, + #[serde(rename = "displayName", skip_serializing_if = "Option::is_none")] + pub display_name: Option, + #[serde(rename = "displayNameClaim", skip_serializing_if = "Option::is_none")] + pub display_name_claim: Option, +} + +impl OpenIDSpecificAccessKeyInfo { + pub fn is_empty(&self) -> bool { + self.config_name.is_none() + && self.user_id.is_none() + && self.user_id_claim.is_none() + && self.display_name.is_none() + && self.display_name_claim.is_none() + } +} + +#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct InfoAccessKeyResp { + pub access_key: String, + #[serde(flatten)] + pub info: InfoServiceAccountResp, + pub user_type: String, + pub user_provider: String, + #[serde(rename = "ldapSpecificInfo", skip_serializing_if = "LDAPSpecificAccessKeyInfo::is_empty")] + pub ldap_specific_info: LDAPSpecificAccessKeyInfo, + #[serde( + rename = "openIDSpecificInfo", + skip_serializing_if = "OpenIDSpecificAccessKeyInfo::is_empty" + )] + pub open_id_specific_info: OpenIDSpecificAccessKeyInfo, +} + #[derive(Debug, Serialize, Deserialize)] pub struct UpdateServiceAccountReq { - #[serde(rename = "newPolicy", skip_serializing_if = "Option::is_none")] - pub new_policy: Option, + #[serde( + rename = "newPolicy", + skip_serializing_if = "Option::is_none", + default, + deserialize_with = "deserialize_optional_policy_value" + )] + pub new_policy: Option, #[serde(rename = "newSecretKey", skip_serializing_if = "Option::is_none")] pub new_secret_key: Option, @@ -230,18 +305,95 @@ pub struct UpdateServiceAccountReq { #[serde(rename = "newDescription", skip_serializing_if = "Option::is_none")] pub new_description: Option, - #[serde(rename = "newExpiration", skip_serializing_if = "Option::is_none")] + #[serde(rename = "newExpiration", skip_serializing_if = "Option::is_none", default)] #[serde(with = "time::serde::rfc3339::option")] pub new_expiration: Option, } impl UpdateServiceAccountReq { pub fn validate(&self) -> Result<(), String> { - // TODO: validate - Ok(()) + validate_service_account_name(self.new_name.as_deref())?; + validate_service_account_description(self.new_description.as_deref())?; + validate_service_account_expiration(self.new_expiration) } } +fn deserialize_optional_policy_value<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + let value = Option::::deserialize(deserializer)?; + Ok(value.map(normalize_policy_value)) +} + +fn normalize_policy_value(value: Value) -> Value { + match value { + Value::String(policy) => serde_json::from_str(&policy).unwrap_or(Value::String(policy)), + other => other, + } +} + +fn validate_service_account_name(name: Option<&str>) -> Result<(), String> { + let Some(name) = name else { + return Ok(()); + }; + + if name.is_empty() { + return Ok(()); + } + + if name.len() > 32 { + return Err("name must not be longer than 32 characters".to_string()); + } + + let mut chars = name.chars(); + let Some(first) = chars.next() else { + return Ok(()); + }; + + if !first.is_ascii_alphabetic() { + return Err( + "name must contain only ASCII letters, digits, underscores and hyphens and must start with a letter".to_string(), + ); + } + + if chars.any(|c| !c.is_ascii_alphanumeric() && c != '_' && c != '-') { + return Err( + "name must contain only ASCII letters, digits, underscores and hyphens and must start with a letter".to_string(), + ); + } + + Ok(()) +} + +fn validate_service_account_description(description: Option<&str>) -> Result<(), String> { + let Some(description) = description else { + return Ok(()); + }; + + if description.len() > 256 { + return Err("description must be at most 256 bytes long".to_string()); + } + + Ok(()) +} + +fn validate_service_account_expiration(expiration: Option) -> Result<(), String> { + let Some(expiration) = expiration else { + return Ok(()); + }; + + if expiration.unix_timestamp() == 0 { + return Ok(()); + } + + if expiration < OffsetDateTime::now_utc() { + return Err("the expiration time should be in the future".to_string()); + } + + Ok(()) +} + #[derive(Debug, Serialize, Deserialize, Default)] pub struct AccountInfo { pub account_name: String, @@ -423,6 +575,54 @@ pub struct IAMEntities { pub sts_policies: Vec>>, } +/// PolicyEntitiesResult - contains response to a policy entities query. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PolicyEntitiesResult { + #[serde(rename = "timestamp", with = "time::serde::rfc3339")] + pub timestamp: time::OffsetDateTime, + #[serde(rename = "userMappings", skip_serializing_if = "Vec::is_empty")] + pub user_mappings: Vec, + #[serde(rename = "groupMappings", skip_serializing_if = "Vec::is_empty")] + pub group_mappings: Vec, + #[serde(rename = "policyMappings", skip_serializing_if = "Vec::is_empty")] + pub policy_mappings: Vec, +} + +impl Default for PolicyEntitiesResult { + fn default() -> Self { + Self { + timestamp: time::OffsetDateTime::UNIX_EPOCH, + user_mappings: Vec::new(), + group_mappings: Vec::new(), + policy_mappings: Vec::new(), + } + } +} + +/// UserPolicyEntities - user -> policies mapping +#[derive(Default, Debug, Clone, Serialize, Deserialize)] +pub struct UserPolicyEntities { + pub user: String, + pub policies: Vec, + #[serde(rename = "memberOfMappings", skip_serializing_if = "Vec::is_empty")] + pub member_of_mappings: Vec, +} + +/// GroupPolicyEntities - group -> policies mapping +#[derive(Default, Debug, Clone, Serialize, Deserialize)] +pub struct GroupPolicyEntities { + pub group: String, + pub policies: Vec, +} + +/// PolicyEntities - policy -> user+group mapping +#[derive(Default, Debug, Clone, Serialize, Deserialize)] +pub struct PolicyEntities { + pub policy: String, + pub users: Vec, + pub groups: Vec, +} + /// IAMErrEntities - represents errored out IAM entries while import with error #[derive(Debug, Clone, Serialize, Deserialize, Default)] pub struct IAMErrEntities { @@ -659,13 +859,14 @@ mod tests { #[test] fn test_add_service_account_req_validate_success() { let req = AddServiceAccountReq { - policy: Some("ReadOnlyAccess".to_string()), + policy: Some(serde_json::json!({"Version": "2012-10-17"})), target_user: Some("testuser".to_string()), access_key: "AKIAIOSFODNN7EXAMPLE".to_string(), secret_key: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY".to_string(), name: Some("test-service".to_string()), description: Some("Test service account".to_string()), expiration: None, + comment: None, }; let result = req.validate(); @@ -673,54 +874,82 @@ mod tests { } #[test] - fn test_add_service_account_req_validate_empty_access_key() { + fn test_add_service_account_req_validate_allows_generated_credentials() { let req = AddServiceAccountReq { policy: None, target_user: None, access_key: "".to_string(), - secret_key: "secret".to_string(), - name: Some("test".to_string()), - description: None, - expiration: None, - }; - - let result = req.validate(); - assert!(result.is_err()); - assert!(result.unwrap_err().contains("accessKey is empty")); - } - - #[test] - fn test_add_service_account_req_validate_empty_secret_key() { - let req = AddServiceAccountReq { - policy: None, - target_user: None, - access_key: "AKIAIOSFODNN7EXAMPLE".to_string(), secret_key: "".to_string(), - name: Some("test".to_string()), - description: None, - expiration: None, - }; - - let result = req.validate(); - assert!(result.is_err()); - assert!(result.unwrap_err().contains("secretKey is empty")); - } - - #[test] - fn test_add_service_account_req_validate_empty_name() { - let req = AddServiceAccountReq { - policy: None, - target_user: None, - access_key: "AKIAIOSFODNN7EXAMPLE".to_string(), - secret_key: "secret".to_string(), name: None, description: None, expiration: None, + comment: None, + }; + + assert!(req.validate().is_ok()); + } + + #[test] + fn test_add_service_account_req_deserializes_stringified_policy_json() { + let req: AddServiceAccountReq = serde_json::from_str( + r#"{ + "policy":"{\"Version\":\"2012-10-17\",\"Statement\":[]}", + "accessKey":"AKIAIOSFODNN7EXAMPLE", + "secretKey":"secret" + }"#, + ) + .unwrap(); + + assert_eq!(req.policy, Some(serde_json::json!({"Version":"2012-10-17","Statement":[]}))); + } + + #[test] + fn test_add_service_account_req_allows_missing_policy_field() { + let req: AddServiceAccountReq = serde_json::from_str( + r#"{ + "accessKey":"AKIAIOSFODNN7EXAMPLE", + "secretKey":"secret" + }"#, + ) + .unwrap(); + + assert_eq!(req.policy, None); + } + + #[test] + fn test_add_service_account_req_validate_invalid_name() { + let req = AddServiceAccountReq { + policy: None, + target_user: None, + access_key: "AKIAIOSFODNN7EXAMPLE".to_string(), + secret_key: "secret".to_string(), + name: Some("1invalid".to_string()), + description: None, + expiration: None, + comment: None, }; let result = req.validate(); assert!(result.is_err()); - assert!(result.unwrap_err().contains("name is empty")); + assert!(result.unwrap_err().contains("must start with a letter")); + } + + #[test] + fn test_add_service_account_req_validate_rejects_long_description() { + let req = AddServiceAccountReq { + policy: None, + target_user: None, + access_key: "AKIAIOSFODNN7EXAMPLE".to_string(), + secret_key: "secret".to_string(), + name: Some("test".to_string()), + description: Some("a".repeat(257)), + expiration: None, + comment: None, + }; + + let result = req.validate(); + assert!(result.is_err()); + assert!(result.unwrap_err().contains("at most 256 bytes")); } #[test] @@ -793,7 +1022,7 @@ mod tests { #[test] fn test_update_service_account_req_validate() { let req = UpdateServiceAccountReq { - new_policy: Some("FullAccess".to_string()), + new_policy: Some(serde_json::json!({"Version": "2012-10-17"})), new_secret_key: Some("newsecret".to_string()), new_status: Some("enabled".to_string()), new_name: Some("updated-service".to_string()), @@ -805,6 +1034,25 @@ mod tests { assert!(result.is_ok()); } + #[test] + fn test_update_service_account_req_deserializes_stringified_policy_json() { + let req: UpdateServiceAccountReq = serde_json::from_str( + r#"{ + "newPolicy":"{\"Version\":\"2012-10-17\",\"Statement\":[]}" + }"#, + ) + .unwrap(); + + assert_eq!(req.new_policy, Some(serde_json::json!({"Version":"2012-10-17","Statement":[]}))); + } + + #[test] + fn test_update_service_account_req_allows_missing_policy_field() { + let req: UpdateServiceAccountReq = serde_json::from_str(r#"{}"#).unwrap(); + + assert_eq!(req.new_policy, None); + } + #[test] fn test_account_info_creation() { use crate::BackendInfo; @@ -904,6 +1152,7 @@ mod tests { #[test] fn test_serialization_deserialization_roundtrip() { + let now = OffsetDateTime::now_utc().replace_nanosecond(0).unwrap(); let user_info = UserInfo { auth_info: Some(UserAuthInfo { auth_type: UserAuthType::Ldap, @@ -914,16 +1163,19 @@ mod tests { policy_name: Some("ReadOnlyAccess".to_string()), status: AccountStatus::Enabled, member_of: Some(vec!["group1".to_string()]), - updated_at: None, + updated_at: Some(now), }; let json = serde_json::to_string(&user_info).unwrap(); let deserialized: UserInfo = serde_json::from_str(&json).unwrap(); + assert!(json.contains("\"updatedAt\":\"")); + assert!(json.contains('T')); assert_eq!(deserialized.secret_key.unwrap(), "secret123"); assert_eq!(deserialized.policy_name.unwrap(), "ReadOnlyAccess"); assert_eq!(deserialized.status, AccountStatus::Enabled); assert_eq!(deserialized.member_of.unwrap().len(), 1); + assert_eq!(deserialized.updated_at, Some(now)); } #[test] @@ -962,13 +1214,14 @@ mod tests { fn test_edge_cases() { // Test empty strings and edge cases let req = AddServiceAccountReq { - policy: Some("".to_string()), + policy: Some(serde_json::Value::Null), target_user: Some("".to_string()), access_key: "valid_key".to_string(), secret_key: "valid_secret".to_string(), name: Some("valid_name".to_string()), description: Some("".to_string()), expiration: None, + comment: None, }; // Should still validate successfully with empty optional strings @@ -977,13 +1230,14 @@ mod tests { // Test very long strings let long_string = "a".repeat(1000); let long_req = AddServiceAccountReq { - policy: Some(long_string.clone()), + policy: Some(serde_json::json!({"Statement": [long_string.clone()]})), target_user: Some(long_string.clone()), access_key: long_string.clone(), secret_key: long_string.clone(), - name: Some(long_string.clone()), - description: Some(long_string), + name: Some("valid_name".to_string()), + description: Some("valid description".to_string()), expiration: None, + comment: None, }; assert!(long_req.validate().is_ok()); diff --git a/crates/notify/src/stream.rs b/crates/notify/src/stream.rs index 8c70d3c2d..bbb784a87 100644 --- a/crates/notify/src/stream.rs +++ b/crates/notify/src/stream.rs @@ -18,6 +18,7 @@ use rustfs_targets::{ store::{Key, Store}, target::EntityTarget, }; +use rustfs_utils::get_env_usize; use std::sync::Arc; use std::time::{Duration, Instant}; use tokio::sync::{Semaphore, mpsc}; @@ -179,10 +180,7 @@ pub async fn stream_events_with_batching( // Configuration parameters const DEFAULT_BATCH_SIZE: usize = 1; - let batch_size = std::env::var("RUSTFS_EVENT_BATCH_SIZE") - .ok() - .and_then(|s| s.parse::().ok()) - .unwrap_or(DEFAULT_BATCH_SIZE); + let batch_size = get_env_usize("RUSTFS_EVENT_BATCH_SIZE", DEFAULT_BATCH_SIZE); const BATCH_TIMEOUT: Duration = Duration::from_secs(5); const MAX_RETRIES: usize = 5; const BASE_RETRY_DELAY: Duration = Duration::from_secs(2); diff --git a/crates/obs/src/telemetry/local.rs b/crates/obs/src/telemetry/local.rs index 765bb5e88..c7d8cedf6 100644 --- a/crates/obs/src/telemetry/local.rs +++ b/crates/obs/src/telemetry/local.rs @@ -96,7 +96,14 @@ pub(super) fn init_local_logging( let log_dir_str = config.log_directory.as_deref().filter(|s| !s.is_empty()); if let Some(log_directory) = log_dir_str { - init_file_logging_internal(config, log_directory, logger_level, is_production) + match init_file_logging_internal(config, log_directory, logger_level, is_production) { + Ok(guard) => Ok(guard), + Err(error) if should_fallback_to_stdout(&error) => { + emit_file_logging_fallback_warning(log_directory, &error); + Ok(init_stdout_only(config, logger_level, is_production)) + } + Err(error) => Err(error), + } } else { Ok(init_stdout_only(config, logger_level, is_production)) } @@ -331,6 +338,24 @@ pub fn ensure_dir_permissions(log_directory: &str) -> Result<(), TelemetryError> } } +pub(super) fn should_fallback_to_stdout(error: &TelemetryError) -> bool { + match error { + TelemetryError::SetPermissions(_) => true, + TelemetryError::Io(message) => { + let message = message.to_ascii_lowercase(); + message.contains("permission denied") || message.contains("os error 13") + } + _ => false, + } +} + +pub(super) fn emit_file_logging_fallback_warning(log_directory: &str, error: &TelemetryError) { + eprintln!( + "[WARN] Failed to initialize file observability logging at '{}': {}. Falling back to stdout logging.", + log_directory, error + ); +} + // ─── Cleanup task ───────────────────────────────────────────────────────────── /// Spawn a background task that periodically cleans up old log files. @@ -496,4 +521,17 @@ mod tests { assert!(result.is_err(), "invalid filename must return Err, not panic"); }); } + + #[test] + fn test_permission_denied_errors_fall_back_to_stdout() { + assert!(should_fallback_to_stdout(&TelemetryError::Io( + "Permission denied (os error 13)".to_string() + ))); + assert!(should_fallback_to_stdout(&TelemetryError::SetPermissions( + "dir='/logs', want=0o755, have=0o777, err=Permission denied (os error 13)".to_string() + ))); + assert!(!should_fallback_to_stdout(&TelemetryError::Io( + "No such file or directory (os error 2)".to_string() + ))); + } } diff --git a/crates/obs/src/telemetry/otel.rs b/crates/obs/src/telemetry/otel.rs index fb1acf28d..048e990ac 100644 --- a/crates/obs/src/telemetry/otel.rs +++ b/crates/obs/src/telemetry/otel.rs @@ -175,6 +175,7 @@ pub(super) fn init_observability_http( let mut cleanup_handle = None; let mut tracing_guard = None; // Guard for file writer let mut stdout_guard = None; // Guard for stdout writer (File mode) + let mut force_stdout_logging = false; // ── Case 1: OTLP Logging if !log_ep.is_empty() { @@ -201,40 +202,34 @@ pub(super) fn init_observability_http( { let log_filename = config.log_filename.as_deref().unwrap_or(&service_name); let keep_files = config.log_keep_files.unwrap_or(DEFAULT_LOG_KEEP_FILES); + let file_logging_result = (|| -> Result<_, TelemetryError> { + fs::create_dir_all(log_directory).map_err(|e| TelemetryError::Io(e.to_string()))?; - // 1. Ensure dir exists - if let Err(e) = fs::create_dir_all(log_directory) { - return Err(TelemetryError::Io(e.to_string())); - } - // 2. Permissions - #[cfg(unix)] - crate::telemetry::local::ensure_dir_permissions(log_directory)?; + #[cfg(unix)] + crate::telemetry::local::ensure_dir_permissions(log_directory)?; - // 3. Rotation - let rotation_str = config - .log_rotation_time - .as_deref() - .unwrap_or(DEFAULT_LOG_ROTATION_TIME) - .to_lowercase(); - let match_mode = FileMatchMode::from_config_str(config.log_match_mode.as_deref().unwrap_or(DEFAULT_OBS_LOG_MATCH_MODE)); - let rotation = match rotation_str.as_str() { - "minutely" => Rotation::Minutely, - "hourly" => Rotation::Hourly, - "daily" => Rotation::Daily, - _ => Rotation::Daily, - }; - let max_single_file_size = config - .log_max_single_file_size_bytes - .unwrap_or(DEFAULT_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES); + let rotation_str = config + .log_rotation_time + .as_deref() + .unwrap_or(DEFAULT_LOG_ROTATION_TIME) + .to_lowercase(); + let match_mode = + FileMatchMode::from_config_str(config.log_match_mode.as_deref().unwrap_or(DEFAULT_OBS_LOG_MATCH_MODE)); + let rotation = match rotation_str.as_str() { + "minutely" => Rotation::Minutely, + "hourly" => Rotation::Hourly, + "daily" => Rotation::Daily, + _ => Rotation::Daily, + }; + let max_single_file_size = config + .log_max_single_file_size_bytes + .unwrap_or(DEFAULT_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES); - let file_appender = - RollingAppender::new(log_directory, log_filename.to_string(), rotation, max_single_file_size, match_mode)?; + let file_appender = + RollingAppender::new(log_directory, log_filename.to_string(), rotation, max_single_file_size, match_mode)?; - let (non_blocking, guard) = tracing_appender::non_blocking(file_appender); - tracing_guard = Some(guard); - - file_layer_opt = Some( - tracing_subscriber::fmt::layer() + let (non_blocking, guard) = tracing_appender::non_blocking(file_appender); + let file_layer = tracing_subscriber::fmt::layer() .with_timer(LocalTime::rfc_3339()) .with_target(true) .with_ansi(false) @@ -247,17 +242,28 @@ pub(super) fn init_observability_http( .with_current_span(true) .with_span_list(true) .with_span_events(span_events.clone()) - .with_filter(build_env_filter(logger_level, None)), - ); + .with_filter(build_env_filter(logger_level, None)); + let cleanup_handle = spawn_cleanup_task(config, log_directory, log_filename, keep_files); + Ok((file_layer, guard, cleanup_handle, rotation_str)) + })(); - // The cleanup task keeps rotated files bounded while the OTLP trace and - // metric exporters continue to operate independently. - cleanup_handle = Some(spawn_cleanup_task(config, log_directory, log_filename, keep_files)); + match file_logging_result { + Ok((file_layer, guard, new_cleanup_handle, rotation_str)) => { + tracing_guard = Some(guard); + file_layer_opt = Some(file_layer); + cleanup_handle = Some(new_cleanup_handle); - info!( - "Init file logging at '{}', rotation: {}, keep {} files", - log_directory, rotation_str, keep_files - ); + info!( + "Init file logging at '{}', rotation: {}, keep {} files", + log_directory, rotation_str, keep_files + ); + } + Err(error) if crate::telemetry::local::should_fallback_to_stdout(&error) => { + crate::telemetry::local::emit_file_logging_fallback_warning(log_directory, &error); + force_stdout_logging = true; + } + Err(error) => return Err(error), + } } // ── Tracing subscriber registry ─────────────────────────────────────────── @@ -266,10 +272,9 @@ pub(super) fn init_observability_http( .map(|p| OpenTelemetryLayer::new(p.tracer(service_name.to_string()))); let metrics_layer = meter_provider.as_ref().map(|p| MetricsLayer::new(p.clone())); - // Optional stdout mirror (matching `init_file_logging_internal` logic). - // This is separate from OTLP stdout exporting; it only affects local human - // readable output for the tracing subscriber. - if config.log_stdout_enabled.unwrap_or(DEFAULT_OBS_LOG_STDOUT_ENABLED) || !is_production { + // Optional stdout mirror (matching init_file_logging_internal logic) + // This is separate from OTLP stdout logic. If file logging is enabled, we honor its stdout rules. + if force_stdout_logging || config.log_stdout_enabled.unwrap_or(DEFAULT_OBS_LOG_STDOUT_ENABLED) || !is_production { let (stdout_nb, stdout_g) = tracing_appender::non_blocking(std::io::stdout()); stdout_guard = Some(stdout_g); stdout_layer_opt = Some( diff --git a/crates/policy/Cargo.toml b/crates/policy/Cargo.toml index a1aaaf374..b48f700c4 100644 --- a/crates/policy/Cargo.toml +++ b/crates/policy/Cargo.toml @@ -32,7 +32,7 @@ workspace = true rustfs-credentials = { workspace = true } rustfs-config = { workspace = true, features = ["constants", "opa"] } tokio = { workspace = true, features = ["full"] } -time = { workspace = true, features = ["serde-human-readable"] } +time = { workspace = true, features = ["serde", "parsing", "formatting", "macros"] } serde = { workspace = true, features = ["derive", "rc"] } serde_json.workspace = true thiserror.workspace = true diff --git a/crates/policy/src/auth/mod.rs b/crates/policy/src/auth/mod.rs index 7da06ab40..a00f7bc97 100644 --- a/crates/policy/src/auth/mod.rs +++ b/crates/policy/src/auth/mod.rs @@ -26,6 +26,8 @@ use time::OffsetDateTime; pub struct UserIdentity { pub version: i64, pub credentials: Credentials, + /// updatedAt (RFC3339), legacy RustFS: update_at. Serialize as updatedAt + #[serde(rename = "updatedAt", alias = "update_at", default, with = "crate::serde_datetime::option")] pub update_at: Option, } @@ -74,3 +76,19 @@ impl From for UserIdentity { } } } + +#[cfg(test)] +mod tests { + use super::UserIdentity; + + /// Deserialize UserIdentity from MinIO-style JSON (RFC3339 updatedAt). + #[test] + fn test_user_identity_deserialize_minio_style_rfc3339() { + let minio_style = + r#"{"version":1,"credentials":{"accessKey":"ak","secretKey":"sk12345678"},"updatedAt":"2025-03-07T12:00:00Z"}"#; + let u: UserIdentity = serde_json::from_str(minio_style).expect("deserialize MinIO-style identity"); + assert_eq!(u.version, 1); + assert_eq!(u.credentials.access_key, "ak"); + assert!(u.update_at.is_some()); + } +} diff --git a/crates/policy/src/lib.rs b/crates/policy/src/lib.rs index 471f386e7..c40c24f55 100644 --- a/crates/policy/src/lib.rs +++ b/crates/policy/src/lib.rs @@ -17,5 +17,6 @@ pub mod auth; pub mod error; pub mod format; pub mod policy; +pub mod serde_datetime; pub mod service_type; pub mod utils; diff --git a/crates/policy/src/policy/doc.rs b/crates/policy/src/policy/doc.rs index fde312b64..dc2b83fa9 100644 --- a/crates/policy/src/policy/doc.rs +++ b/crates/policy/src/policy/doc.rs @@ -19,9 +19,27 @@ use super::Policy; #[derive(Serialize, Deserialize, Default, Clone)] pub struct PolicyDoc { + /// Version (omitempty), legacy: version. + #[serde(rename = "Version", alias = "version", default)] pub version: i64, + /// Policy, legacy: policy. Serialize as Policy. + #[serde(rename = "Policy", alias = "policy")] pub policy: Policy, + /// CreateDate (RFC3339), legacy: create_date. Serialize as CreateDate. + #[serde( + rename = "CreateDate", + alias = "create_date", + default, + with = "crate::serde_datetime::option" + )] pub create_date: Option, + /// UpdateDate (RFC3339), legacy: update_date. Serialize as UpdateDate. + #[serde( + rename = "UpdateDate", + alias = "update_date", + default, + with = "crate::serde_datetime::option" + )] pub update_date: Option, } @@ -73,3 +91,43 @@ impl TryFrom> for PolicyDoc { .map_err(|_| serde_json::Error::custom("Failed to parse as PolicyDoc or Policy".to_string())) } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::policy::Policy; + + #[test] + fn test_policy_doc_timestamps_serialize_as_rfc3339() { + let policy = Policy::default(); + let doc = PolicyDoc::new(policy); + let json = serde_json::to_string(&doc).expect("serialize"); + // RFC3339 uses 'T' between date and time and 'Z' or offset for UTC + assert!(json.contains('T'), "PolicyDoc timestamps should be RFC3339 (contain 'T'); got: {}", json); + assert!( + json.contains('Z') || json.contains("+00:00"), + "PolicyDoc timestamps should be RFC3339 (contain 'Z' or +00:00); got: {}", + json + ); + } + + #[test] + fn test_policy_doc_deserialize_minio_style_rfc3339_timestamps() { + let minio_style = r#"{"Version":1,"Policy":{"Version":"2012-10-17","Statement":[]},"CreateDate":"2025-03-07T12:00:00Z","UpdateDate":"2025-03-07T12:00:00Z"}"#; + let doc: PolicyDoc = serde_json::from_str(minio_style).expect("deserialize MinIO-style JSON"); + assert_eq!(doc.version, 1); + assert!(doc.create_date.is_some()); + assert!(doc.update_date.is_some()); + } + + /// Round-trip: serialize then deserialize PolicyDoc; timestamps must match. + #[test] + fn test_policy_doc_timestamp_roundtrip() { + let policy = Policy::default(); + let doc = PolicyDoc::new(policy); + let json = serde_json::to_string(&doc).expect("serialize"); + let restored: PolicyDoc = serde_json::from_str(&json).expect("deserialize"); + assert_eq!(doc.create_date, restored.create_date); + assert_eq!(doc.update_date, restored.update_date); + } +} diff --git a/crates/policy/src/policy/policy.rs b/crates/policy/src/policy/policy.rs index e7ed0c993..457356c78 100644 --- a/crates/policy/src/policy/policy.rs +++ b/crates/policy/src/policy/policy.rs @@ -55,9 +55,9 @@ impl Args<'_> { pub struct Policy { #[serde(default, rename = "ID")] pub id: ID, - #[serde(rename = "Version")] + #[serde(default, rename = "Version")] pub version: String, - #[serde(rename = "Statement")] + #[serde(default, rename = "Statement")] pub statements: Vec, } @@ -1084,6 +1084,14 @@ mod test { ); } + #[test] + fn test_parse_empty_policy_object_as_implied_policy() { + let policy = Policy::parse_config(b"{}").expect("empty JSON object should parse"); + + assert!(policy.version.is_empty()); + assert!(policy.statements.is_empty()); + } + #[test] fn test_statement_with_both_action_and_notaction_is_invalid() { // Test: A statement with both Action and NotAction returns BothActionAndNotAction error diff --git a/crates/policy/src/serde_datetime.rs b/crates/policy/src/serde_datetime.rs new file mode 100644 index 000000000..73ddf37e8 --- /dev/null +++ b/crates/policy/src/serde_datetime.rs @@ -0,0 +1,121 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Serde helpers for IAM timestamps: serialize as RFC3339 (MinIO-compatible), +//! deserialize from RFC3339 or legacy RustFS human-readable format. + +use serde::{Deserialize, Deserializer, Serializer}; +use time::OffsetDateTime; +use time::format_description; +use time::format_description::well_known::Rfc3339; + +/// Legacy RustFS format: `YYYY-MM-DD HH:MM:SS.ffffff +00:00:00` (time crate serde-human-readable style). +static LEGACY_FORMAT: std::sync::OnceLock = std::sync::OnceLock::new(); + +fn legacy_format() -> &'static time::format_description::OwnedFormatItem { + LEGACY_FORMAT.get_or_init(|| { + format_description::parse_owned::<2>( + "[year]-[month]-[day] [hour]:[minute]:[second].[subsecond] [offset_hour sign:mandatory]:[offset_minute]:[offset_second]", + ) + .expect("legacy format description is valid") + }); + LEGACY_FORMAT.get().expect("initialized above") +} + +fn parse_rfc3339_or_legacy(s: &str) -> Result { + OffsetDateTime::parse(s, &Rfc3339).or_else(|_| OffsetDateTime::parse(s, legacy_format()).map_err(Into::into)) +} + +/// Serialize as RFC3339; deserialize from RFC3339 or legacy RustFS format. +pub fn serialize(dt: &OffsetDateTime, serializer: S) -> Result +where + S: Serializer, +{ + time::serde::rfc3339::serialize(dt, serializer) +} + +/// Deserialize from RFC3339 or legacy RustFS human-readable format. +pub fn deserialize<'de, D>(deserializer: D) -> Result +where + D: Deserializer<'de>, +{ + let s = <&str>::deserialize(deserializer)?; + parse_rfc3339_or_legacy(s).map_err(serde::de::Error::custom) +} + +/// Option version: serialize as RFC3339; deserialize from RFC3339 or legacy. +pub mod option { + use serde::{Deserialize, Deserializer, Serializer}; + use time::OffsetDateTime; + + use super::{Rfc3339, parse_rfc3339_or_legacy}; + + pub fn serialize(opt: &Option, serializer: S) -> Result + where + S: Serializer, + { + match opt { + Some(dt) => { + let s = dt.format(&Rfc3339).map_err(serde::ser::Error::custom)?; + serializer.serialize_some(&s) + } + None => serializer.serialize_none(), + } + } + + pub fn deserialize<'de, D>(deserializer: D) -> Result, D::Error> + where + D: Deserializer<'de>, + { + let opt: Option<&str> = Option::deserialize(deserializer)?; + match opt { + None => Ok(None), + Some(s) => parse_rfc3339_or_legacy(s).map(Some).map_err(serde::de::Error::custom), + } + } +} + +#[cfg(test)] +mod tests { + use serde::{Deserialize, Serialize}; + + use super::*; + + #[derive(Serialize, Deserialize)] + #[serde(transparent)] + struct Dt(#[serde(with = "crate::serde_datetime")] OffsetDateTime); + + #[test] + fn test_deserialize_legacy_rustfs_timestamp() { + // Legacy RustFS human-readable format (time serde-human-readable). + let json = r#""2026-03-09 02:22:44.998954 +00:00:00""#; + let Dt(dt) = serde_json::from_str(json).expect("deserialize legacy timestamp"); + assert_eq!(dt.year(), 2026); + assert_eq!(dt.month(), time::Month::March); + assert_eq!(dt.day(), 9); + assert_eq!(dt.hour(), 2); + assert_eq!(dt.minute(), 22); + assert_eq!(dt.second(), 44); + } + + #[test] + fn test_deserialize_rfc3339_timestamp() { + let json = r#""2025-03-07T12:00:00Z""#; + let Dt(dt) = serde_json::from_str(json).expect("deserialize RFC3339"); + assert_eq!(dt.year(), 2025); + assert_eq!(dt.month(), time::Month::March); + assert_eq!(dt.day(), 7); + assert_eq!(dt.hour(), 12); + } +} diff --git a/crates/rio/src/compress_index.rs b/crates/rio/src/compress_index.rs index fa03ce25a..b1f7e7ea7 100644 --- a/crates/rio/src/compress_index.rs +++ b/crates/rio/src/compress_index.rs @@ -71,6 +71,10 @@ impl Index { self.info.len() } + pub fn is_empty(&self) -> bool { + self.info.is_empty() + } + fn alloc_infos(&mut self, n: usize) { if n > MAX_INDEX_ENTRIES { panic!("n > MAX_INDEX_ENTRIES"); diff --git a/crates/rio/src/http_reader.rs b/crates/rio/src/http_reader.rs index 33b88bbaa..5377cba7f 100644 --- a/crates/rio/src/http_reader.rs +++ b/crates/rio/src/http_reader.rs @@ -18,6 +18,7 @@ use futures::{Stream, TryStreamExt as _}; use http::HeaderMap; use pin_project_lite::pin_project; use reqwest::{Certificate, Client, Identity, Method, RequestBuilder}; +use rustfs_utils::get_env_opt_str; use std::error::Error as _; use std::io::{self, Error}; use std::ops::Not as _; @@ -32,11 +33,8 @@ use tracing::error; /// Get the TLS path from the RUSTFS_TLS_PATH environment variable. /// If the variable is not set, return None. fn tls_path() -> Option<&'static std::path::PathBuf> { - static TLS_PATH: LazyLock> = LazyLock::new(|| { - std::env::var("RUSTFS_TLS_PATH") - .ok() - .and_then(|s| if s.is_empty() { None } else { Some(s.into()) }) - }); + static TLS_PATH: LazyLock> = + LazyLock::new(|| get_env_opt_str("RUSTFS_TLS_PATH").and_then(|s| if s.is_empty() { None } else { Some(s.into()) })); TLS_PATH.as_ref() } diff --git a/crates/rio/src/lib.rs b/crates/rio/src/lib.rs index 2d6738e49..fcfc0b3df 100644 --- a/crates/rio/src/lib.rs +++ b/crates/rio/src/lib.rs @@ -49,7 +49,7 @@ pub use writer::*; mod http_reader; pub use http_reader::*; -pub use compress_index::TryGetIndex; +pub use compress_index::{Index, TryGetIndex}; mod etag; diff --git a/crates/scanner/src/data_usage_define.rs b/crates/scanner/src/data_usage_define.rs index 8403d48dd..5164a3d8c 100644 --- a/crates/scanner/src/data_usage_define.rs +++ b/crates/scanner/src/data_usage_define.rs @@ -310,17 +310,6 @@ impl DataUsageHash { let hash = self.calculate_hash(); - warn!( - "mod_alt: key: {} hash: {} cycle: {} cycles: {} mod: {} hash >> 32: {} result: {}", - self.0, - hash, - cycle, - cycles, - (hash >> 32) as u32, - (hash >> 32) as u32 % cycles, - cycle % cycles, - ); - (hash >> 32) as u32 % cycles == cycle % cycles } diff --git a/crates/scanner/src/scanner_io.rs b/crates/scanner/src/scanner_io.rs index 34f9d4a0e..3b24bfc1c 100644 --- a/crates/scanner/src/scanner_io.rs +++ b/crates/scanner/src/scanner_io.rs @@ -162,7 +162,8 @@ impl ScannerIO for ECStore { let all_buckets_clone = all_buckets.iter().map(|b| b.name.clone()).collect::>(); tokio::spawn(async move { - let mut last_update = SystemTime::now(); + let mut last_update = SystemTime::UNIX_EPOCH; + let mut has_sent_once = false; let mut ticker = tokio::time::interval(Duration::from_secs(30)); loop { @@ -184,7 +185,8 @@ impl ScannerIO for ECStore { all_merged.merge(result); } - if all_merged.root().is_some() && all_merged.info.last_update.unwrap() > last_update { + let merged_last_update = all_merged.info.last_update.unwrap_or(SystemTime::UNIX_EPOCH); + if all_merged.root().is_some() && (!has_sent_once || merged_last_update > last_update) { let dui = all_merged.dui(&all_merged.info.name, &all_buckets_clone); if let Err(e) = updates.send(dui).await { error!("Failed to send data usage info: {}", e); @@ -202,12 +204,14 @@ impl ScannerIO for ECStore { all_merged.merge(result); } - if all_merged.root().is_some() && all_merged.info.last_update.unwrap() > last_update { + let merged_last_update = all_merged.info.last_update.unwrap_or(SystemTime::UNIX_EPOCH); + if all_merged.root().is_some() && (!has_sent_once || merged_last_update > last_update) { let dui = all_merged.dui(&all_merged.info.name, &all_buckets_clone); if let Err(e) = updates.send(dui).await { error!("Failed to send data usage info: {}", e); } - last_update = all_merged.info.last_update.unwrap(); + has_sent_once = true; + last_update = merged_last_update; } } } @@ -511,8 +515,8 @@ impl ScannerIODisk for Disk { let fivs = match meta.get_file_info_versions(item.bucket.as_str(), item.object_path().as_str(), false) { Ok(versions) => versions, Err(e) => { - error!("Failed to get file info versions: {}", e); - return Err(StorageError::other("failed to get file info".to_string())); + error!("Failed to get file info versions: {}/{}, err: {e}", item.bucket, item.object_path()); + return Err(StorageError::other("skip file".to_string())); } }; diff --git a/crates/scanner/tests/lifecycle_integration_test.rs b/crates/scanner/tests/lifecycle_integration_test.rs index 31cfba1d2..2cc3683b1 100644 --- a/crates/scanner/tests/lifecycle_integration_test.rs +++ b/crates/scanner/tests/lifecycle_integration_test.rs @@ -267,12 +267,12 @@ async fn create_test_tier(server: u32) { ..Default::default() }) } else if server == 2 { - let test_minio_server = std::env::var("TEST_MINIO_SERVER").unwrap_or_else(|_| "localhost:9000".to_string()); + let test_compatible_server = std::env::var("TEST_MINIO_SERVER").unwrap_or_else(|_| "localhost:9000".to_string()); Some(TierMinIO { access_key: "minioadmin".to_string(), secret_key: "minioadmin".to_string(), bucket: "mblock2".to_string(), - endpoint: format!("http://{}", test_minio_server), + endpoint: format!("http://{}", test_compatible_server), prefix: format!("mypre{}/", uuid::Uuid::new_v4()), region: "".to_string(), ..Default::default() diff --git a/crates/trusted-proxies/src/cloud/detector.rs b/crates/trusted-proxies/src/cloud/detector.rs index 30424a691..b6a3dbb8a 100644 --- a/crates/trusted-proxies/src/cloud/detector.rs +++ b/crates/trusted-proxies/src/cloud/detector.rs @@ -15,6 +15,7 @@ //! Cloud provider detection and metadata fetching. use async_trait::async_trait; +use rustfs_utils::get_env_opt_str; use std::str::FromStr; use std::time::Duration; use tracing::{debug, info, warn}; @@ -56,37 +57,33 @@ impl FromStr for CloudProvider { impl CloudProvider { /// Detects the cloud provider based on environment variables. pub fn detect_from_env() -> Option { + let has_env = |key| get_env_opt_str(key).is_some(); + // Check for AWS environment variables. - if std::env::var("RUSTFS_AWS_EXECUTION_ENV").is_ok() - || std::env::var("RUSTFS_AWS_REGION").is_ok() - || std::env::var("RUSTFS_EC2_INSTANCE_ID").is_ok() - { + if has_env("RUSTFS_AWS_EXECUTION_ENV") || has_env("RUSTFS_AWS_REGION") || has_env("RUSTFS_EC2_INSTANCE_ID") { return Some(Self::Aws); } // Check for Azure environment variables. - if std::env::var("RUSTFS_WEBSITE_SITE_NAME").is_ok() - || std::env::var("RUSTFS_WEBSITE_INSTANCE_ID").is_ok() - || std::env::var("RUSTFS_APPSETTING_WEBSITE_SITE_NAME").is_ok() + if has_env("RUSTFS_WEBSITE_SITE_NAME") + || has_env("RUSTFS_WEBSITE_INSTANCE_ID") + || has_env("RUSTFS_APPSETTING_WEBSITE_SITE_NAME") { return Some(Self::Azure); } // Check for GCP environment variables. - if std::env::var("RUSTFS_GCP_PROJECT").is_ok() - || std::env::var("RUSTFS_GOOGLE_CLOUD_PROJECT").is_ok() - || std::env::var("RUSTFS_GAE_INSTANCE").is_ok() - { + if has_env("RUSTFS_GCP_PROJECT") || has_env("RUSTFS_GOOGLE_CLOUD_PROJECT") || has_env("RUSTFS_GAE_INSTANCE") { return Some(Self::Gcp); } // Check for DigitalOcean environment variables. - if std::env::var("RUSTFS_DIGITALOCEAN_REGION").is_ok() { + if has_env("RUSTFS_DIGITALOCEAN_REGION") { return Some(Self::DigitalOcean); } // Check for Cloudflare environment variables. - if std::env::var("RUSTFS_CF_PAGES").is_ok() || std::env::var("RUSTFS_CF_WORKERS").is_ok() { + if has_env("RUSTFS_CF_PAGES") || has_env("RUSTFS_CF_WORKERS") { return Some(Self::Cloudflare); } @@ -106,6 +103,42 @@ impl CloudProvider { } } +#[cfg(test)] +mod tests { + use super::*; + use temp_env::{with_var, with_vars_unset}; + + #[test] + fn test_detect_from_env_prefers_known_provider_markers() { + with_var("RUSTFS_AWS_REGION", Some("us-east-1"), || { + assert_eq!(CloudProvider::detect_from_env(), Some(CloudProvider::Aws)); + }); + } + + #[test] + fn test_detect_from_env_returns_none_without_markers() { + with_vars_unset( + vec![ + "RUSTFS_AWS_EXECUTION_ENV", + "RUSTFS_AWS_REGION", + "RUSTFS_EC2_INSTANCE_ID", + "RUSTFS_WEBSITE_SITE_NAME", + "RUSTFS_WEBSITE_INSTANCE_ID", + "RUSTFS_APPSETTING_WEBSITE_SITE_NAME", + "RUSTFS_GCP_PROJECT", + "RUSTFS_GOOGLE_CLOUD_PROJECT", + "RUSTFS_GAE_INSTANCE", + "RUSTFS_DIGITALOCEAN_REGION", + "RUSTFS_CF_PAGES", + "RUSTFS_CF_WORKERS", + ], + || { + assert_eq!(CloudProvider::detect_from_env(), None); + }, + ); + } +} + /// Trait for fetching metadata from a specific cloud provider. #[async_trait] pub trait CloudMetadataFetcher: Send + Sync { diff --git a/crates/utils/Cargo.toml b/crates/utils/Cargo.toml index fe53c660c..50789dd4b 100644 --- a/crates/utils/Cargo.toml +++ b/crates/utils/Cargo.toml @@ -26,6 +26,7 @@ categories = ["web-programming", "development-tools", "cryptography"] [dependencies] base64-simd = { workspace = true, optional = true } +blake2 = { workspace = true, optional = true } blake3 = { workspace = true, optional = true } brotli = { workspace = true, optional = true } bytes = { workspace = true, optional = true } @@ -69,6 +70,7 @@ zstd = { workspace = true, optional = true } tempfile = { workspace = true } rand = { workspace = true } tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } +temp-env = { workspace = true } [target.'cfg(windows)'.dependencies] windows = { workspace = true, optional = true, features = ["Win32_Storage_FileSystem", "Win32_Foundation"] } @@ -87,7 +89,7 @@ notify = ["dep:hyper", "dep:s3s", "dep:hashbrown", "dep:thiserror", "dep:serde", compress = ["dep:flate2", "dep:brotli", "dep:snap", "dep:lz4", "dep:zstd"] string = ["dep:regex"] crypto = ["dep:base64-simd", "dep:hex-simd", "dep:hmac", "dep:hyper", "dep:sha1"] -hash = ["dep:highway", "dep:md-5", "dep:sha2", "dep:blake3", "dep:serde", "dep:siphasher", "dep:hex-simd", "dep:crc-fast"] +hash = ["dep:highway", "dep:md-5", "dep:sha2", "dep:blake2", "dep:blake3", "dep:serde", "dep:siphasher", "dep:hex-simd", "dep:crc-fast"] os = ["dep:rustix", "dep:tempfile", "dep:windows"] # operating system utilities integration = [] # integration test features sys = ["dep:sysinfo"] # system information features diff --git a/crates/utils/src/envs.rs b/crates/utils/src/envs.rs index a32eaec0a..c21734f93 100644 --- a/crates/utils/src/envs.rs +++ b/crates/utils/src/envs.rs @@ -13,6 +13,7 @@ // limitations under the License. use std::{ + collections::BTreeSet, collections::HashSet, env, sync::{Mutex, OnceLock}, @@ -30,7 +31,7 @@ use tracing::warn; /// - `i8`: The parsed value as i8 if successful, otherwise the default value. /// pub fn get_env_i8(key: &str, default: i8) -> i8 { - env::var(key).ok().and_then(|v| v.parse().ok()).unwrap_or(default) + parse_env_value(key).unwrap_or(default) } /// Retrieve an environment variable as a specific type, returning None if not set or parsing fails. @@ -43,7 +44,7 @@ pub fn get_env_i8(key: &str, default: i8) -> i8 { /// - `Option`: The parsed value as i8 if successful, otherwise None /// pub fn get_env_opt_i8(key: &str) -> Option { - env::var(key).ok().and_then(|v| v.parse().ok()) + parse_env_value(key) } /// Retrieve an environment variable as a specific type, with a default value if not set or parsing fails. @@ -57,7 +58,7 @@ pub fn get_env_opt_i8(key: &str) -> Option { /// - `u8`: The parsed value as u8 if successful, otherwise the default value. /// pub fn get_env_u8(key: &str, default: u8) -> u8 { - env::var(key).ok().and_then(|v| v.parse().ok()).unwrap_or(default) + parse_env_value(key).unwrap_or(default) } /// Retrieve an environment variable as a specific type, returning None if not set or parsing fails. @@ -70,7 +71,7 @@ pub fn get_env_u8(key: &str, default: u8) -> u8 { /// - `Option`: The parsed value as u8 if successful, otherwise None /// pub fn get_env_opt_u8(key: &str) -> Option { - env::var(key).ok().and_then(|v| v.parse().ok()) + parse_env_value(key) } static WARNED_ENV_MESSAGES: OnceLock>> = OnceLock::new(); @@ -86,19 +87,182 @@ fn log_once(key: &str, message: impl FnOnce() -> String) { } } +fn external_alias_for_key(key: &str) -> Option { + let suffix = key.strip_prefix("RUSTFS_")?; + if is_external_compatible_suffix(suffix) { + Some(format!("{}{}", external_env_prefix(), suffix)) + } else { + None + } +} + fn resolve_env_with_aliases(key: &str, deprecated: &[&str]) -> Option<(String, String)> { if let Ok(value) = env::var(key) { return Some((key.to_string(), value)); } - let (alias, value) = deprecated + if let Some((alias, value)) = deprecated .iter() - .find_map(|alias| env::var(alias).ok().map(|value| (*alias, value)))?; + .find_map(|alias| env::var(alias).ok().map(|value| (*alias, value))) + { + let deprecated_key = format!("env_alias:{alias}->{key}"); + log_once(&deprecated_key, || { + format!("Environment variable {alias} is deprecated, use {key} instead") + }); + return Some((alias.to_string(), value)); + } + + let alias = external_alias_for_key(key)?; + let value = env::var(&alias).ok()?; let deprecated_key = format!("env_alias:{alias}->{key}"); log_once(&deprecated_key, || { format!("Environment variable {alias} is deprecated, use {key} instead") }); - Some((alias.to_string(), value)) + Some((alias, value)) +} + +const EXTERNAL_ENV_PREFIX_BYTES: [u8; 6] = [77, 73, 78, 73, 79, 95]; + +const EXTERNAL_COMPATIBLE_SUFFIXES: &[&str] = &[ + "ACCESS_KEY", + "ACCESS_KEY_FILE", + "ADDRESS", + "API_XFF_HEADER", + "AUDIT_WEBHOOK_AUTH_TOKEN", + "AUDIT_WEBHOOK_CLIENT_CERT", + "AUDIT_WEBHOOK_CLIENT_KEY", + "AUDIT_WEBHOOK_ENABLE", + "AUDIT_WEBHOOK_ENDPOINT", + "AUDIT_WEBHOOK_QUEUE_DIR", + "COMPRESS_ENABLE", + "COMPRESS_EXTENSIONS", + "COMPRESS_MIME_TYPES", + "CONSOLE_ADDRESS", + "DRIVE_ACTIVE_MONITORING", + "ERASURE_SET_DRIVE_COUNT", + "IDENTITY_OPENID_CLAIM_NAME", + "IDENTITY_OPENID_CLAIM_PREFIX", + "IDENTITY_OPENID_CLIENT_ID", + "IDENTITY_OPENID_CLIENT_SECRET", + "IDENTITY_OPENID_CONFIG_URL", + "IDENTITY_OPENID_DISPLAY_NAME", + "IDENTITY_OPENID_REDIRECT_URI", + "IDENTITY_OPENID_SCOPES", + "ILM_EXPIRATION_WORKERS", + "LICENSE", + "NOTIFY_MQTT_BROKER", + "NOTIFY_MQTT_ENABLE", + "NOTIFY_MQTT_KEEP_ALIVE_INTERVAL", + "NOTIFY_MQTT_PASSWORD", + "NOTIFY_MQTT_QOS", + "NOTIFY_MQTT_QUEUE_DIR", + "NOTIFY_MQTT_QUEUE_LIMIT", + "NOTIFY_MQTT_RECONNECT_INTERVAL", + "NOTIFY_MQTT_TOPIC", + "NOTIFY_MQTT_USERNAME", + "NOTIFY_WEBHOOK_AUTH_TOKEN", + "NOTIFY_WEBHOOK_CLIENT_CERT", + "NOTIFY_WEBHOOK_CLIENT_KEY", + "NOTIFY_WEBHOOK_ENABLE", + "NOTIFY_WEBHOOK_ENDPOINT", + "NOTIFY_WEBHOOK_QUEUE_DIR", + "NOTIFY_WEBHOOK_QUEUE_LIMIT", + "POLICY_PLUGIN_AUTH_TOKEN", + "POLICY_PLUGIN_URL", + "PORT", + "REGION", + "ROOT_PASSWORD", + "ROOT_USER", + "SECRET_KEY", + "SECRET_KEY_FILE", + "STORAGE_CLASS_INLINE_BLOCK", + "STORAGE_CLASS_OPTIMIZE", + "STORAGE_CLASS_RRS", + "STORAGE_CLASS_STANDARD", + "VERSION", + "VOLUMES", +]; + +const EXTERNAL_DYNAMIC_COMPATIBLE_PREFIXES: &[&str] = &["AUDIT_MQTT_", "AUDIT_WEBHOOK_", "NOTIFY_MQTT_", "NOTIFY_WEBHOOK_"]; + +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub struct ExternalEnvCompatReport { + pub mapped_pairs: Vec<(String, String)>, + pub conflict_keys: Vec, +} + +impl ExternalEnvCompatReport { + pub fn mapped_count(&self) -> usize { + self.mapped_pairs.len() + } + + pub fn conflict_count(&self) -> usize { + self.conflict_keys.len() + } +} + +fn external_env_prefix() -> &'static str { + static PREFIX: OnceLock = OnceLock::new(); + PREFIX + .get_or_init(|| EXTERNAL_ENV_PREFIX_BYTES.iter().map(|&byte| char::from(byte)).collect()) + .as_str() +} + +fn is_external_compatible_suffix(suffix: &str) -> bool { + EXTERNAL_COMPATIBLE_SUFFIXES.contains(&suffix) + || EXTERNAL_DYNAMIC_COMPATIBLE_PREFIXES + .iter() + .any(|prefix| suffix.starts_with(prefix)) +} + +fn build_external_env_compat_report_from_entries(entries: I) -> ExternalEnvCompatReport +where + I: IntoIterator, +{ + let env_map: std::collections::BTreeMap = entries.into_iter().collect(); + let mut mapped_pairs = BTreeSet::new(); + let mut conflict_keys = BTreeSet::new(); + let source_prefix = external_env_prefix(); + + for (source_key, source_value) in env_map.iter() { + let Some(suffix) = source_key.strip_prefix(source_prefix) else { + continue; + }; + if !is_external_compatible_suffix(suffix) { + continue; + } + let rustfs_key = format!("RUSTFS_{suffix}"); + match env_map.get(&rustfs_key) { + None => { + mapped_pairs.insert((source_key.clone(), rustfs_key)); + } + Some(rustfs_value) if rustfs_value != source_value => { + conflict_keys.insert(rustfs_key); + } + Some(_) => {} + } + } + + ExternalEnvCompatReport { + mapped_pairs: mapped_pairs.into_iter().collect(), + conflict_keys: conflict_keys.into_iter().collect(), + } +} + +/// Build compatibility plan between source-prefixed variables and `RUSTFS_*`. +/// +/// Precedence rule: +/// - If both `RUSTFS_*` and source-prefixed variables exist, keep `RUSTFS_*` and record a conflict. +/// - If only source-prefixed variables exist, mark them as mappable to `RUSTFS_*`. +pub fn build_external_env_compat_report() -> ExternalEnvCompatReport { + build_external_env_compat_report_from_entries(env::vars()) +} + +fn parse_env_value(key: &str) -> Option +where + T: std::str::FromStr, +{ + resolve_env_with_aliases(key, &[]).and_then(|(_, value)| value.parse().ok()) } pub fn get_env_str_with_aliases(key: &str, deprecated: &[&str], default: &str) -> String { @@ -134,7 +298,7 @@ pub fn get_env_bool_with_aliases(key: &str, deprecated: &[&str], default: bool) /// - `i16`: The parsed value as i16 if successful, otherwise the default value. /// pub fn get_env_i16(key: &str, default: i16) -> i16 { - env::var(key).ok().and_then(|v| v.parse().ok()).unwrap_or(default) + parse_env_value(key).unwrap_or(default) } /// Retrieve an environment variable as a specific type, returning None if not set or parsing fails. /// 16-bit type: signed i16 @@ -146,7 +310,7 @@ pub fn get_env_i16(key: &str, default: i16) -> i16 { /// - `Option`: The parsed value as i16 if successful, otherwise None /// pub fn get_env_opt_i16(key: &str) -> Option { - env::var(key).ok().and_then(|v| v.parse().ok()) + parse_env_value(key) } /// Retrieve an environment variable as a specific type, with a default value if not set or parsing fails. @@ -160,7 +324,7 @@ pub fn get_env_opt_i16(key: &str) -> Option { /// - `u16`: The parsed value as u16 if successful, otherwise the default value. /// pub fn get_env_u16(key: &str, default: u16) -> u16 { - env::var(key).ok().and_then(|v| v.parse().ok()).unwrap_or(default) + parse_env_value(key).unwrap_or(default) } /// Retrieve an environment variable as a specific type, returning None if not set or parsing fails. /// 16-bit type: unsigned u16 @@ -172,7 +336,7 @@ pub fn get_env_u16(key: &str, default: u16) -> u16 { /// - `Option`: The parsed value as u16 if successful, otherwise None /// pub fn get_env_u16_opt(key: &str) -> Option { - env::var(key).ok().and_then(|v| v.parse().ok()) + parse_env_value(key) } /// Retrieve an environment variable as a specific type, returning None if not set or parsing fails. /// 16-bit type: unsigned u16 @@ -197,7 +361,7 @@ pub fn get_env_opt_u16(key: &str) -> Option { /// - `i32`: The parsed value as i32 if successful, otherwise the default value. /// pub fn get_env_i32(key: &str, default: i32) -> i32 { - env::var(key).ok().and_then(|v| v.parse().ok()).unwrap_or(default) + parse_env_value(key).unwrap_or(default) } /// Retrieve an environment variable as a specific type, returning None if not set or parsing fails. /// 32-bit type: signed i32 @@ -209,7 +373,7 @@ pub fn get_env_i32(key: &str, default: i32) -> i32 { /// - `Option`: The parsed value as i32 if successful, otherwise None /// pub fn get_env_opt_i32(key: &str) -> Option { - env::var(key).ok().and_then(|v| v.parse().ok()) + parse_env_value(key) } /// Retrieve an environment variable as a specific type, with a default value if not set or parsing fails. @@ -223,7 +387,7 @@ pub fn get_env_opt_i32(key: &str) -> Option { /// - `u32`: The parsed value as u32 if successful, otherwise the default value. /// pub fn get_env_u32(key: &str, default: u32) -> u32 { - env::var(key).ok().and_then(|v| v.parse().ok()).unwrap_or(default) + parse_env_value(key).unwrap_or(default) } /// Retrieve an environment variable as a specific type, returning None if not set or parsing fails. /// 32-bit type: unsigned u32 @@ -235,7 +399,7 @@ pub fn get_env_u32(key: &str, default: u32) -> u32 { /// - `Option`: The parsed value as u32 if successful, otherwise None /// pub fn get_env_opt_u32(key: &str) -> Option { - env::var(key).ok().and_then(|v| v.parse().ok()) + parse_env_value(key) } /// Retrieve an environment variable as a specific type, with a default value if not set or parsing fails. /// @@ -247,7 +411,7 @@ pub fn get_env_opt_u32(key: &str) -> Option { /// - `f32`: The parsed value as f32 if successful, otherwise the default value /// pub fn get_env_f32(key: &str, default: f32) -> f32 { - env::var(key).ok().and_then(|v| v.parse().ok()).unwrap_or(default) + parse_env_value(key).unwrap_or(default) } /// Retrieve an environment variable as a specific type, returning None if not set or parsing fails. /// @@ -258,7 +422,7 @@ pub fn get_env_f32(key: &str, default: f32) -> f32 { /// - `Option`: The parsed value as f32 if successful, otherwise None /// pub fn get_env_opt_f32(key: &str) -> Option { - env::var(key).ok().and_then(|v| v.parse().ok()) + parse_env_value(key) } /// Retrieve an environment variable as a specific type, with a default value if not set or parsing fails. @@ -271,7 +435,7 @@ pub fn get_env_opt_f32(key: &str) -> Option { /// - `i64`: The parsed value as i64 if successful, otherwise the default value /// pub fn get_env_i64(key: &str, default: i64) -> i64 { - env::var(key).ok().and_then(|v| v.parse().ok()).unwrap_or(default) + parse_env_value(key).unwrap_or(default) } /// Retrieve an environment variable as a specific type, returning None if not set or parsing fails. /// @@ -282,7 +446,7 @@ pub fn get_env_i64(key: &str, default: i64) -> i64 { /// - `Option`: The parsed value as i64 if successful, otherwise None /// pub fn get_env_opt_i64(key: &str) -> Option { - env::var(key).ok().and_then(|v| v.parse().ok()) + parse_env_value(key) } /// Retrieve an environment variable as a specific type, returning Option> if not set or parsing fails. @@ -294,7 +458,7 @@ pub fn get_env_opt_i64(key: &str) -> Option { /// - `Option>`: The parsed value as i64 if successful, otherwise None /// pub fn get_env_opt_opt_i64(key: &str) -> Option> { - env::var(key).ok().map(|v| v.parse().ok()) + resolve_env_with_aliases(key, &[]).map(|(_, value)| value.parse().ok()) } /// Retrieve an environment variable as a specific type, with a default value if not set or parsing fails. @@ -307,7 +471,7 @@ pub fn get_env_opt_opt_i64(key: &str) -> Option> { /// - `u64`: The parsed value as u64 if successful, otherwise the default value. /// pub fn get_env_u64(key: &str, default: u64) -> u64 { - env::var(key).ok().and_then(|v| v.parse().ok()).unwrap_or(default) + parse_env_value(key).unwrap_or(default) } /// Retrieve an environment variable as an unsigned 64-bit integer, returning `None` if not set or parsing fails. @@ -341,7 +505,7 @@ pub fn get_env_opt_u64_with_aliases(key: &str, deprecated: &[&str]) -> Option`: The parsed value as u64 if successful, otherwise None /// pub fn get_env_opt_u64(key: &str) -> Option { - env::var(key).ok().and_then(|v| v.parse().ok()) + parse_env_value(key) } /// Retrieve an environment variable as a specific type, with a default value if not set or parsing fails. @@ -354,7 +518,7 @@ pub fn get_env_opt_u64(key: &str) -> Option { /// - `f64`: The parsed value as f64 if successful, otherwise the default value. /// pub fn get_env_f64(key: &str, default: f64) -> f64 { - env::var(key).ok().and_then(|v| v.parse().ok()).unwrap_or(default) + parse_env_value(key).unwrap_or(default) } /// Retrieve an environment variable as a specific type, returning None if not set or parsing fails. @@ -366,7 +530,7 @@ pub fn get_env_f64(key: &str, default: f64) -> f64 { /// - `Option`: The parsed value as f64 if successful, otherwise None /// pub fn get_env_opt_f64(key: &str) -> Option { - env::var(key).ok().and_then(|v| v.parse().ok()) + parse_env_value(key) } /// Retrieve an environment variable as a specific type, with a default value if not set or parsing fails. @@ -379,7 +543,7 @@ pub fn get_env_opt_f64(key: &str) -> Option { /// - `usize`: The parsed value as usize if successful, otherwise the default value. /// pub fn get_env_usize(key: &str, default: usize) -> usize { - env::var(key).ok().and_then(|v| v.parse().ok()).unwrap_or(default) + parse_env_value(key).unwrap_or(default) } /// Retrieve an environment variable as a specific type, returning None if not set or parsing fails. /// @@ -390,7 +554,7 @@ pub fn get_env_usize(key: &str, default: usize) -> usize { /// - `Option`: The parsed value as usize if successful, otherwise None /// pub fn get_env_usize_opt(key: &str) -> Option { - env::var(key).ok().and_then(|v| v.parse().ok()) + parse_env_value(key) } /// Retrieve an environment variable as a specific type, returning None if not set or parsing fails. @@ -427,7 +591,7 @@ pub fn get_env_str(key: &str, default: &str) -> String { /// - `Option`: The environment variable value if set, otherwise None. /// pub fn get_env_opt_str(key: &str) -> Option { - env::var(key).ok() + resolve_env_with_aliases(key, &[]).map(|(_, value)| value) } /// Retrieve an environment variable as a boolean, with a default value if not set or parsing fails. @@ -476,3 +640,105 @@ pub fn get_env_opt_bool(key: &str) -> Option { None }) } + +/// Copy supported external-prefix variables such as `MINIO_*` into their +/// canonical `RUSTFS_*` names in the current process when the canonical key is +/// missing. +#[allow(unsafe_code)] +pub fn apply_external_env_compat() -> ExternalEnvCompatReport { + let report = build_external_env_compat_report(); + for (source_key, rustfs_key) in &report.mapped_pairs { + if let Ok(value) = env::var(source_key) { + // Safety: this helper is intended for early startup bootstrap + // before any background threads are created. + unsafe { + env::set_var(rustfs_key, value); + } + } + } + report +} + +#[cfg(test)] +mod tests { + use super::{apply_external_env_compat, build_external_env_compat_report_from_entries, get_env_str}; + + fn source_key(suffix: &str) -> String { + let mut key = super::external_env_prefix().to_string(); + key.push_str(suffix); + key + } + + #[test] + fn source_value_is_mapped_when_rustfs_missing() { + let report = + build_external_env_compat_report_from_entries(vec![(source_key("STORAGE_CLASS_STANDARD"), "EC:2".to_string())]); + assert_eq!(report.mapped_count(), 1); + assert!( + report + .mapped_pairs + .iter() + .any(|(input_key, rustfs_key)| input_key == &source_key("STORAGE_CLASS_STANDARD") + && rustfs_key == "RUSTFS_STORAGE_CLASS_STANDARD") + ); + assert_eq!(report.conflict_count(), 0); + } + + #[test] + fn rustfs_value_takes_precedence_on_conflict() { + let report = build_external_env_compat_report_from_entries(vec![ + ("RUSTFS_ERASURE_SET_DRIVE_COUNT".to_string(), "8".to_string()), + (source_key("ERASURE_SET_DRIVE_COUNT"), "16".to_string()), + ]); + assert_eq!(report.mapped_count(), 0); + assert_eq!(report.conflict_count(), 1); + assert!(report.conflict_keys.iter().any(|key| key == "RUSTFS_ERASURE_SET_DRIVE_COUNT")); + } + + #[test] + fn dynamic_notify_suffix_is_mapped() { + let report = + build_external_env_compat_report_from_entries(vec![(source_key("NOTIFY_WEBHOOK_ENABLE_PRIMARY"), "on".to_string())]); + assert_eq!(report.mapped_count(), 1); + assert!( + report + .mapped_pairs + .iter() + .any(|(input_key, rustfs_key)| input_key == &source_key("NOTIFY_WEBHOOK_ENABLE_PRIMARY") + && rustfs_key == "RUSTFS_NOTIFY_WEBHOOK_ENABLE_PRIMARY") + ); + assert_eq!(report.conflict_count(), 0); + } + + #[test] + fn unrelated_source_key_is_ignored() { + let report = build_external_env_compat_report_from_entries(vec![(source_key("UNKNOWN_COMPAT_TEST"), "1".to_string())]); + assert_eq!(report.mapped_count(), 0); + assert_eq!(report.conflict_count(), 0); + } + + #[test] + fn minio_alias_is_used_for_rustfs_reads() { + temp_env::with_var("MINIO_ROOT_USER", Some("compat-admin"), || { + temp_env::with_var_unset("RUSTFS_ROOT_USER", || { + assert_eq!(get_env_str("RUSTFS_ROOT_USER", "default-user"), "compat-admin"); + }); + }); + } + + #[test] + fn apply_external_env_compat_copies_missing_rustfs_keys() { + temp_env::with_var("MINIO_ROOT_USER", Some("compat-admin"), || { + temp_env::with_var_unset("RUSTFS_ROOT_USER", || { + let report = apply_external_env_compat(); + assert!( + report + .mapped_pairs + .iter() + .any(|(source_key, rustfs_key)| source_key == "MINIO_ROOT_USER" && rustfs_key == "RUSTFS_ROOT_USER") + ); + assert_eq!(std::env::var("RUSTFS_ROOT_USER").as_deref(), Ok("compat-admin")); + }); + }); + } +} diff --git a/crates/utils/src/hash.rs b/crates/utils/src/hash.rs index c54b0f22c..367b0e194 100644 --- a/crates/utils/src/hash.rs +++ b/crates/utils/src/hash.rs @@ -12,13 +12,30 @@ // See the License for the specific language governing permissions and // limitations under the License. +use blake2::{Blake2b512, Digest as Blake2Digest}; use highway::{HighwayHash, HighwayHasher, Key}; -use md5::{Digest, Md5}; +use md5::Md5; use serde::{Deserialize, Serialize}; use sha2::Sha256; -/// The fixed key for HighwayHash256. DO NOT change for compatibility. -const HIGHWAY_HASH256_KEY: [u64; 4] = [3, 4, 2, 1]; +/// Magic HH-256 key: HH-256 hash of first 100 decimals of π as utf-8 with zero key. +const MAGIC_HIGHWAY_HASH256_KEY: [u8; 32] = [ + 0x4b, 0xe7, 0x34, 0xfa, 0x8e, 0x23, 0x8a, 0xcd, 0x26, 0x3e, 0x83, 0xe6, 0xbb, 0x96, 0x85, 0x52, 0x04, 0x0f, 0x93, 0x5d, 0xa3, + 0x9f, 0x44, 0x14, 0x97, 0xe0, 0x9d, 0x13, 0x22, 0xde, 0x36, 0xa0, +]; + +/// Legacy HH-256 key (main branch): fixed [3,4,2,1] as u64 LE. +const LEGACY_HIGHWAY_HASH256_KEY: [u8; 32] = [ + 3, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, +]; + +fn highway_key_from_bytes(bytes: &[u8; 32]) -> [u64; 4] { + let mut key = [0u64; 4]; + for (i, chunk) in bytes.chunks_exact(8).enumerate() { + key[i] = u64::from_le_bytes(chunk.try_into().unwrap()); + } + key +} #[derive(Serialize, Deserialize, Debug, PartialEq, Default, Clone, Eq, Hash)] /// Supported hash algorithms for bitrot protection. @@ -30,6 +47,8 @@ pub enum HashAlgorithm { // HighwayHash256S represents the Streaming HighwayHash-256 hash function #[default] HighwayHash256S, + /// Legacy HighwayHash256S (main branch) with fixed key [3,4,2,1] + HighwayHash256SLegacy, // BLAKE2b512 represents the BLAKE2b-512 hash function BLAKE2b512, /// MD5 (128-bit) @@ -43,7 +62,8 @@ enum HashEncoded { Sha256([u8; 32]), HighwayHash256([u8; 32]), HighwayHash256S([u8; 32]), - Blake2b512(blake3::Hash), + HighwayHash256SLegacy([u8; 32]), + Blake2b512([u8; 64]), None, } @@ -55,7 +75,8 @@ impl AsRef<[u8]> for HashEncoded { HashEncoded::Sha256(hash) => hash.as_ref(), HashEncoded::HighwayHash256(hash) => hash.as_ref(), HashEncoded::HighwayHash256S(hash) => hash.as_ref(), - HashEncoded::Blake2b512(hash) => hash.as_bytes(), + HashEncoded::HighwayHash256SLegacy(hash) => hash.as_ref(), + HashEncoded::Blake2b512(hash) => hash.as_ref(), HashEncoded::None => &[], } } @@ -83,17 +104,30 @@ impl HashAlgorithm { match self { HashAlgorithm::Md5 => HashEncoded::Md5(Md5::digest(data).into()), HashAlgorithm::HighwayHash256 => { - let mut hasher = HighwayHasher::new(Key(HIGHWAY_HASH256_KEY)); + let key = Key(highway_key_from_bytes(&MAGIC_HIGHWAY_HASH256_KEY)); + let mut hasher = HighwayHasher::new(key); hasher.append(data); HashEncoded::HighwayHash256(u8x32_from_u64x4(hasher.finalize256())) } HashAlgorithm::SHA256 => HashEncoded::Sha256(Sha256::digest(data).into()), HashAlgorithm::HighwayHash256S => { - let mut hasher = HighwayHasher::new(Key(HIGHWAY_HASH256_KEY)); + let key = Key(highway_key_from_bytes(&MAGIC_HIGHWAY_HASH256_KEY)); + let mut hasher = HighwayHasher::new(key); hasher.append(data); HashEncoded::HighwayHash256S(u8x32_from_u64x4(hasher.finalize256())) } - HashAlgorithm::BLAKE2b512 => HashEncoded::Blake2b512(blake3::hash(data)), + HashAlgorithm::HighwayHash256SLegacy => { + let key = Key(highway_key_from_bytes(&LEGACY_HIGHWAY_HASH256_KEY)); + let mut hasher = HighwayHasher::new(key); + hasher.append(data); + HashEncoded::HighwayHash256SLegacy(u8x32_from_u64x4(hasher.finalize256())) + } + HashAlgorithm::BLAKE2b512 => { + let hash = Blake2b512::digest(data); + let mut out = [0u8; 64]; + out.copy_from_slice(hash.as_ref()); + HashEncoded::Blake2b512(out) + } HashAlgorithm::None => HashEncoded::None, } } @@ -108,7 +142,8 @@ impl HashAlgorithm { HashAlgorithm::SHA256 => 32, HashAlgorithm::HighwayHash256 => 32, HashAlgorithm::HighwayHash256S => 32, - HashAlgorithm::BLAKE2b512 => 32, // blake3 outputs 32 bytes by default + HashAlgorithm::HighwayHash256SLegacy => 32, + HashAlgorithm::BLAKE2b512 => 64, HashAlgorithm::Md5 => 16, HashAlgorithm::None => 0, } @@ -167,7 +202,7 @@ mod tests { assert_eq!(HashAlgorithm::HighwayHash256.size(), 32); assert_eq!(HashAlgorithm::HighwayHash256S.size(), 32); assert_eq!(HashAlgorithm::SHA256.size(), 32); - assert_eq!(HashAlgorithm::BLAKE2b512.size(), 32); + assert_eq!(HashAlgorithm::BLAKE2b512.size(), 64); assert_eq!(HashAlgorithm::None.size(), 0); } @@ -220,13 +255,68 @@ mod tests { let data = b"test data"; let hash = HashAlgorithm::BLAKE2b512.hash_encode(data); let hash = hash.as_ref(); - assert_eq!(hash.len(), 32); // blake3 outputs 32 bytes by default + assert_eq!(hash.len(), 64); // BLAKE2b512 should be deterministic let hash2 = HashAlgorithm::BLAKE2b512.hash_encode(data); let hash2 = hash2.as_ref(); assert_eq!(hash, hash2); } + #[test] + fn test_bitrot_selftest() { + let checksums: [(HashAlgorithm, &str); 5] = [ + (HashAlgorithm::SHA256, "a7677ff19e0182e4d52e3a3db727804abc82a5818749336369552e54b838b004"), + ( + HashAlgorithm::BLAKE2b512, + "e519b7d84b1c3c917985f544773a35cf265dcab10948be3550320d156bab612124a5ae2ae5a8c73c0eea360f68b0e28136f26e858756dbfe7375a7389f26c669", + ), + ( + HashAlgorithm::HighwayHash256, + "39c0407ed3f01b18d22c85db4aeff11e060ca5f43131b0126731ca197cd42313", + ), + ( + HashAlgorithm::HighwayHash256S, + "39c0407ed3f01b18d22c85db4aeff11e060ca5f43131b0126731ca197cd42313", + ), + ( + HashAlgorithm::HighwayHash256SLegacy, + "a5592a831588836b0f61bff43da4bd957c376d9b6412a9ecbbd144a3ecf34649", + ), + ]; + for (algo, expected_hex) in checksums { + let block_size = match algo { + HashAlgorithm::SHA256 => 64, + HashAlgorithm::BLAKE2b512 => 128, + HashAlgorithm::HighwayHash256 | HashAlgorithm::HighwayHash256S | HashAlgorithm::HighwayHash256SLegacy => 32, + _ => continue, + }; + let mut msg = Vec::new(); + let mut sum = Vec::new(); + for _ in 0..block_size { + sum = algo.hash_encode(&msg).as_ref().to_vec(); + msg.extend_from_slice(&sum); + } + let got = hex_simd::encode_to_string(&sum, hex_simd::AsciiCase::Lower); + assert_eq!(got, expected_hex, "{:?} selftest mismatch: got {} want {}", algo, got, expected_hex); + } + } + + /// Generates 7557 bytes + /// Pattern: (i*7+13)%256 for each byte. + fn generate_compat_test_data(size: usize) -> Vec { + (0..size).map(|i| ((i * 7 + 13) % 256) as u8).collect() + } + + /// Run: cargo test -p rustfs-utils test_highwayhash_compat + #[test] + fn test_highwayhash_compat() { + let data = generate_compat_test_data(7557); + let hash = HashAlgorithm::HighwayHash256S.hash_encode(&data); + let got = hex_simd::encode_to_string(hash.as_ref(), hex_simd::AsciiCase::Lower); + let expected = "06543bf1c637e67386922a43b71cca08e5faa0f9131105a2bf96dec880529551"; + assert_eq!(got, expected, "HighwayHash256S must match: got {} want {}", got, expected); + } + #[test] fn test_different_data_different_hashes() { let data1 = b"test data 1"; diff --git a/crates/utils/src/http/header_compat.rs b/crates/utils/src/http/header_compat.rs new file mode 100644 index 000000000..6dda300e6 --- /dev/null +++ b/crates/utils/src/http/header_compat.rs @@ -0,0 +1,123 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! HTTP header compatibility: read both x-rustfs-* and x-minio-* headers for MinIO +//! interoperability. Write both when sending replication requests. +//! +//! Use suffix-based API: `get_header(headers, SUFFIX_FORCE_DELETE)` queries both +//! x-rustfs-force-delete and x-minio-force-delete. + +use http::{HeaderMap, HeaderValue}; +use std::borrow::Cow; + +const RUSTFS_PREFIX: &str = "x-rustfs-"; +const MINIO_PREFIX: &str = "x-minio-"; +const MINIO_ENCRYPTION_PREFIX: &str = "x-minio-encryption-"; +const RUSTFS_ENCRYPTION_PREFIX: &str = "x-rustfs-encryption-"; + +// Suffix constants (part after x-rustfs- or x-minio-). Use with get_header/insert_header. +pub const SUFFIX_FORCE_DELETE: &str = "force-delete"; +pub const SUFFIX_INCLUDE_DELETED: &str = "include-deleted"; +pub const SUFFIX_REPLICATION_RESET_STATUS: &str = "replication-reset-status"; +pub const SUFFIX_REPLICATION_ACTUAL_OBJECT_SIZE: &str = "replication-actual-object-size"; +pub const SUFFIX_SOURCE_VERSION_ID: &str = "source-version-id"; +pub const SUFFIX_SOURCE_MTIME: &str = "source-mtime"; +pub const SUFFIX_SOURCE_ETAG: &str = "source-etag"; +pub const SUFFIX_SOURCE_DELETEMARKER: &str = "source-deletemarker"; +pub const SUFFIX_SOURCE_PROXY_REQUEST: &str = "source-proxy-request"; +pub const SUFFIX_SOURCE_REPLICATION_REQUEST: &str = "source-replication-request"; +pub const SUFFIX_SOURCE_REPLICATION_CHECK: &str = "source-replication-check"; +pub const SUFFIX_REPLICATION_SSEC_CRC: &str = "replication-ssec-crc"; + +/// Returns true if the key is an internal encryption metadata key (x-rustfs-encryption-* or +/// x-minio-encryption-*). Case-insensitive for metadata filtering. +pub fn is_encryption_metadata_key(key: &str) -> bool { + let lower = key.to_lowercase(); + lower.starts_with(RUSTFS_ENCRYPTION_PREFIX) || lower.starts_with(MINIO_ENCRYPTION_PREFIX) +} + +fn rustfs_key(suffix: &str) -> String { + format!("{RUSTFS_PREFIX}{suffix}") +} + +fn minio_key(suffix: &str) -> String { + format!("{MINIO_PREFIX}{suffix}") +} + +/// Get header value: tries x-rustfs-{suffix} first, then x-minio-{suffix}. Case-insensitive. +pub fn get_header<'a>(headers: &'a HeaderMap, suffix: &str) -> Option> { + let rk = rustfs_key(suffix); + let mk = minio_key(suffix); + headers + .get(&rk) + .or_else(|| headers.get(&mk)) + .and_then(|v| v.to_str().ok().map(Cow::Borrowed)) +} + +/// Insert header with both x-rustfs-{suffix} and x-minio-{suffix}. +pub fn insert_header(headers: &mut HeaderMap, suffix: &str, value: impl AsRef<[u8]>) { + if let Ok(v) = HeaderValue::from_bytes(value.as_ref()) { + if let Ok(k1) = rustfs_key(suffix).parse::() { + headers.insert(k1, v.clone()); + } + if let Ok(k2) = minio_key(suffix).parse::() { + headers.insert(k2, v); + } + } +} + +/// Get from HashMap: tries x-rustfs-{suffix} first, then x-minio-{suffix}. +pub fn get_header_map(map: &std::collections::HashMap, suffix: &str) -> Option { + let rk = rustfs_key(suffix); + let mk = minio_key(suffix); + map.get(&rk).cloned().or_else(|| map.get(&mk).cloned()) +} + +/// Insert into HashMap with both x-rustfs-{suffix} and x-minio-{suffix}. +pub fn insert_header_map(map: &mut std::collections::HashMap, suffix: &str, value: impl Into) { + let v = value.into(); + map.insert(rustfs_key(suffix), v.clone()); + map.insert(minio_key(suffix), v); +} + +/// Remove from HashMap both x-rustfs-{suffix} and x-minio-{suffix}. +pub fn remove_header_map(map: &mut std::collections::HashMap, suffix: &str) { + map.remove(&rustfs_key(suffix)); + map.remove(&minio_key(suffix)); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_is_encryption_metadata_key() { + assert!(is_encryption_metadata_key("x-rustfs-encryption-iv")); + assert!(is_encryption_metadata_key("X-Rustfs-Encryption-Key")); + assert!(is_encryption_metadata_key("x-minio-encryption-iv")); + assert!(!is_encryption_metadata_key("x-amz-meta-custom")); + assert!(!is_encryption_metadata_key("x-rustfs-internal-healing")); + } + + #[test] + fn test_get_header() { + let mut headers = HeaderMap::new(); + headers.insert("x-minio-force-delete", HeaderValue::from_static("true")); + assert_eq!(get_header(&headers, SUFFIX_FORCE_DELETE).as_deref(), Some("true")); + + let mut headers2 = HeaderMap::new(); + headers2.insert("X-Rustfs-Force-Delete", HeaderValue::from_static("true")); + assert_eq!(get_header(&headers2, SUFFIX_FORCE_DELETE).as_deref(), Some("true")); + } +} diff --git a/crates/utils/src/http/headers.rs b/crates/utils/src/http/headers.rs index 243672556..3c3b1d1e8 100644 --- a/crates/utils/src/http/headers.rs +++ b/crates/utils/src/http/headers.rs @@ -148,41 +148,9 @@ pub const AMZ_META_NAME: &str = "X-Amz-Meta-Name"; pub const AMZ_META_UNENCRYPTED_CONTENT_LENGTH: &str = "X-Amz-Meta-X-Amz-Unencrypted-Content-Length"; pub const AMZ_META_UNENCRYPTED_CONTENT_MD5: &str = "X-Amz-Meta-X-Amz-Unencrypted-Content-Md5"; -pub const RUSTFS_ENCRYPTION: &str = "X-Rustfs-Encryption-"; -pub const RUSTFS_ENCRYPTION_LOWER: &str = "x-rustfs-encryption-"; - -pub const RESERVED_METADATA_PREFIX: &str = "X-RustFS-Internal-"; -pub const RESERVED_METADATA_PREFIX_LOWER: &str = "x-rustfs-internal-"; - -pub const RUSTFS_HEALING: &str = "X-Rustfs-Internal-healing"; -// pub const RUSTFS_DATA_MOVE: &str = "X-Rustfs-Internal-data-mov"; - -// pub const X_RUSTFS_INLINE_DATA: &str = "x-rustfs-inline-data"; - -pub const VERSION_PURGE_STATUS_KEY: &str = "X-Rustfs-Internal-purgestatus"; - -pub const X_RUSTFS_HEALING: &str = "X-Rustfs-Internal-healing"; -pub const X_RUSTFS_DATA_MOV: &str = "X-Rustfs-Internal-data-mov"; pub const AMZ_TAGGING_DIRECTIVE: &str = "X-Amz-Tagging-Directive"; -pub const RUSTFS_DATA_MOVE: &str = "X-Rustfs-Internal-data-mov"; - -pub const RUSTFS_FORCE_DELETE: &str = "X-Rustfs-Force-Delete"; -pub const RUSTFS_INCLUDE_DELETED: &str = "X-Rustfs-Include-Deleted"; - -pub const RUSTFS_REPLICATION_RESET_STATUS: &str = "X-Rustfs-Replication-Reset-Status"; -pub const RUSTFS_REPLICATION_ACTUAL_OBJECT_SIZE: &str = "X-Rustfs-Replication-Actual-Object-Size"; - -pub const RUSTFS_BUCKET_SOURCE_VERSION_ID: &str = "X-Rustfs-Source-Version-Id"; -pub const RUSTFS_BUCKET_SOURCE_MTIME: &str = "X-RustFS-Source-Mtime"; -pub const RUSTFS_BUCKET_SOURCE_ETAG: &str = "X-Rustfs-Source-Etag"; -pub const RUSTFS_BUCKET_REPLICATION_DELETE_MARKER: &str = "X-Rustfs-Source-DeleteMarker"; -pub const RUSTFS_BUCKET_REPLICATION_PROXY_REQUEST: &str = "X-Rustfs-Source-Proxy-Request"; -pub const RUSTFS_BUCKET_REPLICATION_REQUEST: &str = "X-Rustfs-Source-Replication-Request"; -pub const RUSTFS_BUCKET_REPLICATION_CHECK: &str = "X-Rustfs-Source-Replication-Check"; -pub const RUSTFS_BUCKET_REPLICATION_SSEC_CHECKSUM: &str = "X-Rustfs-Source-Replication-Ssec-Crc"; - // SSEC encryption header constants pub const SSEC_ALGORITHM_HEADER: &str = "x-amz-server-side-encryption-customer-algorithm"; pub const SSEC_KEY_HEADER: &str = "x-amz-server-side-encryption-customer-key"; diff --git a/crates/utils/src/http/metadata_compat.rs b/crates/utils/src/http/metadata_compat.rs new file mode 100644 index 000000000..cdb511911 --- /dev/null +++ b/crates/utils/src/http/metadata_compat.rs @@ -0,0 +1,181 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! System metadata compatibility: write both x-rustfs-internal-* and x-minio-internal-* +//! for MinIO interoperability. Read prefers RustFS, fallback to MinIO. + +use std::collections::HashMap; + +pub const RUSTFS_INTERNAL_PREFIX: &str = "x-rustfs-internal-"; +pub const MINIO_INTERNAL_PREFIX: &str = "x-minio-internal-"; + +// Key suffixes (lowercase, no prefix) +pub const SUFFIX_INLINE_DATA: &str = "inline-data"; +pub const SUFFIX_DATA_MOVED: &str = "data-moved"; +/// Transient flag for data movement +pub const SUFFIX_DATA_MOV: &str = "data-mov"; +/// Transient flag for healing +pub const SUFFIX_HEALING: &str = "healing"; +pub const SUFFIX_COMPRESSION: &str = "compression"; +pub const SUFFIX_COMPRESSION_SIZE: &str = "compression-size"; +pub const SUFFIX_ACTUAL_SIZE: &str = "actual-size"; +pub const SUFFIX_ACTUAL_OBJECT_SIZE: &str = "actual-object-size"; +/// Used by replication; key stored with capital A +pub const SUFFIX_ACTUAL_OBJECT_SIZE_CAP: &str = "Actual-Object-Size"; +pub const SUFFIX_CRC: &str = "crc"; +pub const SUFFIX_TRANSITION_STATUS: &str = "transition-status"; +pub const SUFFIX_TRANSITIONED_OBJECTNAME: &str = "transitioned-object"; +pub const SUFFIX_TRANSITIONED_VERSION_ID: &str = "transitioned-versionID"; +pub const SUFFIX_TRANSITION_TIER: &str = "transition-tier"; +pub const SUFFIX_FREE_VERSION: &str = "free-version"; +pub const SUFFIX_PURGESTATUS: &str = "purgestatus"; +pub const SUFFIX_REPLICA_STATUS: &str = "replica-status"; +pub const SUFFIX_REPLICA_TIMESTAMP: &str = "replica-timestamp"; +pub const SUFFIX_REPLICATION_STATUS: &str = "replication-status"; +pub const SUFFIX_REPLICATION_TIMESTAMP: &str = "replication-timestamp"; +pub const SUFFIX_TAGGING_TIMESTAMP: &str = "tagging-timestamp"; +pub const SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP: &str = "objectlock-retention-timestamp"; +pub const SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP: &str = "objectlock-legalhold-timestamp"; +pub const SUFFIX_REPLICATION_RESET: &str = "replication-reset"; +/// Prefix for replication-reset-{arn} keys; use with internal_key_strip_suffix_prefix to extract arn. +pub const SUFFIX_REPLICATION_RESET_ARN_PREFIX: &str = "replication-reset-"; +pub const SUFFIX_TIER_FV_ID: &str = "tier-free-versionID"; +pub const SUFFIX_TIER_FV_MARKER: &str = "tier-free-marker"; +pub const SUFFIX_TIER_SKIP_FV_ID: &str = "tier-skip-fvid"; + +/// Returns true if the key is an internal metadata key (x-rustfs-internal-* or x-minio-internal-*) +/// for xl.meta compatibility. Case-insensitive. +pub fn is_internal_key(key: &str) -> bool { + let lower = key.to_lowercase(); + lower.starts_with(RUSTFS_INTERNAL_PREFIX) || lower.starts_with(MINIO_INTERNAL_PREFIX) +} + +/// Returns true if the key matches the given suffix for either x-rustfs-internal-* or x-minio-internal-*. +pub fn has_internal_suffix(key: &str, suffix: &str) -> bool { + let lower = key.to_lowercase(); + let rustfs_key = format!("{RUSTFS_INTERNAL_PREFIX}{suffix}"); + let minio_key = format!("{MINIO_INTERNAL_PREFIX}{suffix}"); + lower == rustfs_key || lower == minio_key +} + +/// Strips x-rustfs-internal- or x-minio-internal- prefix from key. Returns the suffix part. +/// Case-insensitive. Returns None if key is not an internal key. +pub fn strip_internal_prefix(key: &str) -> Option { + let lower = key.to_lowercase(); + lower + .strip_prefix(RUSTFS_INTERNAL_PREFIX) + .or_else(|| lower.strip_prefix(MINIO_INTERNAL_PREFIX)) + .map(|s| s.to_string()) +} + +/// Returns true if key is internal and its suffix part starts with the given suffix_prefix. +/// E.g. internal_key_starts_with("x-rustfs-internal-replication-reset-arn1", "replication-reset") == true. +pub fn internal_key_starts_with(key: &str, suffix_prefix: &str) -> bool { + strip_internal_prefix(key).is_some_and(|s| s.starts_with(suffix_prefix)) +} + +/// For keys like x-rustfs-internal-replication-reset-{arn}, strips the internal prefix and suffix_prefix, +/// returning the remainder (e.g. "arn1"). Returns None if key does not match. +pub fn internal_key_strip_suffix_prefix(key: &str, suffix_prefix: &str) -> Option { + let rest = strip_internal_prefix(key)?; + rest.strip_prefix(suffix_prefix).map(|s| s.to_string()) +} + +fn both_keys(suffix: &str) -> (String, String) { + (format!("{RUSTFS_INTERNAL_PREFIX}{suffix}"), format!("{MINIO_INTERNAL_PREFIX}{suffix}")) +} + +/// Builds the RustFS internal key for the given suffix. Use when a single key is needed (e.g. for +/// backward compat). Prefer insert_str/get_str when both keys should be written/read. +pub fn internal_key_rustfs(suffix: &str) -> String { + format!("{RUSTFS_INTERNAL_PREFIX}{suffix}") +} + +// === String type (FileInfo.metadata, user_defined) === + +pub fn insert_str(map: &mut HashMap, suffix: &str, value: String) { + let (k1, k2) = both_keys(suffix); + map.insert(k1, value.clone()); + map.insert(k2, value); +} + +pub fn get_str(map: &HashMap, suffix: &str) -> Option { + let (k1, k2) = both_keys(suffix); + map.get(&k1).cloned().or_else(|| map.get(&k2).cloned()) +} + +pub fn contains_key_str(map: &HashMap, suffix: &str) -> bool { + let (k1, k2) = both_keys(suffix); + map.contains_key(&k1) || map.contains_key(&k2) +} + +pub fn remove_str(map: &mut HashMap, suffix: &str) { + let (k1, k2) = both_keys(suffix); + map.remove(&k1); + map.remove(&k2); +} + +// === Vec type (meta_sys) === + +pub fn insert_bytes(map: &mut HashMap>, suffix: &str, value: Vec) { + let (k1, k2) = both_keys(suffix); + let v = value.clone(); + map.insert(k1, value); + map.insert(k2, v); +} + +pub fn get_bytes(map: &HashMap>, suffix: &str) -> Option> { + let (k1, k2) = both_keys(suffix); + map.get(&k1).cloned().or_else(|| map.get(&k2).cloned()) +} + +pub fn contains_key_bytes(map: &HashMap>, suffix: &str) -> bool { + let (k1, k2) = both_keys(suffix); + map.contains_key(&k1) || map.contains_key(&k2) +} + +pub fn remove_bytes(map: &mut HashMap>, suffix: &str) { + let (k1, k2) = both_keys(suffix); + map.remove(&k1); + map.remove(&k2); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_is_internal_key() { + assert!(is_internal_key("x-rustfs-internal-healing")); + assert!(is_internal_key("x-rustfs-internal-purgestatus")); + assert!(is_internal_key("X-RustFS-Internal-purgestatus")); + assert!(is_internal_key("x-minio-internal-compression")); + assert!(is_internal_key("x-minio-internal-replication-status")); + assert!(is_internal_key("X-Minio-Internal-Compression")); + assert!(!is_internal_key("x-amz-meta-custom")); + assert!(!is_internal_key("content-type")); + assert!(!is_internal_key("x-rustfs-meta-custom")); + } + + #[test] + fn test_has_internal_suffix() { + assert!(has_internal_suffix("x-rustfs-internal-purgestatus", SUFFIX_PURGESTATUS)); + assert!(has_internal_suffix("X-Minio-Internal-purgestatus", SUFFIX_PURGESTATUS)); + assert!(has_internal_suffix("x-minio-internal-compression", SUFFIX_COMPRESSION)); + assert!(has_internal_suffix("x-rustfs-internal-healing", SUFFIX_HEALING)); + assert!(has_internal_suffix("x-minio-internal-data-mov", SUFFIX_DATA_MOV)); + assert!(!has_internal_suffix("x-rustfs-internal-purgestatus", SUFFIX_HEALING)); + assert!(!has_internal_suffix("x-amz-meta-custom", SUFFIX_PURGESTATUS)); + } +} diff --git a/crates/utils/src/http/mod.rs b/crates/utils/src/http/mod.rs index de3761719..ab4e966bc 100644 --- a/crates/utils/src/http/mod.rs +++ b/crates/utils/src/http/mod.rs @@ -12,7 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. +pub mod header_compat; pub mod headers; pub mod ip; +pub mod metadata_compat; +pub use header_compat::*; pub use headers::*; pub use ip::*; +pub use metadata_compat::*; diff --git a/crates/utils/src/obj/metadata.rs b/crates/utils/src/obj/metadata.rs index 2cbda85be..6988f5c3c 100644 --- a/crates/utils/src/obj/metadata.rs +++ b/crates/utils/src/obj/metadata.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::http::{RESERVED_METADATA_PREFIX_LOWER, is_minio_header, is_rustfs_header}; +use crate::http::{is_internal_key, is_minio_header, is_rustfs_header}; use std::collections::HashMap; /// Extract user-defined metadata keys from object metadata. @@ -80,7 +80,7 @@ pub fn extract_user_defined_metadata(metadata: &HashMap) -> Hash for (key, value) in metadata { let lower_key = key.to_ascii_lowercase(); - if lower_key.starts_with(RESERVED_METADATA_PREFIX_LOWER) { + if is_internal_key(key) { continue; } @@ -188,7 +188,6 @@ mod tests { let mut metadata: HashMap = HashMap::new(); metadata.insert("x-rustfs-internal-healing".to_string(), "true".to_string()); metadata.insert("x-rustfs-internal-data-mov".to_string(), "value".to_string()); - metadata.insert("X-RustFS-Internal-purgestatus".to_string(), "status".to_string()); metadata.insert("x-rustfs-meta-custom".to_string(), "custom-value".to_string()); metadata.insert("my-key".to_string(), "my-value".to_string()); diff --git a/docker-compose.decommission.yml b/docker-compose.decommission.yml new file mode 100644 index 000000000..8fcd86ddc --- /dev/null +++ b/docker-compose.decommission.yml @@ -0,0 +1,49 @@ +services: + rustfs-decommission-latest: + image: ${RUSTFS_DOCKER_IMAGE:-rustfs-local:decommission-latest} + entrypoint: + - /bin/sh + - -lc + - | + set -eu + mkdir -p \ + /data/pool0/disk1 /data/pool0/disk2 /data/pool0/disk3 /data/pool0/disk4 \ + /data/pool1/disk1 /data/pool1/disk2 /data/pool1/disk3 /data/pool1/disk4 \ + /logs + exec /usr/bin/rustfs '/data/pool0/disk{1...4}' '/data/pool1/disk{1...4}' + ports: + - "9100:9000" + - "9101:9001" + environment: + RUSTFS_VOLUMES: "/data/pool0/disk{1...4} /data/pool1/disk{1...4}" + RUSTFS_ADDRESS: "0.0.0.0:9000" + RUSTFS_CONSOLE_ADDRESS: "0.0.0.0:9001" + RUSTFS_CONSOLE_ENABLE: "true" + RUSTFS_CORS_ALLOWED_ORIGINS: "*" + RUSTFS_CONSOLE_CORS_ALLOWED_ORIGINS: "*" + RUSTFS_ACCESS_KEY: "rustfsadmin" + RUSTFS_SECRET_KEY: "rustfsadmin" + RUSTFS_OBS_LOGGER_LEVEL: "info" + RUSTFS_OBS_LOG_DIRECTORY: "/logs" + RUST_LOG: "info,rustfs=debug,rustfs_ecstore=debug" + volumes: + - ./deploy/data/decommission:/data + - ./deploy/logs/decommission:/logs + healthcheck: + test: + [ + "CMD", + "sh", + "-c", + "curl -f http://127.0.0.1:9000/health && curl -f http://127.0.0.1:9001/rustfs/console/health" + ] + interval: 20s + timeout: 10s + retries: 6 + start_period: 300s + restart: unless-stopped + networks: + - rustfs-decommission-network + +networks: + rustfs-decommission-network: diff --git a/entrypoint.sh b/entrypoint.sh index d65c1ee7f..94e7b8eed 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -31,11 +31,12 @@ process_data_volumes() { VOLUME_LIST="" for vol in $VOLUME_LIST_RAW; do # Helper to manually expand {N..M} since sh doesn't support it on variables - if echo "$vol" | grep -E -q "\{[0-9]+\.\.[0-9]+\}"; then + if echo "$vol" | grep -E -q "\{[0-9]+\.\.\.?[0-9]+\}"; then PREFIX=${vol%%\{*} SUFFIX=${vol##*\}} RANGE=${vol#*\{} RANGE=${RANGE%\}} + RANGE=$(echo "$RANGE" | sed 's/\.\.\./../') START=${RANGE%%..*} END=${RANGE##*..} diff --git a/rustfs/Cargo.toml b/rustfs/Cargo.toml index 8669373f4..7a40250be 100644 --- a/rustfs/Cargo.toml +++ b/rustfs/Cargo.toml @@ -49,6 +49,7 @@ rustfs-appauth = { workspace = true } rustfs-audit = { workspace = true } rustfs-common = { workspace = true } rustfs-config = { workspace = true, features = ["constants", "notify"] } +rustfs-crypto = { workspace = true } rustfs-credentials = { workspace = true } rustfs-ecstore = { workspace = true } rustfs-filemeta.workspace = true @@ -168,6 +169,7 @@ aws-sdk-s3 = { workspace = true } aws-config = { workspace = true } anyhow = { workspace = true } tokio = { workspace = true, features = ["test-util"] } +temp-env = { workspace = true } [build-dependencies] http.workspace = true diff --git a/rustfs/src/admin/console.rs b/rustfs/src/admin/console.rs index 4bb1ef321..cb2a7158b 100644 --- a/rustfs/src/admin/console.rs +++ b/rustfs/src/admin/console.rs @@ -142,7 +142,7 @@ impl Config { Config { port, api: Api { - base_url: format!("{http_prefix}{local_ip}:{port}/{RUSTFS_ADMIN_PREFIX}"), + base_url: build_console_api_base_url(&format!("{http_prefix}{local_ip}:{port}")), }, s3: S3 { endpoint: format!("{http_prefix}{local_ip}:{port}"), @@ -192,6 +192,10 @@ impl Config { } } +fn build_console_api_base_url(base_url: &str) -> String { + format!("{base_url}{RUSTFS_ADMIN_PREFIX}") +} + #[derive(Debug, Serialize, Clone)] struct Api { #[serde(rename = "baseURL")] @@ -350,7 +354,7 @@ async fn config_handler(uri: Uri, headers: HeaderMap) -> impl IntoResponse { }; let url = format!("{}://{}:{}", scheme, host_for_url, cfg.port); - cfg.api.base_url = format!("{url}{RUSTFS_ADMIN_PREFIX}"); + cfg.api.base_url = build_console_api_base_url(&url); cfg.s3.endpoint = url; Response::builder() @@ -646,3 +650,38 @@ pub(crate) fn make_console_server() -> Router { // Build console router with enhanced middleware stack using tower-http features setup_console_middleware_stack(cors_layer, rate_limit_enable, rate_limit_rpm, auth_timeout) } + +#[cfg(test)] +mod tests { + use super::*; + use std::net::{IpAddr, Ipv4Addr}; + + #[test] + fn console_api_base_url_keeps_rustfs_admin_prefix() { + let cfg = Config::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 9001, "test", "2026-03-16T00:00:00Z"); + + assert!( + cfg.api.base_url.ends_with("/rustfs/admin/v3"), + "console baseURL must keep using the RustFS admin prefix" + ); + assert!( + !cfg.api.base_url.ends_with("/minio/admin/v3"), + "console baseURL must not switch to the MinIO admin prefix by default" + ); + } + + #[test] + fn console_api_base_url_builder_preserves_existing_console_contract() { + assert_eq!( + build_console_api_base_url("http://127.0.0.1:9001"), + "http://127.0.0.1:9001/rustfs/admin/v3" + ); + } + + #[test] + fn external_admin_paths_are_not_console_paths() { + assert!(is_console_path("/rustfs/console/")); + assert!(!is_console_path("/minio/admin/v3/info")); + assert!(!is_console_path("/rustfs/admin/v3/info")); + } +} diff --git a/rustfs/src/admin/console_test.rs b/rustfs/src/admin/console_test.rs index 65e58e518..6fa36cc13 100644 --- a/rustfs/src/admin/console_test.rs +++ b/rustfs/src/admin/console_test.rs @@ -15,7 +15,6 @@ #[cfg(test)] mod tests { use crate::config::Opt; - use clap::Parser; use serial_test::serial; #[tokio::test] diff --git a/rustfs/src/admin/handlers/policies.rs b/rustfs/src/admin/handlers/policies.rs index b96ee8d2c..c70769cc8 100644 --- a/rustfs/src/admin/handlers/policies.rs +++ b/rustfs/src/admin/handlers/policies.rs @@ -16,7 +16,7 @@ use crate::{ admin::{ auth::validate_admin_request, router::{AdminOperation, Operation, S3Router}, - utils::has_space_be, + utils::{encode_compatible_admin_payload, has_space_be, read_compatible_admin_body}, }, auth::{check_key_valid, get_session_token}, server::{ADMIN_PREFIX, RemoteAddr}, @@ -28,6 +28,7 @@ use rustfs_config::MAX_ADMIN_REQUEST_BODY_SIZE; use rustfs_credentials::get_global_action_cred; use rustfs_iam::error::is_err_no_such_user; use rustfs_iam::store::MappedPolicy; +use rustfs_madmin::{GroupPolicyEntities, PolicyEntities, PolicyEntitiesResult, UserPolicyEntities}; use rustfs_policy::policy::{ Policy, action::{Action, AdminAction}, @@ -37,10 +38,12 @@ use s3s::{ header::{CONTENT_LENGTH, CONTENT_TYPE}, s3_error, }; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use serde_urlencoded::from_bytes; use std::collections::HashMap; +use time::OffsetDateTime; use tracing::warn; +use url::form_urlencoded; pub fn register_iam_policy_route(r: &mut S3Router) -> std::io::Result<()> { r.insert( @@ -72,6 +75,26 @@ pub fn register_iam_policy_route(r: &mut S3Router) -> std::io::R format!("{}{}", ADMIN_PREFIX, "/v3/set-user-or-group-policy").as_str(), AdminOperation(&SetPolicyForUserOrGroup {}), )?; + r.insert( + Method::PUT, + format!("{}{}", ADMIN_PREFIX, "/v3/set-policy").as_str(), + AdminOperation(&SetPolicyForUserOrGroup {}), + )?; + r.insert( + Method::POST, + format!("{}{}", ADMIN_PREFIX, "/v3/idp/builtin/policy/attach").as_str(), + AdminOperation(&AttachPolicyBuiltin {}), + )?; + r.insert( + Method::POST, + format!("{}{}", ADMIN_PREFIX, "/v3/idp/builtin/policy/detach").as_str(), + AdminOperation(&DetachPolicyBuiltin {}), + )?; + r.insert( + Method::GET, + format!("{}{}", ADMIN_PREFIX, "/v3/idp/builtin/policy-entities").as_str(), + AdminOperation(&ListPolicyEntitiesBuiltin {}), + )?; Ok(()) } @@ -323,11 +346,11 @@ impl Operation for RemoveCannedPolicy { #[derive(Debug, Deserialize, Default)] pub struct SetPolicyForUserOrGroupQuery { - #[serde(rename = "policyName")] + #[serde(rename = "policyName", alias = "policy")] pub policy_name: String, - #[serde(rename = "userOrGroup")] + #[serde(rename = "userOrGroup", alias = "user-or-group")] pub user_or_group: String, - #[serde(rename = "isGroup")] + #[serde(rename = "isGroup", alias = "is-group")] pub is_group: bool, } @@ -419,3 +442,552 @@ impl Operation for SetPolicyForUserOrGroup { Ok(S3Response::with_headers((StatusCode::OK, Body::empty()), header)) } } + +#[derive(Debug, Deserialize, Default)] +struct PolicyAssociationReq { + #[serde(default)] + policies: Vec, + #[serde(default)] + user: String, + #[serde(default)] + group: String, +} + +#[derive(Debug, Serialize)] +struct PolicyAssociationResp { + #[serde(rename = "policiesAttached", skip_serializing_if = "Vec::is_empty")] + policies_attached: Vec, + #[serde(rename = "policiesDetached", skip_serializing_if = "Vec::is_empty")] + policies_detached: Vec, + #[serde(rename = "updatedAt", with = "time::serde::rfc3339")] + updated_at: OffsetDateTime, +} + +#[derive(Debug, Deserialize, Default)] +struct PolicyEntitiesQuery { + users: Vec, + groups: Vec, + policies: Vec, +} + +fn split_policy_names(policy_names: &str) -> Vec { + policy_names + .split(',') + .map(str::trim) + .filter(|name| !name.is_empty()) + .map(ToOwned::to_owned) + .collect() +} + +fn attach_policy_names(existing: &[String], requested: &[String]) -> (Vec, Vec) { + let mut updated = existing.to_vec(); + let mut attached = Vec::new(); + + for policy in requested { + if updated.iter().any(|current| current == policy) { + continue; + } + updated.push(policy.clone()); + attached.push(policy.clone()); + } + + (updated, attached) +} + +fn detach_policy_names(existing: &[String], requested: &[String]) -> (Vec, Vec) { + let mut detached = Vec::new(); + let updated = existing + .iter() + .filter(|policy| { + let should_detach = requested.iter().any(|requested_policy| requested_policy == *policy); + if should_detach { + detached.push((*policy).clone()); + } + !should_detach + }) + .cloned() + .collect(); + + (updated, detached) +} + +fn validate_policy_association_req(req: &PolicyAssociationReq) -> S3Result<()> { + if req.policies.is_empty() { + return Err(s3_error!(InvalidArgument, "no policy names were given")); + } + + if req.policies.iter().any(|policy| policy.is_empty()) { + return Err(s3_error!(InvalidArgument, "an empty policy name was given")); + } + + let has_user = !req.user.is_empty(); + let has_group = !req.group.is_empty(); + + match (has_user, has_group) { + (false, false) => Err(s3_error!(InvalidArgument, "no user or group association was given")), + (true, true) => Err(s3_error!(InvalidArgument, "either a group or a user association must be given, not both")), + _ => Ok(()), + } +} + +fn parse_policy_entities_query(query: Option<&str>) -> PolicyEntitiesQuery { + let mut parsed = PolicyEntitiesQuery::default(); + let Some(query) = query else { + return parsed; + }; + + for (key, value) in form_urlencoded::parse(query.as_bytes()) { + match key.as_ref() { + "user" => parsed.users.push(value.into_owned()), + "group" => parsed.groups.push(value.into_owned()), + "policy" => parsed.policies.push(value.into_owned()), + _ => {} + } + } + + parsed +} + +fn split_policy_list(policy_names: Option<&String>) -> Vec { + policy_names.map_or_else(Vec::new, |names| split_policy_names(names)) +} + +fn direct_user_policy_names(user_info: &rustfs_madmin::UserInfo) -> Vec { + split_policy_list(user_info.policy_name.as_ref()) +} + +fn sorted_group_policy_entities(mut entities: Vec) -> Vec { + entities.sort_by(|left, right| left.group.cmp(&right.group)); + entities +} + +fn build_policy_mappings( + user_mappings: &[UserPolicyEntities], + group_mappings: &[GroupPolicyEntities], + requested_policies: &[String], +) -> Vec { + let mut policy_map: HashMap = HashMap::new(); + + for user_mapping in user_mappings { + for policy in &user_mapping.policies { + let entry = policy_map.entry(policy.clone()).or_insert_with(|| PolicyEntities { + policy: policy.clone(), + ..Default::default() + }); + entry.users.push(user_mapping.user.clone()); + } + } + + for group_mapping in group_mappings { + for policy in &group_mapping.policies { + let entry = policy_map.entry(policy.clone()).or_insert_with(|| PolicyEntities { + policy: policy.clone(), + ..Default::default() + }); + entry.groups.push(group_mapping.group.clone()); + } + } + + let mut results: Vec = policy_map + .into_iter() + .filter_map(|(_, mut mapping)| { + if !requested_policies.is_empty() && !requested_policies.iter().any(|policy| policy == &mapping.policy) { + return None; + } + mapping.users.sort(); + mapping.users.dedup(); + mapping.groups.sort(); + mapping.groups.dedup(); + Some(mapping) + }) + .collect(); + results.sort_by(|left, right| left.policy.cmp(&right.policy)); + results +} + +async fn collect_group_policy_mappings( + iam_store: &std::sync::Arc>, + requested_groups: &[String], +) -> S3Result> { + let mut mappings = HashMap::new(); + let groups = if requested_groups.is_empty() { + iam_store + .list_groups_load() + .await + .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, e.to_string()))? + } else { + requested_groups.to_vec() + }; + + for group in groups { + let group_desc = iam_store.get_group_description(&group).await.map_err(|e| { + warn!("get group description failed, e: {:?}", e); + S3Error::with_message(S3ErrorCode::InternalError, e.to_string()) + })?; + let policies = split_policy_names(&group_desc.policy); + if policies.is_empty() { + continue; + } + mappings.insert(group.clone(), GroupPolicyEntities { group, policies }); + } + + Ok(mappings) +} + +async fn handle_builtin_policy_entities(req: S3Request) -> S3Result> { + let Some(input_cred) = req.credentials else { + return Err(s3_error!(InvalidRequest, "get cred failed")); + }; + + let (cred, owner) = + check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?; + + validate_admin_request( + &req.headers, + &cred, + owner, + false, + vec![ + Action::AdminAction(AdminAction::ListGroupsAdminAction), + Action::AdminAction(AdminAction::ListUsersAdminAction), + Action::AdminAction(AdminAction::ListUserPoliciesAdminAction), + ], + req.extensions.get::>().and_then(|opt| opt.map(|a| a.0)), + ) + .await?; + + let query = parse_policy_entities_query(req.uri.query()); + + let Ok(iam_store) = rustfs_iam::get() else { return Err(s3_error!(InternalError, "iam not init")) }; + + let all_group_policy_mappings = collect_group_policy_mappings(&iam_store, &[]).await?; + let users = iam_store.list_users().await.map_err(|e| { + warn!("list users failed, e: {:?}", e); + S3Error::with_message(S3ErrorCode::InternalError, e.to_string()) + })?; + + let all_user_policy_mappings = users + .iter() + .filter_map(|(user, user_info)| { + let policies = split_policy_list(user_info.policy_name.as_ref()); + if policies.is_empty() { + return None; + } + Some(UserPolicyEntities { + user: user.clone(), + policies, + member_of_mappings: Vec::new(), + }) + }) + .collect::>(); + + let user_mappings = if query.users.is_empty() { + Vec::new() + } else { + let mut mappings = Vec::new(); + for user in &query.users { + let Some(user_info) = users.get(user) else { + continue; + }; + + let mut member_of_mappings = user_info + .member_of + .as_ref() + .map(|groups| { + groups + .iter() + .filter_map(|group| all_group_policy_mappings.get(group).cloned()) + .collect::>() + }) + .unwrap_or_default(); + member_of_mappings = sorted_group_policy_entities(member_of_mappings); + + let policies = split_policy_list(user_info.policy_name.as_ref()); + if policies.is_empty() && member_of_mappings.is_empty() { + continue; + } + + mappings.push(UserPolicyEntities { + user: user.clone(), + policies, + member_of_mappings, + }); + } + mappings.sort_by(|left, right| left.user.cmp(&right.user)); + mappings + }; + + let mut group_mappings = if query.groups.is_empty() { + Vec::new() + } else { + query + .groups + .iter() + .filter_map(|group| all_group_policy_mappings.get(group).cloned()) + .collect::>() + }; + group_mappings = sorted_group_policy_entities(group_mappings); + + let policy_mappings = if query.users.is_empty() && query.groups.is_empty() && query.policies.is_empty() { + build_policy_mappings( + &all_user_policy_mappings, + &all_group_policy_mappings.values().cloned().collect::>(), + &[], + ) + } else if !query.policies.is_empty() { + build_policy_mappings( + &all_user_policy_mappings, + &all_group_policy_mappings.values().cloned().collect::>(), + &query.policies, + ) + } else { + Vec::new() + }; + + let resp = PolicyEntitiesResult { + timestamp: OffsetDateTime::now_utc(), + user_mappings, + group_mappings, + policy_mappings, + }; + + let req_path = req.uri.path().to_string(); + let body = + serde_json::to_vec(&resp).map_err(|e| s3_error!(InternalError, "marshal policy entities body failed, e: {:?}", e))?; + let (body, content_type) = encode_compatible_admin_payload(&req_path, &cred.secret_key, body)?; + + let mut header = HeaderMap::new(); + header.insert(CONTENT_TYPE, content_type.parse().unwrap()); + header.insert(CONTENT_LENGTH, body.len().to_string().parse().unwrap()); + Ok(S3Response::with_headers((StatusCode::OK, Body::from(body)), header)) +} + +async fn handle_builtin_policy_association(req: S3Request, is_attach: bool) -> S3Result> { + let Some(input_cred) = req.credentials else { + return Err(s3_error!(InvalidRequest, "get cred failed")); + }; + + let (cred, owner) = + check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?; + + validate_admin_request( + &req.headers, + &cred, + owner, + false, + vec![Action::AdminAction(AdminAction::AttachPolicyAdminAction)], + req.extensions.get::>().and_then(|opt| opt.map(|a| a.0)), + ) + .await?; + + let req_path = req.uri.path().to_string(); + let body = read_compatible_admin_body(req.input, MAX_ADMIN_REQUEST_BODY_SIZE, &req_path, &cred.secret_key).await?; + let assoc_req: PolicyAssociationReq = serde_json::from_slice(&body) + .map_err(|e| s3_error!(InvalidRequest, "unmarshal policy association body failed, e: {:?}", e))?; + validate_policy_association_req(&assoc_req)?; + + let Ok(iam_store) = rustfs_iam::get() else { return Err(s3_error!(InternalError, "iam not init")) }; + + let (target_name, is_group, existing_policies) = if !assoc_req.user.is_empty() { + match iam_store.is_temp_user(&assoc_req.user).await { + Ok((true, _)) => return Err(s3_error!(InvalidArgument, "temp user can't set policy")), + Ok((false, _)) => {} + Err(err) => { + if !is_err_no_such_user(&err) { + warn!("is temp user failed, e: {:?}", err); + return Err(S3Error::with_message(S3ErrorCode::InternalError, err.to_string())); + } + } + } + + let Some(sys_cred) = get_global_action_cred() else { + return Err(s3_error!(InternalError, "get global action cred failed")); + }; + + if assoc_req.user == sys_cred.access_key { + return Err(s3_error!(InvalidArgument, "can't set policy for system user")); + } + + if iam_store.get_user(&assoc_req.user).await.is_none() { + return Err(s3_error!(InvalidArgument, "user not exist")); + } + + let user_info = iam_store.get_user_info(&assoc_req.user).await.map_err(|e| { + warn!("get user info failed, e: {:?}", e); + S3Error::with_message(S3ErrorCode::InternalError, e.to_string()) + })?; + + (assoc_req.user, false, direct_user_policy_names(&user_info)) + } else { + let group_desc = iam_store.get_group_description(&assoc_req.group).await.map_err(|e| { + warn!("get group description failed, e: {:?}", e); + S3Error::with_message(S3ErrorCode::InternalError, e.to_string()) + })?; + + (assoc_req.group, true, split_policy_names(&group_desc.policy)) + }; + + let (updated_policies, changed_policies) = if is_attach { + attach_policy_names(&existing_policies, &assoc_req.policies) + } else { + detach_policy_names(&existing_policies, &assoc_req.policies) + }; + + let updated_at = iam_store + .policy_db_set(&target_name, rustfs_iam::store::UserType::Reg, is_group, &updated_policies.join(",")) + .await + .map_err(|e| { + warn!("policy db set failed, e: {:?}", e); + S3Error::with_message(S3ErrorCode::InternalError, e.to_string()) + })?; + + let policies_attached = if is_attach { changed_policies.clone() } else { Vec::new() }; + let policies_detached = if is_attach { Vec::new() } else { changed_policies }; + + let resp = PolicyAssociationResp { + policies_attached, + policies_detached, + updated_at, + }; + + let body = serde_json::to_vec(&resp).map_err(|e| s3_error!(InternalError, "marshal body failed, e: {:?}", e))?; + let (body, content_type) = encode_compatible_admin_payload(&req_path, &cred.secret_key, body)?; + + let mut header = HeaderMap::new(); + header.insert(CONTENT_TYPE, content_type.parse().unwrap()); + header.insert(CONTENT_LENGTH, body.len().to_string().parse().unwrap()); + Ok(S3Response::with_headers((StatusCode::OK, Body::from(body)), header)) +} + +pub struct AttachPolicyBuiltin {} +#[async_trait::async_trait] +impl Operation for AttachPolicyBuiltin { + async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { + handle_builtin_policy_association(req, true).await + } +} + +pub struct DetachPolicyBuiltin {} +#[async_trait::async_trait] +impl Operation for DetachPolicyBuiltin { + async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { + handle_builtin_policy_association(req, false).await + } +} + +pub struct ListPolicyEntitiesBuiltin {} +#[async_trait::async_trait] +impl Operation for ListPolicyEntitiesBuiltin { + async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { + handle_builtin_policy_entities(req).await + } +} + +#[cfg(test)] +mod tests { + use super::{ + GroupPolicyEntities, PolicyAssociationReq, SetPolicyForUserOrGroupQuery, UserPolicyEntities, attach_policy_names, + build_policy_mappings, detach_policy_names, direct_user_policy_names, parse_policy_entities_query, + validate_policy_association_req, + }; + use rustfs_madmin::UserInfo; + + #[test] + fn set_policy_query_supports_external_parameter_names() { + let query: SetPolicyForUserOrGroupQuery = + serde_urlencoded::from_str("policy=readwrite&user-or-group=test-user&is-group=true").expect("query should parse"); + + assert_eq!(query.policy_name, "readwrite"); + assert_eq!(query.user_or_group, "test-user"); + assert!(query.is_group); + } + + #[test] + fn set_policy_query_supports_rustfs_parameter_names() { + let query: SetPolicyForUserOrGroupQuery = + serde_urlencoded::from_str("policyName=readwrite&userOrGroup=test-user&isGroup=false").expect("query should parse"); + + assert_eq!(query.policy_name, "readwrite"); + assert_eq!(query.user_or_group, "test-user"); + assert!(!query.is_group); + } + + #[test] + fn policy_association_req_requires_exactly_one_target() { + let err = validate_policy_association_req(&PolicyAssociationReq { + policies: vec!["readonly".to_string()], + user: "user-a".to_string(), + group: "group-a".to_string(), + }) + .expect_err("request should be invalid"); + + assert_eq!(*err.code(), s3s::S3ErrorCode::InvalidArgument); + } + + #[test] + fn attach_policy_names_appends_only_missing_values() { + let existing = vec!["readonly".to_string()]; + let requested = vec!["readonly".to_string(), "writeonly".to_string()]; + + let (updated, attached) = attach_policy_names(&existing, &requested); + + assert_eq!(updated, vec!["readonly".to_string(), "writeonly".to_string()]); + assert_eq!(attached, vec!["writeonly".to_string()]); + } + + #[test] + fn detach_policy_names_removes_only_requested_values() { + let existing = vec!["readonly".to_string(), "writeonly".to_string()]; + let requested = vec!["writeonly".to_string(), "missing".to_string()]; + + let (updated, detached) = detach_policy_names(&existing, &requested); + + assert_eq!(updated, vec!["readonly".to_string()]); + assert_eq!(detached, vec!["writeonly".to_string()]); + } + + #[test] + fn policy_entities_query_supports_repeated_external_parameters() { + let query = parse_policy_entities_query(Some("user=alice&user=bob&group=ops&policy=readonly&policy=writeonly")); + + assert_eq!(query.users, vec!["alice".to_string(), "bob".to_string()]); + assert_eq!(query.groups, vec!["ops".to_string()]); + assert_eq!(query.policies, vec!["readonly".to_string(), "writeonly".to_string()]); + } + + #[test] + fn build_policy_mappings_indexes_users_and_groups() { + let user_mappings = vec![UserPolicyEntities { + user: "alice".to_string(), + policies: vec!["readonly".to_string()], + member_of_mappings: Vec::new(), + }]; + let group_mappings = vec![GroupPolicyEntities { + group: "ops".to_string(), + policies: vec!["readonly".to_string(), "writeonly".to_string()], + }]; + + let mappings = build_policy_mappings(&user_mappings, &group_mappings, &[]); + + assert_eq!(mappings.len(), 2); + assert_eq!(mappings[0].policy, "readonly"); + assert_eq!(mappings[0].users, vec!["alice".to_string()]); + assert_eq!(mappings[0].groups, vec!["ops".to_string()]); + assert_eq!(mappings[1].policy, "writeonly"); + assert_eq!(mappings[1].groups, vec!["ops".to_string()]); + } + + #[test] + fn direct_user_policy_names_only_reads_direct_mapping() { + let user_info = UserInfo { + policy_name: Some("readonly,writeonly".to_string()), + member_of: Some(vec!["disabled-group".to_string()]), + ..Default::default() + }; + + assert_eq!( + direct_user_policy_names(&user_info), + vec!["readonly".to_string(), "writeonly".to_string()] + ); + } +} diff --git a/rustfs/src/admin/handlers/pools.rs b/rustfs/src/admin/handlers/pools.rs index 9c3d385cc..34f5d9930 100644 --- a/rustfs/src/admin/handlers/pools.rs +++ b/rustfs/src/admin/handlers/pools.rs @@ -220,7 +220,12 @@ impl Operation for StartDecommission { )); } - // TODO: check IsRebalanceStarted + if store.is_rebalance_started().await { + return Err(S3Error::with_message( + S3ErrorCode::OperationAborted, + "Decommission cannot be started, rebalance is already in progress".to_string(), + )); + } let query = { if let Some(query) = req.uri.query() { diff --git a/rustfs/src/admin/handlers/quota.rs b/rustfs/src/admin/handlers/quota.rs index 3f9cce4ab..fbbcdba50 100644 --- a/rustfs/src/admin/handlers/quota.rs +++ b/rustfs/src/admin/handlers/quota.rs @@ -31,6 +31,7 @@ use serde_json; use std::sync::Arc; use tokio::sync::RwLock; use tracing::{debug, info, warn}; +use url::form_urlencoded; #[derive(Debug, Deserialize)] pub struct SetBucketQuotaRequest { @@ -39,10 +40,86 @@ pub struct SetBucketQuotaRequest { pub quota_type: String, } +#[derive(Debug, Deserialize)] +struct CompatibleBucketQuotaRequest { + #[serde(default)] + quota: Option, + #[serde(default)] + size: Option, + #[serde(default, alias = "quotatype")] + quota_type: Option, +} + +#[derive(Debug, Serialize)] +struct CompatibleBucketQuotaResponse { + quota: u64, + size: u64, + rate: u64, + requests: u64, + #[serde(skip_serializing_if = "String::is_empty")] + quotatype: String, +} + fn default_quota_type() -> String { rustfs_config::QUOTA_TYPE_HARD.to_string() } +fn is_compat_set_bucket_quota_path(path: &str) -> bool { + path.ends_with("/v3/set-bucket-quota") +} + +fn is_compat_get_bucket_quota_path(path: &str) -> bool { + path.ends_with("/v3/get-bucket-quota") +} + +fn bucket_from_params_or_query(params: &Params<'_, '_>, uri: &hyper::Uri) -> String { + if let Some(bucket) = params.get("bucket") { + return bucket.to_string(); + } + + if let Some(query) = uri.query() { + for (key, value) in form_urlencoded::parse(query.as_bytes()) { + if key == "bucket" { + return value.into_owned(); + } + } + } + + String::new() +} + +fn parse_set_bucket_quota_request(body: &[u8]) -> Result { + if body.is_empty() { + return Ok(SetBucketQuotaRequest { + quota: None, + quota_type: default_quota_type(), + }); + } + + let request: CompatibleBucketQuotaRequest = + serde_json::from_slice(body).map_err(|e| s3_error!(InvalidRequest, "invalid JSON: {}", e))?; + + Ok(SetBucketQuotaRequest { + quota: request + .size + .filter(|quota| *quota > 0) + .or(request.quota.filter(|quota| *quota > 0)), + quota_type: request.quota_type.unwrap_or_else(default_quota_type), + }) +} + +fn compat_bucket_quota_response(quota: &BucketQuota) -> CompatibleBucketQuotaResponse { + let size = quota.quota.unwrap_or(0); + + CompatibleBucketQuotaResponse { + quota: size, + size, + rate: 0, + requests: 0, + quotatype: if size > 0 { "hard".to_string() } else { String::new() }, + } +} + #[derive(Debug, Serialize)] pub struct BucketQuotaResponse { pub bucket: String, @@ -105,6 +182,18 @@ async fn current_usage_from_context(bucket: &str) -> u64 { } pub fn register_quota_route(r: &mut S3Router) -> std::io::Result<()> { + r.insert( + Method::PUT, + format!("{}{}", ADMIN_PREFIX, "/v3/set-bucket-quota").as_str(), + AdminOperation(&SetBucketQuotaHandler {}), + )?; + + r.insert( + Method::GET, + format!("{}{}", ADMIN_PREFIX, "/v3/get-bucket-quota").as_str(), + AdminOperation(&GetBucketQuotaHandler {}), + )?; + r.insert( Method::PUT, format!("{}{}", ADMIN_PREFIX, "/v3/quota/{bucket}").as_str(), @@ -161,7 +250,7 @@ impl Operation for SetBucketQuotaHandler { ) .await?; - let bucket = params.get("bucket").unwrap_or("").to_string(); + let bucket = bucket_from_params_or_query(¶ms, &req.uri); if bucket.is_empty() { return Err(s3_error!(InvalidRequest, "bucket name is required")); } @@ -172,14 +261,7 @@ impl Operation for SetBucketQuotaHandler { .await .map_err(|e| s3_error!(InvalidRequest, "failed to read request body: {}", e))?; - let request: SetBucketQuotaRequest = if body.is_empty() { - SetBucketQuotaRequest { - quota: None, - quota_type: default_quota_type(), - } - } else { - serde_json::from_slice(&body).map_err(|e| s3_error!(InvalidRequest, "invalid JSON: {}", e))? - }; + let request = parse_set_bucket_quota_request(&body)?; if request.quota_type.to_uppercase() != rustfs_config::QUOTA_TYPE_HARD { return Err(s3_error!(InvalidArgument, "{}", rustfs_config::QUOTA_INVALID_TYPE_ERROR_MSG)); @@ -199,15 +281,18 @@ impl Operation for SetBucketQuotaHandler { // Get real-time usage from data usage system let current_usage = current_usage_from_context(&bucket).await; - let response = BucketQuotaResponse { - bucket, - quota: quota.quota, - size: current_usage, - quota_type: rustfs_config::QUOTA_TYPE_HARD.to_string(), - }; + let json = if is_compat_set_bucket_quota_path(req.uri.path()) { + String::new() + } else { + let response = BucketQuotaResponse { + bucket, + quota: quota.quota, + size: current_usage, + quota_type: rustfs_config::QUOTA_TYPE_HARD.to_string(), + }; - let json = - serde_json::to_string(&response).map_err(|e| s3_error!(InternalError, "Failed to serialize response: {}", e))?; + serde_json::to_string(&response).map_err(|e| s3_error!(InternalError, "Failed to serialize response: {}", e))? + }; Ok(S3Response::new((StatusCode::OK, Body::from(json)))) } @@ -226,7 +311,7 @@ impl Operation for GetBucketQuotaHandler { let (cred, owner) = check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &cred.access_key).await?; - let bucket = params.get("bucket").unwrap_or("").to_string(); + let bucket = bucket_from_params_or_query(¶ms, &req.uri); if bucket.is_empty() { return Err(s3_error!(InvalidRequest, "bucket name is required")); } @@ -254,15 +339,19 @@ impl Operation for GetBucketQuotaHandler { _ => s3_error!(InternalError, "Failed to get quota: {}", e), })?; - let response = BucketQuotaResponse { - bucket, - quota: quota.quota, - size: current_usage.unwrap_or(0), - quota_type: rustfs_config::QUOTA_TYPE_HARD.to_string(), - }; + let json = if is_compat_get_bucket_quota_path(req.uri.path()) { + serde_json::to_string(&compat_bucket_quota_response("a)) + .map_err(|e| s3_error!(InternalError, "Failed to serialize response: {}", e))? + } else { + let response = BucketQuotaResponse { + bucket, + quota: quota.quota, + size: current_usage.unwrap_or(0), + quota_type: rustfs_config::QUOTA_TYPE_HARD.to_string(), + }; - let json = - serde_json::to_string(&response).map_err(|e| s3_error!(InternalError, "Failed to serialize response: {}", e))?; + serde_json::to_string(&response).map_err(|e| s3_error!(InternalError, "Failed to serialize response: {}", e))? + }; Ok(S3Response::new((StatusCode::OK, Body::from(json)))) } @@ -486,6 +575,34 @@ mod tests { assert_eq!(default_quota_type(), "HARD"); } + #[test] + fn parse_set_bucket_quota_request_accepts_compat_shape() { + let request = parse_set_bucket_quota_request(br#"{"quota":1073741824,"size":1073741824,"quotatype":"hard"}"#) + .expect("parse quota request"); + + assert_eq!(request.quota, Some(1073741824)); + assert_eq!(request.quota_type, "hard"); + } + + #[test] + fn parse_set_bucket_quota_request_prefers_non_zero_quota_over_zero_size() { + let request = + parse_set_bucket_quota_request(br#"{"quota":1073741824,"size":0,"quotatype":"hard"}"#).expect("parse quota request"); + + assert_eq!(request.quota, Some(1073741824)); + assert_eq!(request.quota_type, "hard"); + } + + #[test] + fn compat_bucket_quota_response_uses_external_field_names() { + let quota = BucketQuota::new(Some(1024)); + let json = serde_json::to_string(&compat_bucket_quota_response("a)).expect("serialize"); + + assert!(json.contains("\"quota\":1024")); + assert!(json.contains("\"size\":1024")); + assert!(json.contains("\"quotatype\":\"hard\"")); + } + #[test] fn test_quota_operation_parsing() { let parse_operation = |operation: &str| match operation.to_uppercase().as_str() { diff --git a/rustfs/src/admin/handlers/service_account.rs b/rustfs/src/admin/handlers/service_account.rs index 29aef0fa6..ccdc13400 100644 --- a/rustfs/src/admin/handlers/service_account.rs +++ b/rustfs/src/admin/handlers/service_account.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::admin::utils::has_space_be; +use crate::admin::utils::{encode_compatible_admin_payload, has_space_be, is_compat_admin_request, read_compatible_admin_body}; use crate::auth::{constant_time_eq, get_condition_values, get_session_token}; use crate::server::{ADMIN_PREFIX, RemoteAddr}; use crate::{ @@ -23,12 +23,14 @@ use http::HeaderMap; use hyper::{Method, StatusCode}; use matchit::Params; use rustfs_config::MAX_ADMIN_REQUEST_BODY_SIZE; -use rustfs_credentials::get_global_action_cred; -use rustfs_iam::error::is_err_no_such_service_account; +use rustfs_credentials::{Credentials as StoredCredentials, get_global_action_cred}; +use rustfs_iam::error::{is_err_no_such_service_account, is_err_no_such_temp_account}; +use rustfs_iam::store::Store as IamStore; use rustfs_iam::sys::{NewServiceAccountOpts, UpdateServiceAccountOpts}; use rustfs_madmin::{ - AddServiceAccountReq, AddServiceAccountResp, Credentials, InfoServiceAccountResp, ListServiceAccountsResp, - ServiceAccountInfo, UpdateServiceAccountReq, + ACCESS_KEY_LIST_ALL, ACCESS_KEY_LIST_STS_ONLY, ACCESS_KEY_LIST_SVCACC_ONLY, ACCESS_KEY_LIST_USERS_ONLY, AddServiceAccountReq, + AddServiceAccountResp, Credentials, InfoAccessKeyResp, InfoServiceAccountResp, LDAPSpecificAccessKeyInfo, ListAccessKeysResp, + ListServiceAccountsResp, OpenIDSpecificAccessKeyInfo, ServiceAccountInfo, TemporaryAccountInfoResp, UpdateServiceAccountReq, }; use rustfs_policy::policy::action::{Action, AdminAction}; use rustfs_policy::policy::{Args, Policy}; @@ -38,7 +40,65 @@ use s3s::{Body, S3Error, S3ErrorCode, S3Request, S3Response, S3Result, header::C use serde::Deserialize; use serde_urlencoded::from_bytes; use std::collections::HashMap; +use time::OffsetDateTime; use tracing::{debug, warn}; +use url::form_urlencoded; + +fn compat_time_sentinel() -> OffsetDateTime { + OffsetDateTime::UNIX_EPOCH +} + +fn list_expiration_or_sentinel(expiration: Option) -> Option { + Some(expiration.unwrap_or_else(compat_time_sentinel)) +} + +fn expiration_for_admin_path(path: &str, expiration: Option) -> Option { + if is_compat_admin_request(path) { + list_expiration_or_sentinel(expiration) + } else { + expiration + } +} + +fn delete_service_account_success_status(path: &str) -> StatusCode { + if is_compat_admin_request(path) { + StatusCode::NO_CONTENT + } else { + StatusCode::OK + } +} + +fn map_service_account_lookup_error(err: rustfs_iam::error::Error, action: &str) -> S3Error { + debug!("{action}, e: {:?}", err); + if is_err_no_such_service_account(&err) { + s3_error!(InvalidRequest, "service account not exist") + } else { + s3_error!(InternalError, "{action}") + } +} + +fn map_temp_account_lookup_error(err: rustfs_iam::error::Error, action: &str) -> S3Error { + debug!("{action}, e: {:?}", err); + if is_err_no_such_temp_account(&err) { + s3_error!(InvalidRequest, "access key not exist") + } else { + s3_error!(InternalError, "{action}") + } +} + +fn parse_update_service_account_policy(new_policy: Option) -> S3Result> { + let Some(policy) = new_policy else { + return Ok(None); + }; + + let policy_bytes = serde_json::to_vec(&policy).map_err(|e| s3_error!(InvalidArgument, "marshal policy failed: {:?}", e))?; + let sp = Policy::parse_config(&policy_bytes).map_err(|e| { + debug!("parse policy failed, e: {:?}", e); + s3_error!(InvalidArgument, "parse policy failed") + })?; + + Ok(Some(sp)) +} pub fn register_service_account_route(r: &mut S3Router) -> std::io::Result<()> { r.insert( @@ -52,24 +112,49 @@ pub fn register_service_account_route(r: &mut S3Router) -> std:: format!("{}{}", ADMIN_PREFIX, "/v3/info-service-account").as_str(), AdminOperation(&InfoServiceAccount {}), )?; + r.insert( + Method::GET, + format!("{}{}", ADMIN_PREFIX, "/v3/temporary-account-info").as_str(), + AdminOperation(&TemporaryAccountInfo {}), + )?; + r.insert( + Method::GET, + format!("{}{}", ADMIN_PREFIX, "/v3/info-access-key").as_str(), + AdminOperation(&InfoAccessKey {}), + )?; r.insert( Method::GET, format!("{}{}", ADMIN_PREFIX, "/v3/list-service-accounts").as_str(), AdminOperation(&ListServiceAccount {}), )?; + r.insert( + Method::GET, + format!("{}{}", ADMIN_PREFIX, "/v3/list-access-keys-bulk").as_str(), + AdminOperation(&ListAccessKeysBulk {}), + )?; r.insert( Method::DELETE, format!("{}{}", ADMIN_PREFIX, "/v3/delete-service-accounts").as_str(), AdminOperation(&DeleteServiceAccount {}), )?; + r.insert( + Method::DELETE, + format!("{}{}", ADMIN_PREFIX, "/v3/delete-service-account").as_str(), + AdminOperation(&DeleteServiceAccount {}), + )?; r.insert( Method::PUT, format!("{}{}", ADMIN_PREFIX, "/v3/add-service-accounts").as_str(), AdminOperation(&AddServiceAccount {}), )?; + r.insert( + Method::PUT, + format!("{}{}", ADMIN_PREFIX, "/v3/add-service-account").as_str(), + AdminOperation(&AddServiceAccount {}), + )?; Ok(()) } @@ -86,17 +171,7 @@ impl Operation for AddServiceAccount { let (cred, owner) = check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &req_cred.access_key).await?; - let mut input = req.input; - let body = match input.store_all_limited(MAX_ADMIN_REQUEST_BODY_SIZE).await { - Ok(b) => b, - Err(e) => { - warn!("get body failed, e: {:?}", e); - return Err(s3_error!( - InvalidRequest, - "service account configuration body too large or failed to read" - )); - } - }; + let body = read_compatible_admin_body(req.input, MAX_ADMIN_REQUEST_BODY_SIZE, req.uri.path(), &cred.secret_key).await?; let create_req: AddServiceAccountReq = serde_json::from_slice(&body[..]).map_err(|e| s3_error!(InvalidRequest, "unmarshal body failed, e: {:?}", e))?; @@ -112,7 +187,9 @@ impl Operation for AddServiceAccount { .map_err(|e| S3Error::with_message(InvalidRequest, e.to_string()))?; let session_policy = if let Some(policy) = &create_req.policy { - let p = Policy::parse_config(policy.as_bytes()).map_err(|e| { + let policy_bytes = + serde_json::to_vec(policy).map_err(|e| s3_error!(InvalidArgument, "marshal policy failed: {:?}", e))?; + let p = Policy::parse_config(&policy_bytes).map_err(|e| { debug!("parse policy failed, e: {:?}", e); s3_error!(InvalidArgument, "parse policy failed") })?; @@ -186,7 +263,7 @@ impl Operation for AddServiceAccount { access_key: create_req.access_key, secret_key: create_req.secret_key, name: create_req.name, - description: create_req.description, + description: create_req.description.or(create_req.comment), expiration: create_req.expiration, session_policy, ..Default::default() @@ -235,9 +312,10 @@ impl Operation for AddServiceAccount { }; let body = serde_json::to_vec(&resp).map_err(|e| s3_error!(InternalError, "marshal body failed, e: {:?}", e))?; + let (body, content_type) = encode_compatible_admin_payload(req.uri.path(), &cred.secret_key, body)?; let mut header = HeaderMap::new(); - header.insert(CONTENT_TYPE, "application/json".parse().unwrap()); + header.insert(CONTENT_TYPE, content_type.parse().unwrap()); Ok(S3Response::with_headers((StatusCode::OK, Body::from(body)), header)) } @@ -245,10 +323,113 @@ impl Operation for AddServiceAccount { #[derive(Debug, Default, Deserialize)] struct AccessKeyQuery { - #[serde(rename = "accessKey")] + #[serde(rename = "accessKey", alias = "access-key")] pub access_key: String, } +fn request_user_name(cred: &StoredCredentials) -> &str { + if cred.parent_user.is_empty() { + &cred.access_key + } else { + &cred.parent_user + } +} + +async fn build_info_service_account_resp( + iam_store: &rustfs_iam::sys::IamSys, + account: &StoredCredentials, + session_policy: Option, +) -> S3Result { + let implied_policy = session_policy + .as_ref() + .is_none_or(|policy| policy.version.is_empty() && policy.statements.is_empty()); + + let effective_policy = if implied_policy { + let policies = iam_store + .policy_db_get(&account.parent_user, &account.groups) + .await + .map_err(|e| { + debug!("get service account policy failed, e: {:?}", e); + s3_error!(InternalError, "get service account policy failed") + })?; + + Some(iam_store.get_combined_policy(&policies).await) + } else { + session_policy + }; + + let policy = effective_policy + .map(|policy| { + serde_json::to_string(&policy).map_err(|e| { + debug!("marshal policy failed, e: {:?}", e); + s3_error!(InternalError, "marshal policy failed") + }) + }) + .transpose()?; + + Ok(InfoServiceAccountResp { + parent_user: account.parent_user.clone(), + account_status: account.status.clone(), + implied_policy, + name: account.name.clone(), + description: account.description.clone(), + expiration: account.expiration, + policy, + }) +} + +fn guess_user_provider(credentials: &StoredCredentials) -> &'static str { + if !credentials.is_service_account() && !credentials.is_temp() { + return "builtin"; + } + + let Some(claims) = credentials.claims.as_ref() else { + return "builtin"; + }; + + if claims.contains_key("ldap:user") || claims.contains_key("ldap:username") { + return "ldap"; + } + + if claims.contains_key("sub") { + return "openid"; + } + + "builtin" +} + +fn ldap_specific_info(claims: Option<&HashMap>) -> LDAPSpecificAccessKeyInfo { + let username = claims + .and_then(|claims| { + claims + .get("ldap:user") + .or_else(|| claims.get("ldap:username")) + .and_then(|value| value.as_str()) + }) + .map(ToOwned::to_owned); + + LDAPSpecificAccessKeyInfo { username } +} + +fn openid_specific_info(claims: Option<&HashMap>) -> OpenIDSpecificAccessKeyInfo { + let user_id = claims + .and_then(|claims| claims.get("sub")) + .and_then(|value| value.as_str()) + .map(ToOwned::to_owned); + let display_name = claims + .and_then(|claims| claims.get("name")) + .and_then(|value| value.as_str()) + .map(ToOwned::to_owned); + + OpenIDSpecificAccessKeyInfo { + config_name: None, + user_id: user_id.clone(), + user_id_claim: user_id.as_ref().map(|_| "sub".to_string()), + display_name: display_name.clone(), + display_name_claim: display_name.as_ref().map(|_| "name".to_string()), + } +} + pub struct UpdateServiceAccount {} #[async_trait::async_trait] impl Operation for UpdateServiceAccount { @@ -271,6 +452,10 @@ impl Operation for UpdateServiceAccount { let access_key = query.access_key; + let Some(input_cred) = req.credentials else { + return Err(s3_error!(InvalidRequest, "get cred failed")); + }; + let Ok(iam_store) = rustfs_iam::get() else { return Err(s3_error!(InvalidRequest, "iam not init")); }; @@ -280,17 +465,9 @@ impl Operation for UpdateServiceAccount { // s3_error!(InternalError, "get service account failed") // })?; - let mut input = req.input; - let body = match input.store_all_limited(MAX_ADMIN_REQUEST_BODY_SIZE).await { - Ok(b) => b, - Err(e) => { - warn!("get body failed, e: {:?}", e); - return Err(s3_error!( - InvalidRequest, - "service account configuration body too large or failed to read" - )); - } - }; + let body = + read_compatible_admin_body(req.input, MAX_ADMIN_REQUEST_BODY_SIZE, req.uri.path(), input_cred.secret_key.expose()) + .await?; let update_req: UpdateServiceAccountReq = serde_json::from_slice(&body[..]).map_err(|e| s3_error!(InvalidRequest, "unmarshal body failed, e: {:?}", e))?; @@ -299,10 +476,6 @@ impl Operation for UpdateServiceAccount { .validate() .map_err(|e| S3Error::with_message(InvalidRequest, e.to_string()))?; - let Some(input_cred) = req.credentials else { - return Err(s3_error!(InvalidRequest, "get cred failed")); - }; - let (cred, owner) = check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?; @@ -329,22 +502,7 @@ impl Operation for UpdateServiceAccount { return Err(s3_error!(AccessDenied, "access denied")); } - let sp = { - if let Some(policy) = update_req.new_policy { - let sp = Policy::parse_config(policy.as_bytes()).map_err(|e| { - debug!("parse policy failed, e: {:?}", e); - s3_error!(InvalidArgument, "parse policy failed") - })?; - - if sp.version.is_empty() && sp.statements.is_empty() { - None - } else { - Some(sp) - } - } else { - None - } - }; + let sp = parse_update_service_account_policy(update_req.new_policy)?; let opts = UpdateServiceAccountOpts { secret_key: update_req.new_secret_key, @@ -355,15 +513,15 @@ impl Operation for UpdateServiceAccount { session_policy: sp, }; - let _ = iam_store.update_service_account(&access_key, opts).await.map_err(|e| { - debug!("update service account failed, e: {:?}", e); - s3_error!(InternalError, "update service account failed") - })?; + let _ = iam_store + .update_service_account(&access_key, opts) + .await + .map_err(|e| map_service_account_lookup_error(e, "update service account failed"))?; let mut header = HeaderMap::new(); header.insert(CONTENT_TYPE, "application/json".parse().unwrap()); header.insert(CONTENT_LENGTH, "0".parse().unwrap()); - Ok(S3Response::with_headers((StatusCode::OK, Body::empty()), header)) + Ok(S3Response::with_headers((StatusCode::NO_CONTENT, Body::empty()), header)) } } @@ -393,10 +551,10 @@ impl Operation for InfoServiceAccount { return Err(s3_error!(InvalidRequest, "iam not init")); }; - let (svc_account, session_policy) = iam_store.get_service_account(&access_key).await.map_err(|e| { - debug!("get service account failed, e: {:?}", e); - s3_error!(InternalError, "get service account failed") - })?; + let (svc_account, session_policy) = iam_store + .get_service_account(&access_key) + .await + .map_err(|e| map_service_account_lookup_error(e, "get service account failed"))?; let Some(input_cred) = req.credentials else { return Err(s3_error!(InvalidRequest, "get cred failed")); @@ -424,64 +582,208 @@ impl Operation for InfoServiceAccount { deny_only: false, }) .await + && request_user_name(&cred) != svc_account.parent_user { - let user = if cred.parent_user.is_empty() { - &cred.access_key + return Err(s3_error!(AccessDenied, "access denied")); + } + + let resp = build_info_service_account_resp(&iam_store, &svc_account, session_policy).await?; + + let body = serde_json::to_vec(&resp).map_err(|e| s3_error!(InternalError, "marshal body failed, e: {:?}", e))?; + let (body, content_type) = encode_compatible_admin_payload(req.uri.path(), &cred.secret_key, body)?; + + let mut header = HeaderMap::new(); + header.insert(CONTENT_TYPE, content_type.parse().unwrap()); + + Ok(S3Response::with_headers((StatusCode::OK, Body::from(body)), header)) + } +} + +pub struct TemporaryAccountInfo {} +#[async_trait::async_trait] +impl Operation for TemporaryAccountInfo { + async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { + warn!("handle TemporaryAccountInfo"); + + let query = { + if let Some(query) = req.uri.query() { + let input: AccessKeyQuery = + from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "get body failed1"))?; + input } else { - &cred.parent_user + AccessKeyQuery::default() + } + }; + + if query.access_key.is_empty() { + return Err(s3_error!(InvalidArgument, "access key is empty")); + } + + let Some(input_cred) = req.credentials else { + return Err(s3_error!(InvalidRequest, "get cred failed")); + }; + + let (cred, owner) = + check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?; + + let Ok(iam_store) = rustfs_iam::get() else { + return Err(s3_error!(InvalidRequest, "iam not init")); + }; + + if !iam_store + .is_allowed(&Args { + account: &cred.access_key, + groups: &cred.groups, + action: Action::AdminAction(AdminAction::ListTemporaryAccountsAdminAction), + bucket: "", + conditions: &get_condition_values( + &req.headers, + &cred, + None, + None, + req.extensions.get::>().and_then(|opt| opt.map(|a| a.0)), + ), + is_owner: owner, + object: "", + claims: cred.claims.as_ref().unwrap_or(&HashMap::new()), + deny_only: false, + }) + .await + { + return Err(s3_error!(AccessDenied, "access denied")); + } + + let (temp_account, session_policy) = iam_store + .get_temporary_account(&query.access_key) + .await + .map_err(|e| map_temp_account_lookup_error(e, "get temporary account failed"))?; + + let resp: TemporaryAccountInfoResp = build_info_service_account_resp(&iam_store, &temp_account, session_policy).await?; + let body = serde_json::to_vec(&resp).map_err(|e| s3_error!(InternalError, "marshal body failed, e: {:?}", e))?; + let (body, content_type) = encode_compatible_admin_payload(req.uri.path(), &cred.secret_key, body)?; + + let mut header = HeaderMap::new(); + header.insert(CONTENT_TYPE, content_type.parse().unwrap()); + + Ok(S3Response::with_headers((StatusCode::OK, Body::from(body)), header)) + } +} + +pub struct InfoAccessKey {} +#[async_trait::async_trait] +impl Operation for InfoAccessKey { + async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { + warn!("handle InfoAccessKey"); + + let query = { + if let Some(query) = req.uri.query() { + let input: AccessKeyQuery = + from_bytes(query.as_bytes()).map_err(|_e| s3_error!(InvalidArgument, "get body failed1"))?; + input + } else { + AccessKeyQuery::default() + } + }; + + let Some(input_cred) = req.credentials else { + return Err(s3_error!(InvalidRequest, "get cred failed")); + }; + + let (cred, owner) = + check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?; + + let access_key = if query.access_key.is_empty() { + cred.access_key.clone() + } else { + query.access_key + }; + + let Ok(iam_store) = rustfs_iam::get() else { + return Err(s3_error!(InvalidRequest, "iam not init")); + }; + + let target_cred = iam_store.get_user(&access_key).await.map(|identity| identity.credentials); + + if !iam_store + .is_allowed(&Args { + account: &cred.access_key, + groups: &cred.groups, + action: Action::AdminAction(AdminAction::ListServiceAccountsAdminAction), + bucket: "", + conditions: &get_condition_values( + &req.headers, + &cred, + None, + None, + req.extensions.get::>().and_then(|opt| opt.map(|a| a.0)), + ), + is_owner: owner, + object: "", + claims: cred.claims.as_ref().unwrap_or(&HashMap::new()), + deny_only: false, + }) + .await + { + let Some(target_cred) = target_cred.as_ref() else { + return Err(s3_error!(AccessDenied, "access denied")); }; - if user != &svc_account.parent_user { + + if request_user_name(&cred) != target_cred.parent_user { return Err(s3_error!(AccessDenied, "access denied")); } } - let implied_policy = if let Some(policy) = session_policy.as_ref() { - policy.version.is_empty() && policy.statements.is_empty() + let Some(target_cred) = target_cred else { + return Err(s3_error!(InvalidRequest, "access key not exist")); + }; + + let (user_type, session_policy) = if target_cred.is_temp() { + let (_, session_policy) = iam_store.get_temporary_account(&access_key).await.map_err(|e| { + debug!("get temporary account failed, e: {:?}", e); + if is_err_no_such_temp_account(&e) { + s3_error!(InvalidRequest, "access key not exist") + } else { + s3_error!(InternalError, "get temporary account failed") + } + })?; + ("STS".to_string(), session_policy) + } else if target_cred.is_service_account() { + let (_, session_policy) = iam_store.get_service_account(&access_key).await.map_err(|e| { + debug!("get service account failed, e: {:?}", e); + if is_err_no_such_service_account(&e) { + s3_error!(InvalidRequest, "access key not exist") + } else { + s3_error!(InternalError, "get service account failed") + } + })?; + ("Service Account".to_string(), session_policy) } else { - true + return Err(s3_error!(InvalidRequest, "access key not exist")); }; - let svc_account_policy = { - if !implied_policy { - session_policy + let user_provider = guess_user_provider(&target_cred).to_string(); + let resp = InfoAccessKeyResp { + access_key, + info: build_info_service_account_resp(&iam_store, &target_cred, session_policy).await?, + user_type, + user_provider: user_provider.clone(), + ldap_specific_info: if user_provider == "ldap" { + ldap_specific_info(target_cred.claims.as_ref()) } else { - let policies = iam_store - .policy_db_get(&svc_account.parent_user, &svc_account.groups) - .await - .map_err(|e| { - debug!("get service account policy failed, e: {:?}", e); - s3_error!(InternalError, "get service account policy failed") - })?; - - Some(iam_store.get_combined_policy(&policies).await) - } - }; - - let policy = { - if let Some(policy) = svc_account_policy { - Some(serde_json::to_string(&policy).map_err(|e| { - debug!("marshal policy failed, e: {:?}", e); - s3_error!(InternalError, "marshal policy failed") - })?) + LDAPSpecificAccessKeyInfo::default() + }, + open_id_specific_info: if user_provider == "openid" { + openid_specific_info(target_cred.claims.as_ref()) } else { - None - } - }; - - let resp = InfoServiceAccountResp { - parent_user: svc_account.parent_user, - account_status: svc_account.status, - implied_policy, - name: svc_account.name, - description: svc_account.description, - expiration: svc_account.expiration, - policy, + OpenIDSpecificAccessKeyInfo::default() + }, }; let body = serde_json::to_vec(&resp).map_err(|e| s3_error!(InternalError, "marshal body failed, e: {:?}", e))?; + let (body, content_type) = encode_compatible_admin_payload(req.uri.path(), &cred.secret_key, body)?; let mut header = HeaderMap::new(); - header.insert(CONTENT_TYPE, "application/json".parse().unwrap()); + header.insert(CONTENT_TYPE, content_type.parse().unwrap()); Ok(S3Response::with_headers((StatusCode::OK, Body::from(body)), header)) } @@ -583,15 +885,239 @@ impl Operation for ListServiceAccount { access_key: sa.access_key, name: sa.name, description: sa.description, - expiration: sa.expiration, + expiration: expiration_for_admin_path(req.uri.path(), sa.expiration), }) .collect(); let data = serde_json::to_vec(&ListServiceAccountsResp { accounts }) .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("marshal users err {e}")))?; + let (data, content_type) = encode_compatible_admin_payload(req.uri.path(), &cred.secret_key, data)?; let mut header = HeaderMap::new(); - header.insert(CONTENT_TYPE, "application/json".parse().unwrap()); + header.insert(CONTENT_TYPE, content_type.parse().unwrap()); + + Ok(S3Response::with_headers((StatusCode::OK, Body::from(data)), header)) + } +} + +#[derive(Debug, Deserialize)] +struct ListAccessKeysQuery { + users: Vec, + all: bool, + list_type: String, +} + +fn default_access_key_list_type() -> String { + ACCESS_KEY_LIST_ALL.to_string() +} + +impl Default for ListAccessKeysQuery { + fn default() -> Self { + Self { + users: Vec::new(), + all: false, + list_type: default_access_key_list_type(), + } + } +} + +fn parse_bool_param(value: &str) -> bool { + matches!(value, "true" | "1" | "on" | "yes") +} + +fn parse_list_access_keys_query(query: Option<&str>) -> ListAccessKeysQuery { + let mut parsed = ListAccessKeysQuery::default(); + + let Some(query) = query else { + return parsed; + }; + + for (key, value) in form_urlencoded::parse(query.as_bytes()) { + match key.as_ref() { + "users" => parsed.users.push(value.into_owned()), + "all" => parsed.all = parse_bool_param(value.as_ref()), + "listType" => parsed.list_type = value.into_owned(), + _ => {} + } + } + + parsed +} + +pub struct ListAccessKeysBulk {} +#[async_trait::async_trait] +impl Operation for ListAccessKeysBulk { + async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { + warn!("handle ListAccessKeysBulk"); + + let query = { parse_list_access_keys_query(req.uri.query()) }; + + if query.all && !query.users.is_empty() { + return Err(s3_error!(InvalidRequest, "either specify users or all, not both")); + } + + let Some(input_cred) = req.credentials else { + return Err(s3_error!(InvalidRequest, "get cred failed")); + }; + + let (cred, owner) = + check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?; + + let Ok(iam_store) = rustfs_iam::get() else { + return Err(s3_error!(InvalidRequest, "iam not init")); + }; + + let mut requested_users = query.users; + let mut self_only = !query.all && requested_users.is_empty(); + if !query.all && requested_users.len() == 1 { + let requested_user = &requested_users[0]; + if requested_user == &cred.access_key || requested_user == &cred.parent_user { + self_only = true; + } + } + + if query.all + && !iam_store + .is_allowed(&Args { + account: &cred.access_key, + groups: &cred.groups, + action: Action::AdminAction(AdminAction::ListUsersAdminAction), + bucket: "", + conditions: &get_condition_values( + &req.headers, + &cred, + None, + None, + req.extensions.get::>().and_then(|opt| opt.map(|a| a.0)), + ), + is_owner: owner, + object: "", + claims: cred.claims.as_ref().unwrap_or(&HashMap::new()), + deny_only: false, + }) + .await + { + return Err(s3_error!(AccessDenied, "access denied")); + } + + if !iam_store + .is_allowed(&Args { + account: &cred.access_key, + groups: &cred.groups, + action: Action::AdminAction(AdminAction::ListServiceAccountsAdminAction), + bucket: "", + conditions: &get_condition_values( + &req.headers, + &cred, + None, + None, + req.extensions.get::>().and_then(|opt| opt.map(|a| a.0)), + ), + is_owner: owner, + object: "", + claims: cred.claims.as_ref().unwrap_or(&HashMap::new()), + deny_only: self_only, + }) + .await + { + return Err(s3_error!(AccessDenied, "access denied")); + } + + if self_only && requested_users.is_empty() { + requested_users.push(request_user_name(&cred).to_string()); + } + + let checked_user_list = if query.all { + let mut users = iam_store + .list_users() + .await + .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("list users err {e}")))? + .into_keys() + .collect::>(); + + if let Some(sys_cred) = get_global_action_cred() { + users.push(sys_cred.access_key); + } + users + } else { + let mut checked = Vec::new(); + for user in requested_users { + if iam_store.get_user(&user).await.is_some() { + checked.push(user); + } + } + checked + }; + + let (list_sts_keys, list_service_accounts) = match query.list_type.as_str() { + ACCESS_KEY_LIST_USERS_ONLY => (false, false), + ACCESS_KEY_LIST_STS_ONLY => (true, false), + ACCESS_KEY_LIST_SVCACC_ONLY => (false, true), + ACCESS_KEY_LIST_ALL => (true, true), + _ => return Err(s3_error!(InvalidRequest, "invalid list type")), + }; + + let mut access_key_map = HashMap::new(); + for user in checked_user_list { + let mut access_keys = ListAccessKeysResp::default(); + + if list_sts_keys { + let sts_keys = iam_store.list_sts_accounts(&user).await.map_err(|e| { + debug!("list sts account failed: {e:?}"); + s3_error!(InternalError, "list sts account failed") + })?; + + access_keys.sts_keys = sts_keys + .into_iter() + .map(|sts| ServiceAccountInfo { + parent_user: String::new(), + account_status: String::new(), + implied_policy: false, + access_key: sts.access_key, + name: sts.name, + description: sts.description, + expiration: expiration_for_admin_path(req.uri.path(), sts.expiration), + }) + .collect(); + + if !list_service_accounts && access_keys.sts_keys.is_empty() { + continue; + } + } + + if list_service_accounts { + let service_accounts = iam_store.list_service_accounts(&user).await.map_err(|e| { + debug!("list service account failed: {e:?}"); + s3_error!(InternalError, "list service account failed") + })?; + + access_keys.service_accounts = service_accounts + .into_iter() + .map(|svc| ServiceAccountInfo { + parent_user: String::new(), + account_status: String::new(), + implied_policy: false, + access_key: svc.access_key, + name: svc.name, + description: svc.description, + expiration: expiration_for_admin_path(req.uri.path(), svc.expiration), + }) + .collect(); + + if !list_sts_keys && access_keys.service_accounts.is_empty() { + continue; + } + } + + access_key_map.insert(user, access_keys); + } + + let data = serde_json::to_vec(&access_key_map) + .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("marshal access keys err {e}")))?; + let (data, content_type) = encode_compatible_admin_payload(req.uri.path(), &cred.secret_key, data)?; + + let mut header = HeaderMap::new(); + header.insert(CONTENT_TYPE, content_type.parse().unwrap()); Ok(S3Response::with_headers((StatusCode::OK, Body::from(data)), header)) } @@ -682,6 +1208,158 @@ impl Operation for DeleteServiceAccount { let mut header = HeaderMap::new(); header.insert(CONTENT_TYPE, "application/json".parse().unwrap()); header.insert(CONTENT_LENGTH, "0".parse().unwrap()); - Ok(S3Response::with_headers((StatusCode::OK, Body::empty()), header)) + Ok(S3Response::with_headers( + (delete_service_account_success_status(req.uri.path()), Body::empty()), + header, + )) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use rustfs_credentials::IAM_POLICY_CLAIM_NAME_SA; + use serde_json::json; + use serde_urlencoded::from_bytes; + + #[test] + fn access_key_query_supports_external_alias() { + let query: AccessKeyQuery = from_bytes(b"access-key=test-access-key").expect("parse query"); + assert_eq!(query.access_key, "test-access-key"); + } + + #[test] + fn guess_user_provider_detects_builtin_accounts() { + let credentials = StoredCredentials { + access_key: "builtin-user".to_string(), + secret_key: "secret-key".to_string(), + ..Default::default() + }; + + assert_eq!(guess_user_provider(&credentials), "builtin"); + } + + #[test] + fn guess_user_provider_detects_ldap_accounts() { + let credentials = StoredCredentials { + access_key: "svc".to_string(), + secret_key: "secret-key".to_string(), + parent_user: "parent".to_string(), + claims: Some(HashMap::from([ + (IAM_POLICY_CLAIM_NAME_SA.to_string(), json!("embedded")), + ("ldap:user".to_string(), json!("uid=rustfs,ou=people,dc=example,dc=com")), + ])), + ..Default::default() + }; + + assert_eq!(guess_user_provider(&credentials), "ldap"); + } + + #[test] + fn guess_user_provider_detects_openid_accounts() { + let credentials = StoredCredentials { + access_key: "sts".to_string(), + secret_key: "secret-key".to_string(), + session_token: "session-token".to_string(), + parent_user: "parent".to_string(), + claims: Some(HashMap::from([("sub".to_string(), json!("subject-123"))])), + ..Default::default() + }; + + assert_eq!(guess_user_provider(&credentials), "openid"); + } + + #[test] + fn provider_specific_info_uses_known_claims() { + let claims = HashMap::from([ + ("ldap:user".to_string(), json!("uid=rustfs,ou=people,dc=example,dc=com")), + ("sub".to_string(), json!("subject-123")), + ("name".to_string(), json!("RustFS User")), + ]); + + let ldap_info = ldap_specific_info(Some(&claims)); + let openid_info = openid_specific_info(Some(&claims)); + + assert_eq!(ldap_info.username.as_deref(), Some("uid=rustfs,ou=people,dc=example,dc=com")); + assert_eq!(openid_info.user_id.as_deref(), Some("subject-123")); + assert_eq!(openid_info.user_id_claim.as_deref(), Some("sub")); + assert_eq!(openid_info.display_name.as_deref(), Some("RustFS User")); + assert_eq!(openid_info.display_name_claim.as_deref(), Some("name")); + } + + #[test] + fn list_access_keys_query_parses_external_parameters() { + let query = parse_list_access_keys_query(Some("users=alice&users=bob&all=true&listType=svcacc-only")); + + assert_eq!(query.users, vec!["alice".to_string(), "bob".to_string()]); + assert!(query.all); + assert_eq!(query.list_type, ACCESS_KEY_LIST_SVCACC_ONLY); + } + + #[test] + fn list_access_keys_query_defaults_to_all_list_type() { + let query = ListAccessKeysQuery::default(); + assert_eq!(query.list_type, ACCESS_KEY_LIST_ALL); + } + + #[test] + fn delete_service_account_uses_external_success_status() { + assert_eq!( + delete_service_account_success_status("/minio/admin/v3/delete-service-account"), + StatusCode::NO_CONTENT + ); + } + + #[test] + fn delete_service_account_keeps_rustfs_console_success_status() { + assert_eq!( + delete_service_account_success_status("/rustfs/admin/v3/delete-service-account"), + StatusCode::OK + ); + } + + #[test] + fn expiration_for_external_admin_path_uses_sentinel() { + assert_eq!( + expiration_for_admin_path("/minio/admin/v3/list-service-accounts", None), + Some(OffsetDateTime::UNIX_EPOCH) + ); + } + + #[test] + fn expiration_for_rustfs_admin_path_preserves_none() { + assert_eq!(expiration_for_admin_path("/rustfs/admin/v3/list-service-accounts", None), None); + } + + #[test] + fn map_service_account_lookup_error_reports_missing_accounts_explicitly() { + let err = map_service_account_lookup_error( + rustfs_iam::error::Error::NoSuchServiceAccount("missing".to_string()), + "get service account failed", + ); + + assert_eq!(*err.code(), S3ErrorCode::InvalidRequest); + assert_eq!(err.message(), Some("service account not exist")); + } + + #[test] + fn map_temp_account_lookup_error_reports_missing_access_keys_explicitly() { + let err = map_temp_account_lookup_error( + rustfs_iam::error::Error::NoSuchTempAccount("missing".to_string()), + "get temporary account failed", + ); + + assert_eq!(*err.code(), S3ErrorCode::InvalidRequest); + assert_eq!(err.message(), Some("access key not exist")); + } + + #[test] + fn parse_update_service_account_policy_keeps_explicit_empty_policy() { + let policy = parse_update_service_account_policy(Some(json!({}))).expect("empty policy should parse"); + + assert!(policy.is_some()); + let policy = policy.unwrap(); + assert!(policy.version.is_empty()); + assert!(policy.statements.is_empty()); } } diff --git a/rustfs/src/admin/handlers/user.rs b/rustfs/src/admin/handlers/user.rs index a9b5183fc..d05b8a3ec 100644 --- a/rustfs/src/admin/handlers/user.rs +++ b/rustfs/src/admin/handlers/user.rs @@ -17,7 +17,7 @@ use crate::{ admin::{ auth::validate_admin_request, router::{AdminOperation, Operation, S3Router}, - utils::has_space_be, + utils::{encode_compatible_admin_payload, has_space_be, read_compatible_admin_body}, }, auth::{check_key_valid, constant_time_eq, get_session_token}, server::RemoteAddr, @@ -28,7 +28,7 @@ use rustfs_config::{MAX_ADMIN_REQUEST_BODY_SIZE, MAX_IAM_IMPORT_SIZE}; use rustfs_credentials::{Credentials, get_global_action_cred}; use rustfs_iam::{ store::{GroupInfo, MappedPolicy, UserType}, - sys::NewServiceAccountOpts, + sys::{NewServiceAccountOpts, UpdateServiceAccountOpts}, }; use rustfs_madmin::{ AccountStatus, AddOrUpdateUserReq, IAMEntities, IAMErrEntities, IAMErrEntity, IAMErrPolicyEntity, @@ -50,7 +50,7 @@ use zip::{ZipArchive, ZipWriter, result::ZipError, write::SimpleFileOptions}; #[derive(Debug, Deserialize, Default)] pub struct AddUserQuery { - #[serde(rename = "accessKey")] + #[serde(rename = "accessKey", alias = "access-key")] pub access_key: Option, pub status: Option, } @@ -73,6 +73,30 @@ fn should_check_deny_only(target_access_key: &str, requester: &Credentials) -> b && !requester.is_service_account() } +fn should_reject_group_import_name(group_name: &str, group_lookup: &rustfs_iam::error::Error) -> bool { + has_space_be(group_name) || !matches!(group_lookup, rustfs_iam::error::Error::NoSuchGroup(_)) +} + +fn should_restore_group_as_disabled(status: &str) -> bool { + status.eq_ignore_ascii_case(rustfs_iam::sys::STATUS_DISABLED) +} + +fn imported_service_account_status(status: &str) -> Option { + if status.eq_ignore_ascii_case(rustfs_policy::auth::ACCOUNT_OFF) + || status.eq_ignore_ascii_case(rustfs_madmin::AccountStatus::Disabled.as_ref()) + { + return Some(rustfs_policy::auth::ACCOUNT_OFF.to_string()); + } + + if status.eq_ignore_ascii_case(rustfs_policy::auth::ACCOUNT_ON) + || status.eq_ignore_ascii_case(rustfs_madmin::AccountStatus::Enabled.as_ref()) + { + return Some(rustfs_policy::auth::ACCOUNT_ON.to_string()); + } + + None +} + pub struct AddUser {} #[async_trait::async_trait] impl Operation for AddUser { @@ -100,14 +124,7 @@ impl Operation for AddUser { return Err(s3_error!(InvalidArgument, "access key is empty")); } - let mut input = req.input; - let body = match input.store_all_limited(MAX_ADMIN_REQUEST_BODY_SIZE).await { - Ok(b) => b, - Err(e) => { - warn!("get body failed, e: {:?}", e); - return Err(s3_error!(InvalidRequest, "get body failed")); - } - }; + let body = read_compatible_admin_body(req.input, MAX_ADMIN_REQUEST_BODY_SIZE, req.uri.path(), &cred.secret_key).await?; // let body_bytes = decrypt_data(input_cred.secret_key.expose().as_bytes(), &body) // .map_err(|e| S3Error::with_message(S3ErrorCode::InvalidArgument, format!("decrypt_data err {}", e)))?; @@ -282,9 +299,10 @@ impl Operation for ListUsers { let data = serde_json::to_vec(&users) .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("marshal users err {e}")))?; + let (data, content_type) = encode_compatible_admin_payload(req.uri.path(), &cred.secret_key, data)?; let mut header = HeaderMap::new(); - header.insert(CONTENT_TYPE, "application/json".parse().unwrap()); + header.insert(CONTENT_TYPE, content_type.parse().unwrap()); Ok(S3Response::with_headers((StatusCode::OK, Body::from(data)), header)) } @@ -435,6 +453,7 @@ const ALL_SVC_ACCTS_FILE: &str = "svcaccts.json"; const USER_POLICY_MAPPINGS_FILE: &str = "user_mappings.json"; const GROUP_POLICY_MAPPINGS_FILE: &str = "group_mappings.json"; const STS_USER_POLICY_MAPPINGS_FILE: &str = "stsuser_mappings.json"; +const GROUP_POLICY_MAPPING_USER_TYPE: UserType = UserType::Reg; const IAM_ASSETS_DIR: &str = "iam-assets"; @@ -619,7 +638,7 @@ impl Operation for ExportIam { GROUP_POLICY_MAPPINGS_FILE => { let mut group_policy_mappings = HashMap::new(); iam_store - .load_mapped_policies(UserType::Reg, true, &mut group_policy_mappings) + .load_mapped_policies(GROUP_POLICY_MAPPING_USER_TYPE, true, &mut group_policy_mappings) .await .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, e.to_string()))?; @@ -805,7 +824,7 @@ impl Operation for ImportIam { .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, e.to_string()))?; for (group_name, group_info) in groups { if let Err(e) = iam_store.get_group_description(&group_name).await - && (matches!(e, rustfs_iam::error::Error::NoSuchGroup(_)) || has_space_be(&group_name)) + && should_reject_group_import_name(&group_name, &e) { return Err(s3_error!(InvalidArgument, "group not found or has space be")); } @@ -816,6 +835,14 @@ impl Operation for ImportIam { error: e.to_string(), }); } else { + if should_restore_group_as_disabled(&group_info.status) { + iam_store.set_group_status(&group_name, false).await.map_err(|e| { + S3Error::with_message( + S3ErrorCode::InternalError, + format!("set group status failed, name: {group_name}, err: {e}"), + ) + })?; + } added.groups.push(group_name.clone()); } } @@ -893,6 +920,29 @@ impl Operation for ImportIam { error: e.to_string(), }); } else { + if let Some(status) = imported_service_account_status(&req.status) + && status == rustfs_policy::auth::ACCOUNT_OFF + { + iam_store + .update_service_account( + &ak, + UpdateServiceAccountOpts { + session_policy: None, + secret_key: None, + name: None, + description: None, + expiration: None, + status: Some(status), + }, + ) + .await + .map_err(|e| { + S3Error::with_message( + S3ErrorCode::InternalError, + format!("update service account status failed, name: {ak}, err: {e}"), + ) + })?; + } added.service_accounts.push(ak.clone()); } } @@ -973,7 +1023,7 @@ impl Operation for ImportIam { } if let Err(e) = iam_store - .policy_db_set(&group_name, UserType::None, true, &policies.policies) + .policy_db_set(&group_name, GROUP_POLICY_MAPPING_USER_TYPE, true, &policies.policies) .await { failed.group_policies.push(IAMErrPolicyEntity { @@ -1064,8 +1114,12 @@ impl Operation for ImportIam { #[cfg(test)] mod tests { - use super::should_check_deny_only; + use super::{ + GROUP_POLICY_MAPPING_USER_TYPE, imported_service_account_status, should_check_deny_only, should_reject_group_import_name, + should_restore_group_as_disabled, + }; use rustfs_credentials::{Credentials, IAM_POLICY_CLAIM_NAME_SA}; + use rustfs_iam::error::Error as IamError; use serde_json::Value; use std::collections::HashMap; @@ -1119,4 +1173,40 @@ mod tests { }; assert!(!should_check_deny_only("alice", &cred)); } + + #[test] + fn test_group_import_allows_missing_group_without_spaces() { + assert!(!should_reject_group_import_name( + "new-group", + &IamError::NoSuchGroup("new-group".to_string()) + )); + } + + #[test] + fn test_group_import_rejects_group_names_with_spaces() { + assert!(should_reject_group_import_name( + " bad-group", + &IamError::NoSuchGroup(" bad-group".to_string()) + )); + } + + #[test] + fn test_group_import_restores_disabled_status_only_when_needed() { + assert!(should_restore_group_as_disabled("disabled")); + assert!(!should_restore_group_as_disabled("enabled")); + } + + #[test] + fn test_imported_service_account_status_maps_on_and_off() { + assert_eq!(imported_service_account_status("off").as_deref(), Some("off")); + assert_eq!(imported_service_account_status("on").as_deref(), Some("on")); + assert_eq!(imported_service_account_status("disabled").as_deref(), Some("off")); + assert_eq!(imported_service_account_status("enabled").as_deref(), Some("on")); + assert!(imported_service_account_status("unknown").is_none()); + } + + #[test] + fn test_group_policy_mappings_use_regular_user_type() { + assert_eq!(GROUP_POLICY_MAPPING_USER_TYPE, rustfs_iam::store::UserType::Reg); + } } diff --git a/rustfs/src/admin/route_registration_test.rs b/rustfs/src/admin/route_registration_test.rs index 488d39e9c..07bbf9b5b 100644 --- a/rustfs/src/admin/route_registration_test.rs +++ b/rustfs/src/admin/route_registration_test.rs @@ -17,13 +17,17 @@ use crate::admin::{ router::{AdminOperation, S3Router}, rpc, }; -use crate::server::{ADMIN_PREFIX, HEALTH_PREFIX, HEALTH_READY_PATH, PROFILE_CPU_PATH, PROFILE_MEMORY_PATH}; +use crate::server::{ADMIN_PREFIX, HEALTH_PREFIX, HEALTH_READY_PATH, MINIO_ADMIN_PREFIX, PROFILE_CPU_PATH, PROFILE_MEMORY_PATH}; use hyper::Method; fn admin_path(path: &str) -> String { format!("{}{}", ADMIN_PREFIX, path) } +fn compat_admin_alias_path(path: &str) -> String { + format!("{}{}", MINIO_ADMIN_PREFIX, path) +} + fn assert_route(router: &S3Router, method: Method, path: &str) { assert!( router.contains_route(method.clone(), path), @@ -69,9 +73,17 @@ fn test_register_routes_cover_representative_admin_paths() { assert_route(&router, Method::DELETE, &admin_path("/v3/group/test-group")); assert_route(&router, Method::PUT, &admin_path("/v3/update-group-members")); assert_route(&router, Method::PUT, &admin_path("/v3/add-service-accounts")); + assert_route(&router, Method::PUT, &admin_path("/v3/add-service-account")); + assert_route(&router, Method::GET, &admin_path("/v3/temporary-account-info")); + assert_route(&router, Method::GET, &admin_path("/v3/info-access-key")); + assert_route(&router, Method::GET, &admin_path("/v3/list-access-keys-bulk")); assert_route(&router, Method::GET, &admin_path("/v3/export-iam")); assert_route(&router, Method::PUT, &admin_path("/v3/import-iam")); assert_route(&router, Method::GET, &admin_path("/v3/list-canned-policies")); + assert_route(&router, Method::PUT, &admin_path("/v3/set-policy")); + assert_route(&router, Method::POST, &admin_path("/v3/idp/builtin/policy/attach")); + assert_route(&router, Method::POST, &admin_path("/v3/idp/builtin/policy/detach")); + assert_route(&router, Method::GET, &admin_path("/v3/idp/builtin/policy-entities")); assert_route(&router, Method::GET, &admin_path("/v3/target/list")); assert_route(&router, Method::GET, &admin_path("/v3/accountinfo")); @@ -89,6 +101,8 @@ fn test_register_routes_cover_representative_admin_paths() { assert_route(&router, Method::GET, &admin_path("/v3/tier")); assert_route(&router, Method::POST, &admin_path("/v3/tier/clear")); + assert_route(&router, Method::PUT, &admin_path("/v3/set-bucket-quota")); + assert_route(&router, Method::GET, &admin_path("/v3/get-bucket-quota")); assert_route(&router, Method::PUT, &admin_path("/v3/quota/test-bucket")); assert_route(&router, Method::GET, &admin_path("/v3/quota-stats/test-bucket")); @@ -107,6 +121,44 @@ fn test_register_routes_cover_representative_admin_paths() { assert_route(&router, Method::HEAD, "/rustfs/rpc/read_file_stream"); } +#[test] +fn test_admin_alias_paths_match_existing_admin_routes() { + let mut router: S3Router = S3Router::new(false); + + health::register_health_route(&mut router).expect("register health route"); + sts::register_admin_auth_route(&mut router).expect("register sts route"); + user::register_user_route(&mut router).expect("register user route"); + system::register_system_route(&mut router).expect("register system route"); + pools::register_pool_route(&mut router).expect("register pool route"); + rebalance::register_rebalance_route(&mut router).expect("register rebalance route"); + quota::register_quota_route(&mut router).expect("register quota route"); + + for (method, path) in [ + (Method::GET, compat_admin_alias_path("/v3/is-admin")), + (Method::GET, compat_admin_alias_path("/v3/info")), + (Method::GET, compat_admin_alias_path("/v3/storageinfo")), + (Method::GET, compat_admin_alias_path("/v3/pools/list")), + (Method::PUT, compat_admin_alias_path("/v3/add-service-account")), + (Method::GET, compat_admin_alias_path("/v3/temporary-account-info")), + (Method::GET, compat_admin_alias_path("/v3/info-access-key")), + (Method::GET, compat_admin_alias_path("/v3/list-access-keys-bulk")), + (Method::PUT, compat_admin_alias_path("/v3/set-policy")), + (Method::PUT, compat_admin_alias_path("/v3/set-bucket-quota")), + (Method::GET, compat_admin_alias_path("/v3/get-bucket-quota")), + (Method::POST, compat_admin_alias_path("/v3/idp/builtin/policy/attach")), + (Method::POST, compat_admin_alias_path("/v3/idp/builtin/policy/detach")), + (Method::GET, compat_admin_alias_path("/v3/idp/builtin/policy-entities")), + (Method::POST, compat_admin_alias_path("/v3/rebalance/start")), + ] { + assert!( + router.contains_compatible_route(method.clone(), &path), + "expected MinIO admin alias path to match: {} {}", + method.as_str(), + path + ); + } +} + #[test] fn test_phase5_admin_info_and_rpc_read_file_contract() { let system_src = include_str!("handlers/system.rs"); diff --git a/rustfs/src/admin/router.rs b/rustfs/src/admin/router.rs index 270ddc171..abf61af7e 100644 --- a/rustfs/src/admin/router.rs +++ b/rustfs/src/admin/router.rs @@ -14,7 +14,9 @@ use crate::admin::console::{is_console_path, make_console_server}; use crate::admin::handlers::oidc::is_oidc_path; -use crate::server::{ADMIN_PREFIX, HEALTH_PREFIX, HEALTH_READY_PATH, PROFILE_CPU_PATH, PROFILE_MEMORY_PATH, RPC_PREFIX}; +use crate::server::{ + ADMIN_PREFIX, HEALTH_PREFIX, HEALTH_READY_PATH, MINIO_ADMIN_PREFIX, PROFILE_CPU_PATH, PROFILE_MEMORY_PATH, RPC_PREFIX, +}; use hyper::HeaderMap; use hyper::Method; use hyper::StatusCode; @@ -43,6 +45,18 @@ fn is_public_health_path(path: &str) -> bool { path == HEALTH_PREFIX || path == HEALTH_READY_PATH } +fn is_admin_path(path: &str) -> bool { + path.starts_with(ADMIN_PREFIX) || path.starts_with(MINIO_ADMIN_PREFIX) +} + +fn canonicalize_admin_path(path: &str) -> std::borrow::Cow<'_, str> { + if let Some(suffix) = path.strip_prefix(MINIO_ADMIN_PREFIX) { + return std::borrow::Cow::Owned(format!("{ADMIN_PREFIX}{suffix}")); + } + + std::borrow::Cow::Borrowed(path) +} + impl S3Router { pub fn new(console_enabled: bool) -> Self { let router = Router::new(); @@ -81,6 +95,12 @@ impl S3Router { let route = Self::make_route_str(method, path); self.router.at(&route).is_ok() } + + pub(crate) fn contains_compatible_route(&self, method: Method, path: &str) -> bool { + let canonical_path = canonicalize_admin_path(path); + let route = Self::make_route_str(method, canonical_path.as_ref()); + self.router.at(&route).is_ok() + } } impl Default for S3Router { @@ -120,7 +140,7 @@ where return true; } - path.starts_with(ADMIN_PREFIX) || path.starts_with(RPC_PREFIX) || is_console_path(path) + is_admin_path(path) || path.starts_with(RPC_PREFIX) || is_console_path(path) } // check_access before call @@ -207,7 +227,8 @@ where return Err(s3_error!(InternalError, "console is not enabled")); } - let uri = format!("{}|{}", &req.method, req.uri.path()); + let canonical_path = canonicalize_admin_path(req.uri.path()); + let uri = format!("{}|{}", &req.method, canonical_path.as_ref()); if let Ok(mat) = self.router.at(&uri) { let op: &T = mat.value; @@ -238,6 +259,24 @@ impl Operation for AdminOperation { } } +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn canonicalize_admin_path_maps_compat_prefix_to_rustfs_prefix() { + assert_eq!(canonicalize_admin_path("/minio/admin/v3/info").as_ref(), "/rustfs/admin/v3/info"); + assert_eq!(canonicalize_admin_path("/rustfs/admin/v3/info").as_ref(), "/rustfs/admin/v3/info"); + } + + #[test] + fn is_admin_path_accepts_rustfs_and_compat_prefixes() { + assert!(is_admin_path("/rustfs/admin/v3/info")); + assert!(is_admin_path("/minio/admin/v3/info")); + assert!(!is_admin_path("/bucket/object")); + } +} + #[allow(dead_code)] #[derive(Debug, Clone)] pub struct Extra { diff --git a/rustfs/src/admin/utils.rs b/rustfs/src/admin/utils.rs index 80c3ea2ac..ac5768e09 100644 --- a/rustfs/src/admin/utils.rs +++ b/rustfs/src/admin/utils.rs @@ -12,6 +12,90 @@ // See the License for the specific language governing permissions and // limitations under the License. +use crate::server::MINIO_ADMIN_PREFIX; +use rustfs_crypto::{decrypt_data, decrypt_stream_io, encrypt_stream_io}; +use s3s::{Body, S3Result, s3_error}; + pub(crate) fn has_space_be(s: &str) -> bool { s.trim().len() != s.len() } + +pub(crate) fn is_compat_admin_request(path: &str) -> bool { + path.starts_with(MINIO_ADMIN_PREFIX) +} + +pub(crate) async fn read_compatible_admin_body( + mut input: Body, + max_len: usize, + path: &str, + secret_key: &str, +) -> S3Result> { + let body = input + .store_all_limited(max_len) + .await + .map_err(|e| s3_error!(InvalidRequest, "failed to read request body: {}", e))?; + + if is_compat_admin_request(path) { + decrypt_stream_io(secret_key.as_bytes(), body.as_ref()) + .or_else(|_| decrypt_data(secret_key.as_bytes(), body.as_ref())) + .map_err(|e| s3_error!(InvalidRequest, "failed to decrypt MinIO admin payload: {}", e)) + } else { + Ok(body.to_vec()) + } +} + +pub(crate) fn encode_compatible_admin_payload(path: &str, secret_key: &str, data: Vec) -> S3Result<(Vec, &'static str)> { + if is_compat_admin_request(path) { + let encrypted = encrypt_stream_io(secret_key.as_bytes(), &data) + .map_err(|e| s3_error!(InternalError, "failed to encrypt MinIO admin payload: {}", e))?; + Ok((encrypted, "application/octet-stream")) + } else { + Ok((data, "application/json")) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use rustfs_crypto::encrypt_data; + use s3s::Body; + + #[test] + fn detects_compat_admin_paths_only_for_external_prefix() { + assert!(is_compat_admin_request("/minio/admin/v3/list-users")); + assert!(!is_compat_admin_request("/rustfs/admin/v3/list-users")); + } + + #[test] + fn encodes_plain_payload_for_rustfs_admin_paths() { + let payload = b"{\"ok\":true}".to_vec(); + let (encoded, content_type) = + encode_compatible_admin_payload("/rustfs/admin/v3/list-users", "secret", payload.clone()).expect("encode payload"); + + assert_eq!(encoded, payload); + assert_eq!(content_type, "application/json"); + } + + #[test] + fn encodes_compat_payload_with_compatible_encryption() { + let payload = b"{\"ok\":true}".to_vec(); + let (encoded, content_type) = + encode_compatible_admin_payload("/minio/admin/v3/list-users", "secret", payload.clone()).expect("encode payload"); + + assert_ne!(encoded, payload); + assert_eq!(content_type, "application/octet-stream"); + assert_eq!(decrypt_stream_io(b"secret", &encoded).expect("decrypt payload"), payload); + } + + #[tokio::test] + async fn reads_legacy_compat_payload_as_fallback() { + let payload = b"{\"ok\":true}".to_vec(); + let encrypted = encrypt_data(b"secret", &payload).expect("encrypt payload"); + + let decoded = read_compatible_admin_body(Body::from(encrypted), 1024, "/minio/admin/v3/list-users", "secret") + .await + .expect("decode payload"); + + assert_eq!(decoded, payload); + } +} diff --git a/rustfs/src/app/bucket_usecase.rs b/rustfs/src/app/bucket_usecase.rs index a94c17c70..3484080e5 100644 --- a/rustfs/src/app/bucket_usecase.rs +++ b/rustfs/src/app/bucket_usecase.rs @@ -19,8 +19,8 @@ use crate::auth::get_condition_values; use crate::error::ApiError; use crate::server::RemoteAddr; use crate::storage::access::{ReqInfo, authorize_request, req_info_ref}; -use crate::storage::ecfs::{RUSTFS_OWNER, default_owner}; use crate::storage::helper::OperationHelper; +use crate::storage::s3_api::bucket::{build_list_buckets_output, build_list_objects_v2_output}; use crate::storage::s3_api::{acl, encryption, replication, tagging}; use crate::storage::*; use futures::StreamExt; @@ -51,7 +51,7 @@ use rustfs_targets::{ EventName, arn::{ARN, TargetIDError}, }; -use rustfs_utils::http::RUSTFS_FORCE_DELETE; +use rustfs_utils::http::{SUFFIX_FORCE_DELETE, get_header}; use rustfs_utils::string::parse_bool; use s3s::dto::*; use s3s::region::Region; @@ -59,7 +59,6 @@ use s3s::xml; use s3s::{S3Error, S3ErrorCode, S3Request, S3Response, S3Result, s3_error}; use std::{collections::HashSet, fmt::Display, sync::Arc}; use tracing::{debug, error, info, instrument, warn}; -use urlencoding::encode; fn serialize_config(value: &T) -> S3Result> { serialize(value).map_err(to_internal_error) @@ -69,6 +68,17 @@ fn to_internal_error(err: impl Display) -> S3Error { S3Error::with_message(S3ErrorCode::InternalError, format!("{err}")) } +fn create_bucket_exists_response(is_owner: bool) -> S3Result> { + if is_owner { + return Ok(S3Response::new(CreateBucketOutput::default())); + } + + Err(s3_error!( + BucketAlreadyExists, + "The requested bucket name is not available. The bucket namespace is shared by all users of the system. Please select a different name and try again." + )) +} + fn resolve_notification_region(global_region: Option, request_region: Option) -> String { global_region .or(request_region) @@ -147,8 +157,8 @@ impl DefaultBucketUsecase { } let helper = OperationHelper::new(&req, EventName::BucketCreated, S3Operation::CreateBucket); - let requester_id = match req_info_ref(&req) { - Ok(r) => r.cred.as_ref().map(|c| c.access_key.clone()), + let requester_is_owner = match req_info_ref(&req) { + Ok(r) => r.is_owner, Err(_) => { return Err(S3Error::with_message(S3ErrorCode::InternalError, "Missing request info".to_string())); } @@ -179,18 +189,7 @@ impl DefaultBucketUsecase { Err(StorageError::BucketExists(_)) => { // Per S3 spec: bucket namespace is global. Owner recreating returns 200 OK; // non-owner gets 409 BucketAlreadyExists. - let is_owner = requester_id.as_deref().is_some_and(|req_id| req_id == default_owner().id); - - if is_owner { - let output = CreateBucketOutput::default(); - let result = Ok(S3Response::new(output)); - let _ = helper.complete(&result); - return result; - } - let result = Err(s3_error!( - BucketAlreadyExists, - "The requested bucket name is not available. The bucket namespace is shared by all users of the system. Please select a different name and try again." - )); + let result = create_bucket_exists_response(requester_is_owner); let _ = helper.complete(&result); return result; } @@ -247,19 +246,11 @@ impl DefaultBucketUsecase { return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); }; - // get value from header, support mc style - let force_str = req - .headers - .get(RUSTFS_FORCE_DELETE) - .map(|v| v.to_str().unwrap_or_default()) - .unwrap_or( - req.headers - .get("x-minio-force-delete") - .map(|v| v.to_str().unwrap_or_default()) - .unwrap_or_default(), - ); + let force_str = get_header(&req.headers, SUFFIX_FORCE_DELETE) + .map(|v| v.into_owned()) + .unwrap_or_default(); - let force = parse_bool(force_str).unwrap_or_default(); + let force = parse_bool(&force_str).unwrap_or_default(); if force { authorize_request(&mut req, Action::S3Action(S3Action::ForceDeleteBucketAction)).await?; @@ -404,20 +395,7 @@ impl DefaultBucketUsecase { store.list_bucket(&BucketOptions::default()).await.map_err(ApiError::from)? }; - let buckets: Vec = bucket_infos - .iter() - .map(|v| Bucket { - creation_date: v.created.map(Timestamp::from), - name: Some(v.name.clone()), - ..Default::default() - }) - .collect(); - - Ok(S3Response::new(ListBucketsOutput { - buckets: Some(buckets), - owner: Some(RUSTFS_OWNER.to_owned()), - ..Default::default() - })) + Ok(S3Response::new(build_list_buckets_output(&bucket_infos))) } pub async fn execute_delete_bucket_encryption( @@ -1413,10 +1391,9 @@ impl DefaultBucketUsecase { let store = get_validated_store(&bucket).await?; - let incl_deleted = req - .headers - .get(rustfs_utils::http::headers::RUSTFS_INCLUDE_DELETED) - .is_some_and(|v| v.to_str().unwrap_or_default() == "true"); + let incl_deleted = rustfs_utils::http::get_header(&req.headers, rustfs_utils::http::SUFFIX_INCLUDE_DELETED) + .map(|v| v.as_ref() == "true") + .unwrap_or_default(); let object_infos = store .list_objects_v2( @@ -1432,84 +1409,18 @@ impl DefaultBucketUsecase { .await .map_err(ApiError::from)?; - // warn!("object_infos objects {:?}", object_infos.objects); - - // Apply URL encoding if encoding_type is "url" - // Note: S3 URL encoding should encode special characters but preserve path separators (/) - let should_encode = encoding_type.as_ref().map(|e| e.as_str() == "url").unwrap_or(false); - - // Helper function to encode S3 keys/prefixes (preserving /) - // S3 URL encoding encodes special characters but keeps '/' unencoded - let encode_s3_name = |name: &str| -> String { - name.split('/') - .map(|part| encode(part).to_string()) - .collect::>() - .join("/") - }; - - let objects: Vec = object_infos - .objects - .iter() - .filter(|v| !v.name.is_empty()) - .map(|v| { - let key = if should_encode { - encode_s3_name(&v.name) - } else { - v.name.to_owned() - }; - let mut obj = Object { - key: Some(key), - last_modified: v.mod_time.map(Timestamp::from), - size: Some(v.get_actual_size().unwrap_or_default()), - e_tag: v.etag.clone().map(|etag| to_s3s_etag(&etag)), - storage_class: v.storage_class.clone().map(ObjectStorageClass::from), - ..Default::default() - }; - - if fetch_owner.is_some_and(|v| v) { - obj.owner = Some(Owner { - display_name: Some("rustfs".to_owned()), - id: Some("v0.1".to_owned()), - }); - } - obj - }) - .collect(); - - let common_prefixes: Vec = object_infos - .prefixes - .into_iter() - .map(|v| { - let prefix = if should_encode { encode_s3_name(&v) } else { v }; - CommonPrefix { prefix: Some(prefix) } - }) - .collect(); - - // KeyCount should include both objects and common prefixes per S3 API spec - let key_count = (objects.len() + common_prefixes.len()) as i32; - - // Encode next_continuation_token to base64 - let next_continuation_token = object_infos - .next_continuation_token - .map(|token| base64_simd::STANDARD.encode_to_string(token.as_bytes())); - - let output = ListObjectsV2Output { - is_truncated: Some(object_infos.is_truncated), - continuation_token: response_continuation_token, - next_continuation_token, - start_after: response_start_after, - key_count: Some(key_count), - max_keys: Some(max_keys), - contents: Some(objects), + let output = build_list_objects_v2_output( + object_infos, + fetch_owner.unwrap_or_default(), + max_keys, + bucket, + prefix, delimiter, - encoding_type: encoding_type.clone(), - name: Some(bucket), - prefix: Some(prefix), - common_prefixes: Some(common_prefixes), - ..Default::default() - }; + encoding_type, + response_continuation_token, + response_start_after, + ); - // let output = ListObjectsV2Output { ..Default::default() }; Ok(S3Response::new(output)) } @@ -1679,6 +1590,12 @@ mod tests { } } + fn build_request_with_req_info(input: T, method: Method, req_info: ReqInfo) -> S3Request { + let mut req = build_request(input, method); + req.extensions.insert(req_info); + req + } + #[test] fn resolve_notification_region_prefers_global_region() { let binding = resolve_notification_region(Some("us-east-1".parse().unwrap()), Some("ap-southeast-1".parse().unwrap())); @@ -1697,6 +1614,36 @@ mod tests { assert_eq!(binding, RUSTFS_REGION); } + #[test] + fn create_bucket_exists_response_returns_ok_for_owner() { + let response = create_bucket_exists_response(true).expect("owner recreate should succeed"); + assert_eq!(response.output.location, None); + } + + #[test] + fn create_bucket_exists_response_returns_bucket_already_exists_for_non_owner() { + let err = create_bucket_exists_response(false).expect_err("non-owner recreate should fail"); + assert_eq!(err.code(), &S3ErrorCode::BucketAlreadyExists); + } + + #[test] + fn build_request_with_req_info_preserves_owner_state() { + let input = CreateBucketInput::builder() + .bucket("test-bucket".to_string()) + .build() + .unwrap(); + let req = build_request_with_req_info( + input, + Method::PUT, + ReqInfo { + is_owner: true, + ..Default::default() + }, + ); + + assert!(req_info_ref(&req).expect("req info should be present").is_owner); + } + #[tokio::test] async fn execute_create_bucket_returns_internal_error_when_store_uninitialized() { let input = CreateBucketInput::builder() @@ -1886,6 +1833,7 @@ mod tests { ..Default::default() }), abort_incomplete_multipart_upload: None, + del_marker_expiration: None, filter: None, id: None, noncurrent_version_expiration: None, @@ -1900,6 +1848,7 @@ mod tests { ..Default::default() }), abort_incomplete_multipart_upload: None, + del_marker_expiration: None, filter: None, id: Some("rule-1".to_string()), noncurrent_version_expiration: None, @@ -1914,6 +1863,7 @@ mod tests { ..Default::default() }), abort_incomplete_multipart_upload: None, + del_marker_expiration: None, filter: None, id: None, noncurrent_version_expiration: None, @@ -1939,6 +1889,7 @@ mod tests { ..Default::default() }), abort_incomplete_multipart_upload: None, + del_marker_expiration: None, filter: None, id: None, noncurrent_version_expiration: None, diff --git a/rustfs/src/app/multipart_usecase.rs b/rustfs/src/app/multipart_usecase.rs index e2232169a..b2df090bd 100644 --- a/rustfs/src/app/multipart_usecase.rs +++ b/rustfs/src/app/multipart_usecase.rs @@ -17,17 +17,17 @@ use crate::app::context::{AppContext, get_global_app_context}; use crate::error::ApiError; use crate::storage::concurrency::get_concurrency_manager; -use crate::storage::ecfs::RUSTFS_OWNER; use crate::storage::entity; use crate::storage::helper::OperationHelper; use crate::storage::options::{ copy_src_opts, extract_metadata, get_complete_multipart_upload_opts, get_content_sha256_with_query, parse_copy_source_range, put_opts, }; +use crate::storage::s3_api::multipart::build_list_parts_output; use crate::storage::*; use bytes::Bytes; use futures::StreamExt; -use rustfs_config::RUSTFS_REGION; +use http::{HeaderMap, Uri}; use rustfs_ecstore::bucket::quota::checker::QuotaChecker; use rustfs_ecstore::bucket::{ metadata_sys, @@ -47,18 +47,18 @@ use rustfs_s3_common::S3Operation; use rustfs_targets::EventName; use rustfs_utils::CompressionAlgorithm; use rustfs_utils::http::{ - AMZ_CHECKSUM_TYPE, - headers::{AMZ_DECODED_CONTENT_LENGTH, AMZ_OBJECT_TAGGING, RESERVED_METADATA_PREFIX_LOWER}, + AMZ_CHECKSUM_TYPE, get_source_scheme, + headers::{AMZ_DECODED_CONTENT_LENGTH, AMZ_OBJECT_TAGGING}, }; use s3s::dto::*; -use s3s::region::Region; use s3s::{S3Error, S3ErrorCode, S3Request, S3Response, S3Result, s3_error}; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::str::FromStr; use std::sync::Arc; use tokio::sync::RwLock; use tokio_util::io::StreamReader; use tracing::{info, instrument, warn}; +use urlencoding::encode; /// Returns InvalidRange error if CopySourceRange end exceeds the source object size. /// Used by execute_upload_part_copy to reject out-of-bounds ranges per S3 spec. @@ -72,6 +72,74 @@ fn validate_copy_source_range_not_exceeds(range_spec: &HTTPRangeSpec, object_siz Ok(()) } +fn validate_complete_multipart_parts(parts: &[CompletePart]) -> S3Result<()> { + if parts.windows(2).any(|window| window[0].part_num >= window[1].part_num) { + return Err(s3_error!(InvalidPartOrder, "Part numbers must be strictly increasing")); + } + + Ok(()) +} + +fn normalize_complete_multipart_parts(parts: Vec) -> S3Result> { + // For duplicate part numbers, keep the last occurrence from the request. + // This matches retry/resend semantics where later uploads override earlier ones. + let mut seen = HashSet::with_capacity(parts.len()); + let mut deduped_reversed = Vec::with_capacity(parts.len()); + for part in parts.into_iter().rev() { + if seen.insert(part.part_num) { + deduped_reversed.push(part); + } + } + deduped_reversed.reverse(); + + validate_complete_multipart_parts(&deduped_reversed)?; + Ok(deduped_reversed) +} + +fn encode_s3_path(path: &str) -> String { + path.split('/') + .map(|part| encode(part).to_string()) + .collect::>() + .join("/") +} + +fn extract_request_scheme(headers: &HeaderMap, uri: &Uri) -> String { + get_source_scheme(headers) + .and_then(|value| { + value + .split(',') + .next() + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(ToOwned::to_owned) + }) + .or_else(|| uri.scheme_str().map(str::to_owned)) + .unwrap_or_else(|| "http".to_string()) + .to_ascii_lowercase() +} + +fn extract_request_host(headers: &HeaderMap, uri: &Uri) -> Option { + headers + .get(http::header::HOST) + .and_then(|value| value.to_str().ok()) + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(ToOwned::to_owned) + .or_else(|| uri.authority().map(|authority| authority.as_str().to_string())) +} + +fn build_complete_multipart_location(headers: &HeaderMap, uri: &Uri, bucket: &str, key: &str) -> String { + let object_path = format!("/{}/{}", encode(bucket), encode_s3_path(key)); + + match extract_request_host(headers, uri) { + Some(host) => { + let scheme = extract_request_scheme(headers, uri); + format!("{scheme}://{host}{object_path}") + } + None => object_path, + } +} + #[derive(Clone, Default)] pub struct DefaultMultipartUsecase { context: Option>, @@ -93,10 +161,6 @@ impl DefaultMultipartUsecase { self.context.as_ref().and_then(|context| context.bucket_metadata().handle()) } - fn global_region(&self) -> Option { - self.context.as_ref().and_then(|context| context.region().get()) - } - #[instrument(level = "debug", skip(self))] pub async fn execute_abort_multipart_upload( &self, @@ -213,23 +277,7 @@ impl DefaultMultipartUsecase { .map(CompletePart::from) .collect::>(); - // is part number sorted? - if !uploaded_parts_vec.is_sorted_by_key(|p| p.part_num) { - return Err(s3_error!(InvalidPart, "Part numbers must be sorted")); - } - - // Handle duplicate part numbers: according to S3 specification, when the same part number - // is uploaded multiple times, the last uploaded part (in the list order) should be used. - // This can happen in concurrent upload scenarios where a part is re-uploaded before completion. - // We deduplicate by keeping the last occurrence of each part number using a HashMap. - let mut part_map: HashMap = HashMap::new(); - for part in uploaded_parts_vec { - part_map.insert(part.part_num, part); - } - - // Reconstruct the parts list in sorted order, keeping only the last occurrence of each part number - let mut uploaded_parts: Vec = part_map.into_values().collect(); - uploaded_parts.sort_by_key(|p| p.part_num); + let uploaded_parts = normalize_complete_multipart_parts(uploaded_parts_vec)?; // TODO: check object lock @@ -354,15 +402,12 @@ impl DefaultMultipartUsecase { } } - let region = self - .global_region() - .map(|region| region.to_string()) - .unwrap_or_else(|| RUSTFS_REGION.to_string()); + let location = build_complete_multipart_location(&req.headers, &req.uri, &bucket, &key); let output = CompleteMultipartUploadOutput { bucket: Some(bucket.clone()), key: Some(key.clone()), e_tag: obj_info.etag.clone().map(|etag| to_s3s_etag(&etag)), - location: Some(region.clone()), + location: Some(location.clone()), server_side_encryption: server_side_encryption.clone(), ssekms_key_id: ssekms_key_id.clone(), checksum_crc32: checksum_crc32.clone(), @@ -378,7 +423,7 @@ impl DefaultMultipartUsecase { bucket: Some(bucket.clone()), key: Some(key.clone()), e_tag: obj_info.etag.clone().map(|etag| to_s3s_etag(&etag)), - location: Some(region), + location: Some(location), server_side_encryption, ssekms_key_id, checksum_crc32, @@ -482,8 +527,9 @@ impl DefaultMultipartUsecase { }; if is_compressible(&req.headers, &key) { - metadata.insert( - format!("{RESERVED_METADATA_PREFIX_LOWER}compression"), + rustfs_utils::http::insert_str( + &mut metadata, + rustfs_utils::http::SUFFIX_COMPRESSION, CompressionAlgorithm::default().to_string(), ); } @@ -603,9 +649,7 @@ impl DefaultMultipartUsecase { StreamReader::new(body_stream.map(|f| f.map_err(|e| std::io::Error::other(e.to_string())))), ); - let is_compressible = fi - .user_defined - .contains_key(format!("{RESERVED_METADATA_PREFIX_LOWER}compression").as_str()); + let is_compressible = rustfs_utils::http::contains_key_str(&fi.user_defined, rustfs_utils::http::SUFFIX_COMPRESSION); let mut reader: Box = Box::new(WarpReader::new(body)); @@ -857,39 +901,7 @@ impl DefaultMultipartUsecase { .await .map_err(ApiError::from)?; - let output = ListPartsOutput { - bucket: Some(res.bucket), - key: Some(res.object), - upload_id: Some(res.upload_id), - parts: Some( - res.parts - .into_iter() - .map(|p| Part { - e_tag: p.etag.map(|etag| to_s3s_etag(&etag)), - last_modified: p.last_mod.map(Timestamp::from), - part_number: Some(p.part_num as i32), - size: Some(p.size as i64), - ..Default::default() - }) - .collect(), - ), - owner: Some(RUSTFS_OWNER.to_owned()), - initiator: Some(Initiator { - id: RUSTFS_OWNER.id.clone(), - display_name: RUSTFS_OWNER.display_name.clone(), - }), - is_truncated: Some(res.is_truncated), - next_part_number_marker: res.next_part_number_marker.try_into().ok(), - max_parts: res.max_parts.try_into().ok(), - part_number_marker: res.part_number_marker.try_into().ok(), - storage_class: if res.storage_class.is_empty() { - None - } else { - Some(res.storage_class.into()) - }, - ..Default::default() - }; - Ok(S3Response::new(output)) + Ok(S3Response::new(build_list_parts_output(res))) } #[instrument(level = "debug", skip(self, req))] @@ -1014,9 +1026,7 @@ impl DefaultMultipartUsecase { .map_err(ApiError::from)?; let src_stream = src_reader.stream; - let is_compressible = mp_info - .user_defined - .contains_key(format!("{RESERVED_METADATA_PREFIX_LOWER}compression").as_str()); + let is_compressible = rustfs_utils::http::contains_key_str(&mp_info.user_defined, rustfs_utils::http::SUFFIX_COMPRESSION); let mut reader: Box = Box::new(WarpReader::new(src_stream)); @@ -1129,7 +1139,7 @@ impl DefaultMultipartUsecase { #[cfg(test)] mod tests { use super::*; - use http::{Extensions, HeaderMap, Method, Uri}; + use http::{Extensions, HeaderMap, Method, Uri, header::HeaderValue}; fn build_request(input: T, method: Method) -> S3Request { S3Request { @@ -1149,6 +1159,41 @@ mod tests { DefaultMultipartUsecase::without_context() } + #[test] + fn test_build_complete_multipart_location_uses_forwarded_proto_and_encodes_key() { + let mut headers = HeaderMap::new(); + headers.insert(http::header::HOST, HeaderValue::from_static("storage.example.com:9000")); + headers.insert("x-forwarded-proto", HeaderValue::from_static("https")); + + let location = build_complete_multipart_location( + &headers, + &Uri::from_static("/bucket/object?uploadId=1"), + "bucket", + "dir/file name.txt", + ); + + assert_eq!(location, "https://storage.example.com:9000/bucket/dir/file%20name.txt"); + } + + #[test] + fn test_build_complete_multipart_location_falls_back_to_uri_authority_and_scheme() { + let location = build_complete_multipart_location( + &HeaderMap::new(), + &"https://gateway.example.com:9443/complete".parse::().unwrap(), + "bucket", + "object.txt", + ); + + assert_eq!(location, "https://gateway.example.com:9443/bucket/object.txt"); + } + + #[test] + fn test_build_complete_multipart_location_returns_path_without_host() { + let location = build_complete_multipart_location(&HeaderMap::new(), &Uri::from_static("/"), "bucket", "nested/object"); + + assert_eq!(location, "/bucket/nested/object"); + } + #[tokio::test] async fn execute_abort_multipart_upload_returns_internal_error_when_store_uninitialized() { let input = AbortMultipartUploadInput::builder() @@ -1191,6 +1236,81 @@ mod tests { assert_eq!(err.code(), &S3ErrorCode::InvalidPart); } + #[tokio::test] + async fn execute_complete_multipart_upload_allows_duplicate_part_numbers_by_using_last_occurrence() { + let multipart_upload = CompletedMultipartUpload { + parts: Some(vec![ + CompletedPart { + part_number: Some(1), + ..Default::default() + }, + CompletedPart { + part_number: Some(1), + ..Default::default() + }, + ]), + }; + let input = CompleteMultipartUploadInput::builder() + .bucket("bucket".to_string()) + .key("object".to_string()) + .upload_id("upload-id".to_string()) + .multipart_upload(Some(multipart_upload)) + .build() + .unwrap(); + let req = build_request(input, Method::POST); + + let err = make_usecase().execute_complete_multipart_upload(req).await.unwrap_err(); + assert_ne!(err.code(), &S3ErrorCode::InvalidPartOrder); + } + + #[tokio::test] + async fn execute_complete_multipart_upload_rejects_out_of_order_parts() { + let multipart_upload = CompletedMultipartUpload { + parts: Some(vec![ + CompletedPart { + part_number: Some(2), + ..Default::default() + }, + CompletedPart { + part_number: Some(1), + ..Default::default() + }, + ]), + }; + let input = CompleteMultipartUploadInput::builder() + .bucket("bucket".to_string()) + .key("object".to_string()) + .upload_id("upload-id".to_string()) + .multipart_upload(Some(multipart_upload)) + .build() + .unwrap(); + let req = build_request(input, Method::POST); + + let err = make_usecase().execute_complete_multipart_upload(req).await.unwrap_err(); + assert_eq!(err.code(), &S3ErrorCode::InvalidPartOrder); + } + + #[test] + fn normalize_complete_multipart_parts_keeps_last_duplicate_part() { + let input = vec![ + CompletePart { + part_num: 1, + etag: Some("old".to_string()), + ..Default::default() + }, + CompletePart { + part_num: 1, + etag: Some("new".to_string()), + ..Default::default() + }, + ]; + + let normalized = normalize_complete_multipart_parts(input).expect("normalization should succeed"); + assert_eq!(normalized.len(), 1); + assert_eq!(normalized[0].part_num, 1); + assert_eq!(normalized[0].etag.as_deref(), Some("new")); + } + #[tokio::test] async fn execute_list_multipart_uploads_returns_internal_error_when_store_uninitialized() { let input = ListMultipartUploadsInput::builder() diff --git a/rustfs/src/app/object_usecase.rs b/rustfs/src/app/object_usecase.rs index 256042154..aac013524 100644 --- a/rustfs/src/app/object_usecase.rs +++ b/rustfs/src/app/object_usecase.rs @@ -82,13 +82,14 @@ use rustfs_s3select_api::{ use rustfs_s3select_query::get_global_db; use rustfs_targets::EventName; use rustfs_utils::http::{ - AMZ_BUCKET_REPLICATION_STATUS, AMZ_CHECKSUM_MODE, AMZ_CHECKSUM_TYPE, RESERVED_METADATA_PREFIX, + AMZ_BUCKET_REPLICATION_STATUS, AMZ_CHECKSUM_MODE, AMZ_CHECKSUM_TYPE, SUFFIX_ACTUAL_SIZE, SUFFIX_COMPRESSION, + SUFFIX_COMPRESSION_SIZE, SUFFIX_REPLICATION_STATUS, SUFFIX_REPLICATION_TIMESTAMP, headers::{ AMZ_DECODED_CONTENT_LENGTH, AMZ_OBJECT_LOCK_LEGAL_HOLD, AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, AMZ_OBJECT_LOCK_MODE, AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER, AMZ_OBJECT_TAGGING, AMZ_RESTORE_EXPIRY_DAYS, AMZ_RESTORE_REQUEST_DATE, AMZ_STORAGE_CLASS, AMZ_TAG_COUNT, - RESERVED_METADATA_PREFIX_LOWER, }, + insert_str, remove_str, }; use rustfs_utils::path::{is_dir_object, path_join_buf}; use rustfs_utils::{ @@ -426,9 +427,8 @@ impl DefaultObjectUsecase { if is_compressible(&req.headers, &key) && size > MIN_COMPRESSIBLE_SIZE as i64 { let algorithm = CompressionAlgorithm::default(); - metadata.insert(format!("{RESERVED_METADATA_PREFIX_LOWER}compression"), algorithm.to_string()); - - metadata.insert(format!("{RESERVED_METADATA_PREFIX_LOWER}actual-size",), size.to_string()); + insert_str(&mut metadata, SUFFIX_COMPRESSION, algorithm.to_string()); + insert_str(&mut metadata, SUFFIX_ACTUAL_SIZE, size.to_string()); let mut hrd = HashReader::new(reader, size as i64, size as i64, md5hex, sha256hex, false).map_err(ApiError::from)?; @@ -437,10 +437,8 @@ impl DefaultObjectUsecase { } opts.want_checksum = hrd.checksum(); - opts.user_defined - .insert(format!("{RESERVED_METADATA_PREFIX_LOWER}compression"), algorithm.to_string()); - opts.user_defined - .insert(format!("{RESERVED_METADATA_PREFIX_LOWER}actual-size",), size.to_string()); + insert_str(&mut opts.user_defined, SUFFIX_COMPRESSION, algorithm.to_string()); + insert_str(&mut opts.user_defined, SUFFIX_ACTUAL_SIZE, size.to_string()); reader = Box::new(CompressReader::new(hrd, algorithm)); size = HashReader::SIZE_PRESERVE_LAYER; @@ -497,10 +495,12 @@ impl DefaultObjectUsecase { let dsc = must_replicate(&bucket, &key, repoptions).await; if dsc.replicate_any() { - let k = format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, "replication-timestamp"); - opts.user_defined.insert(k, jiff::Zoned::now().to_string()); - let k = format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, "replication-status"); - opts.user_defined.insert(k, dsc.pending_status().unwrap_or_default()); + insert_str(&mut opts.user_defined, SUFFIX_REPLICATION_TIMESTAMP, jiff::Zoned::now().to_string()); + insert_str( + &mut opts.user_defined, + SUFFIX_REPLICATION_STATUS, + dsc.pending_status().unwrap_or_default(), + ); } let obj_info = store @@ -2104,11 +2104,8 @@ impl DefaultObjectUsecase { let mut compress_metadata = HashMap::new(); if is_compressible(&req.headers, &key) && actual_size > MIN_COMPRESSIBLE_SIZE as i64 { - compress_metadata.insert( - format!("{RESERVED_METADATA_PREFIX_LOWER}compression"), - CompressionAlgorithm::default().to_string(), - ); - compress_metadata.insert(format!("{RESERVED_METADATA_PREFIX_LOWER}actual-size",), actual_size.to_string()); + insert_str(&mut compress_metadata, SUFFIX_COMPRESSION, CompressionAlgorithm::default().to_string()); + insert_str(&mut compress_metadata, SUFFIX_ACTUAL_SIZE, actual_size.to_string()); let hrd = EtagReader::new(reader, None); @@ -2117,24 +2114,9 @@ impl DefaultObjectUsecase { reader = Box::new(CompressReader::new(hrd, CompressionAlgorithm::default())); length = HashReader::SIZE_PRESERVE_LAYER; } else { - src_info - .user_defined - .remove(&format!("{RESERVED_METADATA_PREFIX_LOWER}compression")); - src_info - .user_defined - .remove(&format!("{RESERVED_METADATA_PREFIX}compression")); - src_info - .user_defined - .remove(&format!("{RESERVED_METADATA_PREFIX_LOWER}actual-size")); - src_info - .user_defined - .remove(&format!("{RESERVED_METADATA_PREFIX}actual-size")); - src_info - .user_defined - .remove(&format!("{RESERVED_METADATA_PREFIX_LOWER}compression-size")); - src_info - .user_defined - .remove(&format!("{RESERVED_METADATA_PREFIX}compression-size")); + remove_str(&mut src_info.user_defined, SUFFIX_COMPRESSION); + remove_str(&mut src_info.user_defined, SUFFIX_ACTUAL_SIZE); + remove_str(&mut src_info.user_defined, SUFFIX_COMPRESSION_SIZE); } // Handle MetadataDirective REPLACE: replace user metadata while preserving system metadata. @@ -3480,11 +3462,8 @@ impl DefaultObjectUsecase { let actual_size = size; if is_compressible(&HeaderMap::new(), &fpath) && size > MIN_COMPRESSIBLE_SIZE as i64 { - metadata.insert( - format!("{RESERVED_METADATA_PREFIX_LOWER}compression"), - CompressionAlgorithm::default().to_string(), - ); - metadata.insert(format!("{RESERVED_METADATA_PREFIX_LOWER}actual-size"), size.to_string()); + insert_str(&mut metadata, SUFFIX_COMPRESSION, CompressionAlgorithm::default().to_string()); + insert_str(&mut metadata, SUFFIX_ACTUAL_SIZE, size.to_string()); let hrd = HashReader::new(reader, size, actual_size, None, None, false).map_err(ApiError::from)?; diff --git a/rustfs/src/config/config_test.rs b/rustfs/src/config/config_test.rs index 86d1286ee..63e908138 100644 --- a/rustfs/src/config/config_test.rs +++ b/rustfs/src/config/config_test.rs @@ -15,8 +15,7 @@ #[cfg(test)] #[allow(unsafe_op_in_unsafe_fn)] mod tests { - use crate::config::Opt; - use clap::Parser; + use crate::config::{Config, Opt}; use rustfs_ecstore::disks_layout::DisksLayout; use serial_test::serial; use std::env; @@ -56,6 +55,19 @@ mod tests { verify_fn(&layout); } + #[test] + #[serial] + fn test_server_subcommand_and_legacy_equivalence() { + // rustfs server and rustfs (legacy) must produce identical results + let legacy_args = vec!["rustfs", "/data/vol1"]; + let server_args = vec!["rustfs", "server", "/data/vol1"]; + let opt_legacy = Opt::parse_from(legacy_args); + let opt_server = Opt::parse_from(server_args); + assert_eq!(opt_legacy.volumes, opt_server.volumes); + assert_eq!(opt_legacy.address, opt_server.address); + assert_eq!(opt_legacy.console_address, opt_server.console_address); + } + #[test] #[serial] fn test_default_console_configuration() { @@ -109,6 +121,64 @@ mod tests { assert_eq!(console_port, 9001); } + #[test] + #[serial] + fn test_external_prefixed_envs_are_accepted_by_parser() { + temp_env::with_vars( + [ + ("MINIO_VOLUMES", Some("/compat/vol1")), + ("MINIO_ADDRESS", Some(":9100")), + ("RUSTFS_VOLUMES", None), + ("RUSTFS_ADDRESS", None), + ], + || { + let opt = Opt::parse_from(["rustfs"]); + assert_eq!(opt.volumes, vec!["/compat/vol1"]); + assert_eq!(opt.address, ":9100"); + assert_eq!(std::env::var("RUSTFS_VOLUMES").as_deref(), Ok("/compat/vol1")); + assert_eq!(std::env::var("RUSTFS_ADDRESS").as_deref(), Ok(":9100")); + }, + ); + } + + #[test] + #[serial] + fn test_root_envs_are_used_for_bootstrap_credentials() { + temp_env::with_vars( + [ + ("RUSTFS_VOLUMES", Some("/compat/vol1")), + ("RUSTFS_ROOT_USER", Some("root-user")), + ("RUSTFS_ROOT_PASSWORD", Some("root-password")), + ("RUSTFS_ACCESS_KEY", None), + ("RUSTFS_SECRET_KEY", None), + ], + || { + let config = Config::from_opt(Opt::parse_from(["rustfs"])).expect("config should parse"); + assert_eq!(config.access_key, "root-user"); + assert_eq!(config.secret_key, "root-password"); + }, + ); + } + + #[test] + #[serial] + fn test_access_key_env_takes_precedence_over_root_aliases() { + temp_env::with_vars( + [ + ("RUSTFS_VOLUMES", Some("/compat/vol1")), + ("RUSTFS_ACCESS_KEY", Some("canonical-access")), + ("RUSTFS_SECRET_KEY", Some("canonical-secret")), + ("RUSTFS_ROOT_USER", Some("root-user")), + ("RUSTFS_ROOT_PASSWORD", Some("root-password")), + ], + || { + let config = Config::from_opt(Opt::parse_from(["rustfs"])).expect("config should parse"); + assert_eq!(config.access_key, "canonical-access"); + assert_eq!(config.secret_key, "canonical-secret"); + }, + ); + } + #[test] #[serial] fn test_volumes_and_disk_layout_parsing() { diff --git a/rustfs/src/config/mod.rs b/rustfs/src/config/mod.rs index c68345161..ea6b20a64 100644 --- a/rustfs/src/config/mod.rs +++ b/rustfs/src/config/mod.rs @@ -12,10 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -use clap::Parser; use clap::builder::NonEmptyStringValueParser; +use clap::{Args, Parser, Subcommand}; use const_str::concat; use rustfs_config::RUSTFS_REGION; +use rustfs_utils::{apply_external_env_compat, get_env_opt_str}; use std::path::PathBuf; use std::string::ToString; @@ -50,9 +51,47 @@ const LONG_VERSION: &str = concat!( concat!("git status :\n", build::GIT_STATUS_FILE), ); +/// Known subcommands. When the first arg matches one of these, it is treated as a subcommand. +const KNOWN_SUBCOMMANDS: &[&str] = &["server"]; + +/// Preprocess argv for legacy compatibility: `rustfs ` and `rustfs --address ...` are +/// treated as `rustfs server ` and `rustfs server --address ...` respectively. +/// Also: `rustfs` with no args becomes `rustfs server` (volumes from env). +fn preprocess_args_for_legacy(args: Vec) -> Vec { + if args.len() < 2 { + // rustfs -> rustfs server (volumes from RUSTFS_VOLUMES env) + return vec![args[0].clone(), "server".to_string()]; + } + let first = &args[1]; + // If first arg looks like a subcommand, do nothing + if KNOWN_SUBCOMMANDS.contains(&first.as_str()) { + return args; + } + // If first arg is a global flag (--help, --version), do nothing + if first == "--help" || first == "-h" || first == "--version" || first == "-V" { + return args; + } + // Legacy: rustfs or rustfs --address ... -> rustfs server + let mut out = vec![args[0].clone(), "server".to_string()]; + out.extend(args[1..].iter().cloned()); + out +} + #[derive(Parser, Clone)] -#[command(version = SHORT_VERSION, long_version = LONG_VERSION)] -pub struct Opt { +#[command(name = "rustfs", version = SHORT_VERSION, long_version = LONG_VERSION)] +struct Cli { + #[command(subcommand)] + command: Option, +} + +#[derive(Subcommand, Clone)] +enum Commands { + /// Start the object storage server (default when no subcommand is given) + Server(ServerOpts), +} + +#[derive(Args, Clone)] +struct ServerOpts { /// DIR points to a directory on a filesystem. #[arg( required = true, @@ -164,6 +203,96 @@ pub struct Opt { pub buffer_profile: String, } +/// Parsed server options. Public for tests and backward compatibility. +/// Use `Opt::parse_from` or `Config::parse()` to obtain. +#[derive(Clone)] +pub struct Opt { + pub volumes: Vec, + pub address: String, + pub server_domains: Vec, + pub access_key: Option, + pub access_key_file: Option, + pub secret_key: Option, + pub secret_key_file: Option, + pub console_enable: bool, + pub console_address: String, + pub obs_endpoint: String, + pub tls_path: Option, + pub license: Option, + pub region: Option, + pub kms_enable: bool, + pub kms_backend: String, + pub kms_key_dir: Option, + pub kms_vault_address: Option, + pub kms_vault_token: Option, + pub kms_default_key_id: Option, + pub buffer_profile_disable: bool, + pub buffer_profile: String, +} + +impl Opt { + fn from_server_opts(o: ServerOpts) -> Self { + Self { + volumes: o.volumes, + address: o.address, + server_domains: o.server_domains, + access_key: o.access_key, + access_key_file: o.access_key_file, + secret_key: o.secret_key, + secret_key_file: o.secret_key_file, + console_enable: o.console_enable, + console_address: o.console_address, + obs_endpoint: o.obs_endpoint, + tls_path: o.tls_path, + license: o.license, + region: o.region, + kms_enable: o.kms_enable, + kms_backend: o.kms_backend, + kms_key_dir: o.kms_key_dir, + kms_vault_address: o.kms_vault_address, + kms_vault_token: o.kms_vault_token, + kms_default_key_id: o.kms_default_key_id, + buffer_profile_disable: o.buffer_profile_disable, + buffer_profile: o.buffer_profile, + } + } + + /// Parse from preprocessed args. Supports both `rustfs ` and `rustfs server `. + pub fn parse_from(args: I) -> Self + where + I: IntoIterator, + T: Into + Clone, + { + let _ = apply_external_env_compat(); + let args: Vec = args.into_iter().map(|a| a.into().to_string_lossy().into_owned()).collect(); + let args = preprocess_args_for_legacy(args); + let cli = Cli::parse_from(args); + let Commands::Server(opts) = cli.command.expect("server is the default subcommand"); + Self::from_server_opts(opts) + } + + /// Try parse from args, returns error on invalid input. + #[allow(dead_code)] // used in config_test + pub fn try_parse_from(args: I) -> Result + where + I: IntoIterator, + T: Into + Clone, + { + let _ = apply_external_env_compat(); + let args: Vec = args.into_iter().map(|a| a.into().to_string_lossy().into_owned()).collect(); + let args = preprocess_args_for_legacy(args); + let cli = Cli::try_parse_from(args)?; + let Commands::Server(opts) = cli.command.expect("server is the default subcommand"); + Ok(Self::from_server_opts(opts)) + } + + /// Parse from env::args(). Used by Config::parse(). + fn parse() -> Self { + let args: Vec = std::env::args().collect(); + Self::parse_from(args) + } +} + #[derive(Clone)] pub struct Config { /// DIR points to a directory on a filesystem. @@ -227,11 +356,7 @@ pub struct Config { } impl Config { - /// parse the command line arguments and environment arguments from [`Opt`] and convert them - /// into a ready to use [`Config`] - /// - /// This includes some intermediate checks for mutually exclusive options - pub fn parse() -> std::io::Result { + fn from_opt(opt: Opt) -> std::io::Result { let Opt { volumes, address, @@ -254,7 +379,7 @@ impl Config { kms_default_key_id, buffer_profile_disable, buffer_profile, - } = Opt::parse(); + } = opt; let access_key = access_key .map(Ok) @@ -262,6 +387,7 @@ impl Config { let path = access_key_file.as_ref()?; Some(std::fs::read_to_string(path)) }) + .or_else(|| get_env_opt_str("RUSTFS_ROOT_USER").map(Ok)) .transpose()? .unwrap_or_else(|| { // neither argument was specified ... using default @@ -276,6 +402,7 @@ impl Config { let path = secret_key_file.as_ref()?; Some(std::fs::read_to_string(path)) }) + .or_else(|| get_env_opt_str("RUSTFS_ROOT_PASSWORD").map(Ok)) .transpose()? .unwrap_or_else(|| { // neither argument was specified ... using default @@ -309,6 +436,16 @@ impl Config { buffer_profile, }) } + + /// Parse the command line arguments and environment arguments from [`Opt`] and convert them + /// into a ready to use [`Config`]. + /// + /// Supports both `rustfs ` (legacy) and `rustfs server `. + /// + /// This includes some intermediate checks for mutually exclusive options. + pub fn parse() -> std::io::Result { + Self::from_opt(Opt::parse()) + } } impl std::fmt::Debug for Config { diff --git a/rustfs/src/main.rs b/rustfs/src/main.rs index 9e9b1f9bb..b72265188 100644 --- a/rustfs/src/main.rs +++ b/rustfs/src/main.rs @@ -50,6 +50,7 @@ use rustfs_credentials::init_global_action_credentials; use rustfs_ecstore::store::init_lock_clients; use rustfs_ecstore::{ bucket::metadata_sys::init_bucket_metadata_sys, + bucket::migration::{try_migrate_bucket_metadata, try_migrate_iam_config}, bucket::replication::{get_global_replication_pool, init_background_replication}, config as ecconfig, endpoints::EndpointServerPools, @@ -69,7 +70,9 @@ use rustfs_iam::{init_iam_sys, init_oidc_sys}; use rustfs_metrics::init_metrics_system; use rustfs_obs::{init_obs, set_global_guard}; use rustfs_scanner::init_data_scanner; -use rustfs_utils::{get_env_bool_with_aliases, net::parse_and_resolve_address}; +use rustfs_utils::{ + ExternalEnvCompatReport, apply_external_env_compat, get_env_bool_with_aliases, net::parse_and_resolve_address, +}; use rustls::crypto::aws_lc_rs::default_provider; use std::io::{Error, Result}; use std::sync::Arc; @@ -98,6 +101,10 @@ static GLOBAL: profiling::allocator::TracingAllocator = static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc; fn main() { + if let Err(err) = bootstrap_external_prefix_compat() { + eprintln!("[WARN] Failed to bootstrap external-prefix compatibility: {err}"); + } + let runtime = server::tokio_runtime_builder() .build() .expect("Failed to build Tokio runtime"); @@ -108,6 +115,41 @@ fn main() { std::process::exit(1); } } + +fn bootstrap_external_prefix_compat() -> Result<()> { + let env_compat_report = apply_external_env_compat(); + if env_compat_report.conflict_count() > 0 { + // RUSTFS_* is the canonical namespace in this codebase, so on key conflicts we keep RUSTFS_* + // to preserve explicit user/operator overrides and avoid changing existing runtime behavior. + eprintln!( + "[WARN] Found {} source/RUSTFS_ conflict(s), keeping RUSTFS_ values: {}", + env_compat_report.conflict_count(), + env_compat_report.conflict_keys.join(", ") + ); + } + + if env_compat_report.mapped_count() == 0 { + return Ok(()); + } + + eprintln!( + "[INFO] Applying external-prefix compatibility in-process for {} variable(s): {}", + env_compat_report.mapped_count(), + format_external_prefix_mappings(&env_compat_report) + ); + + Ok(()) +} + +fn format_external_prefix_mappings(report: &ExternalEnvCompatReport) -> String { + report + .mapped_pairs + .iter() + .map(|(source_key, rustfs_key)| format!("{source_key}->{rustfs_key}")) + .collect::>() + .join(", ") +} + async fn async_main() -> Result<()> { // Parse the obtained parameters let config = config::Config::parse()?; @@ -298,6 +340,7 @@ async fn run(config: config::Config) -> Result<()> { })?; ecconfig::init(); + ecconfig::try_migrate_server_config(store.clone()).await; // // Initialize global configuration system let mut retry_count = 0; @@ -398,10 +441,13 @@ async fn run(config: config::Config) -> Result<()> { // Collect bucket names into a vector let buckets: Vec = buckets_list.into_iter().map(|v| v.name).collect(); + try_migrate_bucket_metadata(store.clone()).await; + if let Some(pool) = get_global_replication_pool() { pool.init_resync(ctx.clone(), buckets.clone()).await?; } + try_migrate_iam_config(store.clone()).await; init_bucket_metadata_sys(store.clone(), buckets.clone()).await; // 3. Initialize IAM System (Blocking load) @@ -634,3 +680,29 @@ async fn handle_shutdown( state_manager.update(ServiceState::Stopped); info!(target: "rustfs::main::handle_shutdown", "Server stopped successfully."); } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn format_external_prefix_mappings_lists_mapped_pairs() { + let report = ExternalEnvCompatReport { + mapped_pairs: vec![ + ("MINIO_ROOT_USER".to_string(), "RUSTFS_ROOT_USER".to_string()), + ( + "MINIO_NOTIFY_WEBHOOK_ENABLE_PRIMARY".to_string(), + "RUSTFS_NOTIFY_WEBHOOK_ENABLE_PRIMARY".to_string(), + ), + ], + conflict_keys: Vec::new(), + }; + + let formatted = format_external_prefix_mappings(&report); + + assert_eq!( + formatted, + "MINIO_ROOT_USER->RUSTFS_ROOT_USER, MINIO_NOTIFY_WEBHOOK_ENABLE_PRIMARY->RUSTFS_NOTIFY_WEBHOOK_ENABLE_PRIMARY" + ); + } +} diff --git a/rustfs/src/server/http.rs b/rustfs/src/server/http.rs index 32fa1053a..cb3b62d19 100644 --- a/rustfs/src/server/http.rs +++ b/rustfs/src/server/http.rs @@ -21,7 +21,7 @@ use crate::server::{ ReadinessGateLayer, RemoteAddr, ServiceState, ServiceStateManager, compress::{CompressionConfig, CompressionPredicate}, hybrid::hybrid, - layer::{ConditionalCorsLayer, ObjectAttributesEtagFixLayer, RedirectLayer}, + layer::{AdminChunkedContentLengthCompatLayer, ConditionalCorsLayer, ObjectAttributesEtagFixLayer, RedirectLayer}, }; use crate::storage; use crate::storage::tonic_service::make_server; @@ -621,6 +621,7 @@ fn process_connection( None }) .layer(SetRequestIdLayer::x_request_id(MakeRequestUuid)) + .layer(AdminChunkedContentLengthCompatLayer) .layer(CatchPanicLayer::new()) // CRITICAL: Insert ReadinessGateLayer before business logic // This stops requests from hitting IAMAuth or Storage if they are not ready. diff --git a/rustfs/src/server/layer.rs b/rustfs/src/server/layer.rs index f3e7eddc9..da2c95046 100644 --- a/rustfs/src/server/layer.rs +++ b/rustfs/src/server/layer.rs @@ -15,13 +15,14 @@ use crate::admin::console::is_console_path; use crate::server::cors; use crate::server::hybrid::HybridBody; -use crate::server::{ADMIN_PREFIX, CONSOLE_PREFIX, RPC_PREFIX, RUSTFS_ADMIN_PREFIX}; +use crate::server::{ADMIN_PREFIX, CONSOLE_PREFIX, MINIO_ADMIN_PREFIX, MINIO_ADMIN_V3_PREFIX, RPC_PREFIX, RUSTFS_ADMIN_PREFIX}; use crate::storage::apply_cors_headers; use bytes::Bytes; use http::{HeaderMap, HeaderValue, Method, Request as HttpRequest, Response, StatusCode}; use http_body::Body; use http_body_util::BodyExt; use hyper::body::Incoming; +use rustfs_utils::get_env_opt_str; use std::future::Future; use std::pin::Pin; use std::sync::Arc; @@ -98,6 +99,64 @@ where } } +#[derive(Clone)] +pub struct AdminChunkedContentLengthCompatLayer; + +impl Layer for AdminChunkedContentLengthCompatLayer { + type Service = AdminChunkedContentLengthCompatService; + + fn layer(&self, inner: S) -> Self::Service { + AdminChunkedContentLengthCompatService { inner } + } +} + +#[derive(Clone)] +pub struct AdminChunkedContentLengthCompatService { + inner: S, +} + +impl Service> for AdminChunkedContentLengthCompatService +where + S: Service, Response = Response> + Clone + Send + 'static, + S::Future: Send + 'static, + S::Error: Into> + Send + 'static, + ResBody: Send + 'static, +{ + type Response = Response; + type Error = Box; + type Future = Pin> + Send>>; + + fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { + self.inner.poll_ready(cx).map_err(Into::into) + } + + fn call(&mut self, mut req: HttpRequest) -> Self::Future { + if should_force_zero_content_length_for_admin_empty_body(&req) { + req.headers_mut() + .insert(http::header::CONTENT_LENGTH, HeaderValue::from_static("0")); + } + + let mut inner = self.inner.clone(); + Box::pin(async move { inner.call(req).await.map_err(Into::into) }) + } +} + +fn should_force_zero_content_length_for_admin_empty_body(req: &HttpRequest) -> bool { + req.method() == Method::PUT + && is_empty_body_admin_put_path(req.uri().path()) + && !req.headers().contains_key(http::header::CONTENT_LENGTH) +} + +fn is_empty_body_admin_put_path(path: &str) -> bool { + matches!( + path, + "/minio/admin/v3/set-user-status" + | "/minio/admin/v3/set-group-status" + | "/rustfs/admin/v3/set-user-status" + | "/rustfs/admin/v3/set-group-status" + ) +} + #[derive(Clone)] pub struct ObjectAttributesEtagFixLayer; @@ -220,7 +279,9 @@ fn is_object_attributes_request(req: &HttpRequest) -> bool { let path = req.uri().path(); if path.starts_with(ADMIN_PREFIX) + || path.starts_with(MINIO_ADMIN_PREFIX) || path.starts_with(RUSTFS_ADMIN_PREFIX) + || path.starts_with(MINIO_ADMIN_V3_PREFIX) || path.starts_with(CONSOLE_PREFIX) || path.starts_with(RPC_PREFIX) { @@ -253,7 +314,7 @@ pub struct ConditionalCorsLayer { impl ConditionalCorsLayer { pub fn new() -> Self { - let cors_origins = std::env::var("RUSTFS_CORS_ALLOWED_ORIGINS").ok().filter(|s| !s.is_empty()); + let cors_origins = get_env_opt_str("RUSTFS_CORS_ALLOWED_ORIGINS").filter(|s| !s.is_empty()); Self { cors_origins } } @@ -263,6 +324,7 @@ impl ConditionalCorsLayer { fn is_s3_path(path: &str) -> bool { // Exclude Admin, Console, RPC, and configured special paths !path.starts_with(ADMIN_PREFIX) + && !path.starts_with(MINIO_ADMIN_PREFIX) && !path.starts_with(RPC_PREFIX) && !is_console_path(path) && !Self::EXCLUDED_EXACT_PATHS.contains(&path) @@ -501,8 +563,44 @@ where #[cfg(test)] mod tests { use super::*; + use http::Request; use http_body_util::BodyExt; use http_body_util::Full; + use temp_env::with_var; + + #[test] + fn admin_chunked_put_without_content_length_is_normalized() { + let request = Request::builder() + .method(Method::PUT) + .uri("/minio/admin/v3/set-user-status?accessKey=test&status=enabled") + .body(()) + .expect("request"); + + assert!(should_force_zero_content_length_for_admin_empty_body(&request)); + } + + #[test] + fn admin_request_with_explicit_content_length_is_left_unchanged() { + let request = Request::builder() + .method(Method::PUT) + .uri("/minio/admin/v3/set-group-status?group=test&status=enabled") + .header(http::header::CONTENT_LENGTH, "0") + .body(()) + .expect("request"); + + assert!(!should_force_zero_content_length_for_admin_empty_body(&request)); + } + + #[test] + fn non_admin_chunked_put_is_not_normalized() { + let request = Request::builder() + .method(Method::PUT) + .uri("/bucket/object") + .body(()) + .expect("request"); + + assert!(!should_force_zero_content_length_for_admin_empty_body(&request)); + } #[test] fn test_strip_quotes_from_first_etag_removes_quotes() { @@ -553,6 +651,7 @@ mod tests { assert!(ConditionalCorsLayer::is_s3_path("/my-bucket/key")); assert!(ConditionalCorsLayer::is_s3_path("/")); assert!(!ConditionalCorsLayer::is_s3_path("/rustfs/admin/v3/info")); + assert!(!ConditionalCorsLayer::is_s3_path("/minio/admin/v3/info")); assert!(!ConditionalCorsLayer::is_s3_path("/health")); assert!(!ConditionalCorsLayer::is_s3_path("/health/ready")); } @@ -594,6 +693,14 @@ mod tests { ); } + #[test] + fn test_conditional_cors_layer_reads_env() { + with_var("RUSTFS_CORS_ALLOWED_ORIGINS", Some("https://allowed.com"), || { + let cors = ConditionalCorsLayer::new(); + assert_eq!(cors.cors_origins.as_deref(), Some("https://allowed.com")); + }); + } + #[tokio::test] async fn test_resolve_s3_options_cors_headers_no_headers_without_match() { let mut req_headers = HeaderMap::new(); diff --git a/rustfs/src/server/prefix.rs b/rustfs/src/server/prefix.rs index ab48d54b9..6a61c8f97 100644 --- a/rustfs/src/server/prefix.rs +++ b/rustfs/src/server/prefix.rs @@ -36,10 +36,17 @@ pub(crate) const HEALTH_READY_PATH: &str = "/health/ready"; /// such as configuration, monitoring, and management. pub(crate) const ADMIN_PREFIX: &str = "/rustfs/admin"; +/// MinIO-compatible administrative prefix accepted by RustFS. +/// This alias allows stock MinIO admin tooling to reach RustFS handlers. +pub(crate) const MINIO_ADMIN_PREFIX: &str = "/minio/admin"; + /// Environment variable name for overriding the default /// administrative prefix path. pub(crate) const RUSTFS_ADMIN_PREFIX: &str = "/rustfs/admin/v3"; +/// MinIO-compatible admin API prefix accepted by RustFS. +pub(crate) const MINIO_ADMIN_V3_PREFIX: &str = "/minio/admin/v3"; + /// Predefined console prefix for RustFS server routes. /// This prefix is used for endpoints that handle console-related tasks /// such as user interface and management. diff --git a/rustfs/src/server/readiness.rs b/rustfs/src/server/readiness.rs index 5f5e949b5..bbc9a7be7 100644 --- a/rustfs/src/server/readiness.rs +++ b/rustfs/src/server/readiness.rs @@ -102,9 +102,11 @@ where // 2) Prefix matching: the entire set of route prefixes (including their subpaths) let is_prefix_probe = path.starts_with(crate::server::RUSTFS_ADMIN_PREFIX) + || path.starts_with(crate::server::MINIO_ADMIN_V3_PREFIX) || path.starts_with(crate::server::CONSOLE_PREFIX) || path.starts_with(crate::server::RPC_PREFIX) || path.starts_with(crate::server::ADMIN_PREFIX) + || path.starts_with(crate::server::MINIO_ADMIN_PREFIX) || path.starts_with(crate::server::TONIC_PREFIX); let is_probe = is_exact_probe || is_prefix_probe; diff --git a/rustfs/src/storage/ecfs.rs b/rustfs/src/storage/ecfs.rs index 9497fbeea..2b7e19502 100644 --- a/rustfs/src/storage/ecfs.rs +++ b/rustfs/src/storage/ecfs.rs @@ -23,30 +23,11 @@ use rustfs_ecstore::{ }; use rustfs_s3_common::{S3Operation, record_s3_op}; use s3s::{S3, S3Error, S3ErrorCode, S3Request, S3Response, S3Result, dto::*, s3_error}; -use std::{fmt::Debug, sync::LazyLock}; +use std::fmt::Debug; use tokio::io::{AsyncRead, AsyncSeek}; use tracing::{debug, error, instrument, warn}; use uuid::Uuid; -const DEFAULT_OWNER_ID: &str = "rustfsadmin"; -const DEFAULT_OWNER_DISPLAY_NAME: &str = "RustFS Tester"; - -pub(crate) static RUSTFS_OWNER: LazyLock = LazyLock::new(|| Owner { - display_name: Some(DEFAULT_OWNER_DISPLAY_NAME.to_owned()), - id: Some(DEFAULT_OWNER_ID.to_owned()), -}); - -#[derive(Clone, Debug, Default)] -pub(crate) struct StoredOwner { - pub(crate) id: String, -} - -pub(crate) fn default_owner() -> StoredOwner { - StoredOwner { - id: DEFAULT_OWNER_ID.to_string(), - } -} - #[derive(Debug, Clone)] pub struct FS { // pub store: ECStore, diff --git a/rustfs/src/storage/ecfs_extend.rs b/rustfs/src/storage/ecfs_extend.rs index 91bbcf10a..ef3be5b7e 100644 --- a/rustfs/src/storage/ecfs_extend.rs +++ b/rustfs/src/storage/ecfs_extend.rs @@ -31,7 +31,7 @@ use rustfs_targets::EventName; use rustfs_targets::arn::{TargetID, TargetIDError}; use rustfs_utils::http::{ AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER, - RESERVED_METADATA_PREFIX_LOWER, + SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, insert_str, }; use s3s::dto::{ Delimiter, LambdaFunctionConfiguration, NotificationConfigurationFilter, ObjectLockConfiguration, ObjectLockEnabled, @@ -284,8 +284,9 @@ pub(crate) fn parse_object_lock_retention(retention: Option // This is intentional behavior. Empty string represents "retention cleared" which is different from "retention never set". Consistent with minio eval_metadata.insert(AMZ_OBJECT_LOCK_MODE_LOWER.to_string(), mode); eval_metadata.insert(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER.to_string(), retain_until_date); - eval_metadata.insert( - format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, "objectlock-retention-timestamp"), + insert_str( + &mut eval_metadata, + SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, format!("{}.{:09}Z", now.format(&Rfc3339).unwrap(), now.nanosecond()), ); } @@ -310,8 +311,9 @@ pub(crate) fn parse_object_lock_legal_hold(legal_hold: Option) -> s let mut user_defined = HashMap::new(); let mut replication_request = false; - if headers.get(RUSTFS_BUCKET_REPLICATION_REQUEST) == Some(&REPLICATION_REQUEST_TRUE) { + if get_header(headers, SUFFIX_SOURCE_REPLICATION_REQUEST).as_deref() == Some("true") { replication_request = true; - if let Some(actual_size_str) = headers - .get(RUSTFS_REPLICATION_ACTUAL_OBJECT_SIZE) - .and_then(|h| h.to_str().ok()) - { - user_defined.insert(format!("{RESERVED_METADATA_PREFIX_LOWER}Actual-Object-Size"), actual_size_str.to_string()); + if let Some(actual_size_str) = get_header(headers, SUFFIX_REPLICATION_ACTUAL_OBJECT_SIZE) { + rustfs_utils::http::insert_str( + &mut user_defined, + rustfs_utils::http::SUFFIX_ACTUAL_OBJECT_SIZE_CAP, + actual_size_str.into_owned(), + ); } else { - tracing::warn!("Failed to get or parse {} header", RUSTFS_REPLICATION_ACTUAL_OBJECT_SIZE); + tracing::warn!("Failed to get or parse replication actual object size header (x-rustfs-* or x-minio-*)"); } } - if let Some(v) = headers.get(RUSTFS_BUCKET_REPLICATION_SSEC_CHECKSUM) { - user_defined.insert( - RUSTFS_BUCKET_REPLICATION_SSEC_CHECKSUM.to_string(), - v.to_str().unwrap_or_default().to_owned(), - ); + if let Some(v) = get_header(headers, SUFFIX_REPLICATION_SSEC_CRC) { + insert_header_map(&mut user_defined, SUFFIX_REPLICATION_SSEC_CRC, v.into_owned()); } let mut opts = ObjectOptions { @@ -299,26 +288,14 @@ pub fn copy_src_opts(_bucket: &str, _object: &str, headers: &HeaderMap, metadata: HashMap) -> Result { let mut opts = get_default_opts(headers, metadata, false)?; - if headers.get(RUSTFS_BUCKET_REPLICATION_REQUEST) == Some(&REPLICATION_REQUEST_TRUE) { + if get_header(headers, SUFFIX_SOURCE_REPLICATION_REQUEST).as_deref() == Some("true") { opts.replication_request = true; - if let Some(v) = headers.get(RUSTFS_BUCKET_SOURCE_MTIME) { - match v.to_str() { - Ok(s) => { - let trimmed_s = s.trim(); - match time::OffsetDateTime::parse(trimmed_s, &time::format_description::well_known::Rfc3339) { - Ok(mtime) => opts.mod_time = Some(mtime), - Err(e) => { - tracing::warn!( - "Invalid X-RustFS-Source-Mtime value '{}' (replication request=true): {}", - trimmed_s, - e - ); - opts.mod_time = None; - } - } - } + if let Some(v) = get_header(headers, SUFFIX_SOURCE_MTIME) { + let trimmed_s = v.trim(); + match time::OffsetDateTime::parse(trimmed_s, &time::format_description::well_known::Rfc3339) { + Ok(mtime) => opts.mod_time = Some(mtime), Err(e) => { - tracing::warn!("X-RustFS-Source-Mtime header is not valid UTF-8 (replication request=true): {}", e); + tracing::warn!("Invalid source-mtime value '{}' (replication request=true): {}", trimmed_s, e); opts.mod_time = None; } } @@ -379,12 +356,17 @@ pub fn extract_metadata_from_mime_with_object_name( skip_content_type: bool, object_name: Option<&str>, ) { + const USER_METADATA_PREFIXES: &[&str] = &["x-amz-meta-", "x-rustfs-meta-", "x-minio-meta-"]; + for (k, v) in headers.iter() { if k.as_str() == "content-type" && skip_content_type { continue; } - if let Some(key) = k.as_str().strip_prefix("x-amz-meta-") { + if let Some(key) = USER_METADATA_PREFIXES + .iter() + .find_map(|prefix| k.as_str().strip_prefix(prefix)) + { if key.is_empty() { continue; } @@ -393,11 +375,6 @@ pub fn extract_metadata_from_mime_with_object_name( continue; } - if let Some(key) = k.as_str().strip_prefix("x-rustfs-meta-") { - metadata.insert(key.to_owned(), String::from_utf8_lossy(v.as_bytes()).to_string()); - continue; - } - for hd in SUPPORTED_HEADERS.iter() { if k.as_str() == *hd { let raw = String::from_utf8_lossy(v.as_bytes()).to_string(); @@ -446,13 +423,13 @@ pub(crate) fn filter_object_metadata(metadata: &HashMap) -> Opti let mut filtered_metadata = HashMap::new(); for (k, v) in metadata { let lower_key = k.to_ascii_lowercase(); - // Skip internal/reserved metadata - if lower_key.starts_with(RESERVED_METADATA_PREFIX_LOWER) { + // Skip internal/reserved metadata (x-rustfs-internal-* or x-minio-internal-*) + if is_internal_key(&lower_key) { continue; } - // Skip internal encryption metadata - if lower_key.starts_with(RUSTFS_ENCRYPTION_LOWER) { + // Skip internal encryption metadata (x-rustfs-encryption-* or x-minio-encryption-*) + if is_encryption_metadata_key(&lower_key) { continue; } @@ -804,28 +781,28 @@ mod tests { let mut headers = create_test_headers(); let metadata = create_test_metadata(); - // Test without RUSTFS_FORCE_DELETE header - should default to false + // Test without force-delete header - should default to false let result = del_opts("test-bucket", "test-object", None, &headers, metadata.clone()).await; assert!(result.is_ok()); let opts = result.unwrap(); assert!(!opts.delete_prefix); // Test with RUSTFS_FORCE_DELETE header set to "true" - headers.insert(RUSTFS_FORCE_DELETE, HeaderValue::from_static("true")); + insert_header(&mut headers, SUFFIX_FORCE_DELETE, "true"); let result = del_opts("test-bucket", "test-object", None, &headers, metadata.clone()).await; assert!(result.is_ok()); let opts = result.unwrap(); assert!(opts.delete_prefix); // Test with RUSTFS_FORCE_DELETE header set to "false" - headers.insert(RUSTFS_FORCE_DELETE, HeaderValue::from_static("false")); + insert_header(&mut headers, SUFFIX_FORCE_DELETE, "false"); let result = del_opts("test-bucket", "test-object", None, &headers, metadata.clone()).await; assert!(result.is_ok()); let opts = result.unwrap(); assert!(!opts.delete_prefix); // Test with RUSTFS_FORCE_DELETE header set to other value - headers.insert(RUSTFS_FORCE_DELETE, HeaderValue::from_static("maybe")); + insert_header(&mut headers, SUFFIX_FORCE_DELETE, "maybe"); let result = del_opts("test-bucket", "test-object", None, &headers, metadata).await; assert!(result.is_ok()); let opts = result.unwrap(); @@ -994,9 +971,9 @@ mod tests { #[test] fn test_put_opts_from_headers_with_replication_request() { let mut headers = HeaderMap::new(); - headers.insert(RUSTFS_BUCKET_REPLICATION_REQUEST, REPLICATION_REQUEST_TRUE.clone()); + insert_header(&mut headers, SUFFIX_SOURCE_REPLICATION_REQUEST, "true"); let valid_mtime = "2024-05-20T10:30:00+08:00"; - headers.insert(RUSTFS_BUCKET_SOURCE_MTIME, HeaderValue::from_static(valid_mtime)); + insert_header(&mut headers, SUFFIX_SOURCE_MTIME, valid_mtime); let metadata = HashMap::new(); @@ -1011,8 +988,8 @@ mod tests { assert_eq!(opts.mod_time, Some(expected_mtime)); let mut headers_invalid_mtime = HeaderMap::new(); - headers_invalid_mtime.insert(RUSTFS_BUCKET_REPLICATION_REQUEST, REPLICATION_REQUEST_TRUE.clone()); - headers_invalid_mtime.insert(RUSTFS_BUCKET_SOURCE_MTIME, HeaderValue::from_static("invalid-time")); + insert_header(&mut headers_invalid_mtime, SUFFIX_SOURCE_REPLICATION_REQUEST, "true"); + insert_header(&mut headers_invalid_mtime, SUFFIX_SOURCE_MTIME, "invalid-time"); let result_invalid = put_opts_from_headers(&headers_invalid_mtime, HashMap::new()); assert!(result_invalid.is_ok()); let opts_invalid = result_invalid.unwrap(); @@ -1090,6 +1067,21 @@ mod tests { assert_eq!(metadata.get("category"), Some(&"documents".to_string())); } + #[test] + fn test_extract_metadata_from_mime_minio_meta() { + let mut headers = HeaderMap::new(); + headers.insert("x-minio-meta-origin", HeaderValue::from_static("gateway")); + headers.insert("x-minio-meta-source-id", HeaderValue::from_static("abc123")); + headers.insert("x-minio-meta-", HeaderValue::from_static("empty-key")); + + let mut metadata = HashMap::new(); + extract_metadata_from_mime(&headers, &mut metadata); + + assert_eq!(metadata.get("origin"), Some(&"gateway".to_string())); + assert_eq!(metadata.get("source-id"), Some(&"abc123".to_string())); + assert!(!metadata.contains_key("")); + } + #[test] fn test_extract_metadata_from_mime_supported_headers() { let mut headers = HeaderMap::new(); @@ -1249,6 +1241,7 @@ mod tests { headers.insert("content-type", HeaderValue::from_static("application/xml")); headers.insert("x-amz-meta-version", HeaderValue::from_static("1.0")); headers.insert("x-rustfs-meta-source", HeaderValue::from_static("upload")); + headers.insert("x-minio-meta-origin", HeaderValue::from_static("replication")); headers.insert("cache-control", HeaderValue::from_static("public")); headers.insert("authorization", HeaderValue::from_static("Bearer xyz")); // Should be ignored @@ -1257,6 +1250,7 @@ mod tests { assert_eq!(metadata.get("content-type"), Some(&"application/xml".to_string())); assert_eq!(metadata.get("version"), Some(&"1.0".to_string())); assert_eq!(metadata.get("source"), Some(&"upload".to_string())); + assert_eq!(metadata.get("origin"), Some(&"replication".to_string())); assert_eq!(metadata.get("cache-control"), Some(&"public".to_string())); assert!(!metadata.contains_key("authorization")); } diff --git a/rustfs/src/storage/s3_api/bucket.rs b/rustfs/src/storage/s3_api/bucket.rs index ceed9ef65..cfec55025 100644 --- a/rustfs/src/storage/s3_api/bucket.rs +++ b/rustfs/src/storage/s3_api/bucket.rs @@ -17,7 +17,7 @@ use rustfs_ecstore::client::object_api_utils::to_s3s_etag; use rustfs_ecstore::store_api::{BucketInfo, ListObjectVersionsInfo, ListObjectsV2Info}; use s3s::dto::{ Bucket, CommonPrefix, DeleteMarkerEntry, EncodingType, ListBucketsOutput, ListObjectVersionsOutput, ListObjectsOutput, - ListObjectsV2Output, Object, ObjectStorageClass, ObjectVersion, ObjectVersionStorageClass, Owner, Timestamp, + ListObjectsV2Output, Object, ObjectStorageClass, ObjectVersion, ObjectVersionStorageClass, Timestamp, }; use s3s::{S3Error, S3ErrorCode}; use tracing::debug; @@ -265,10 +265,7 @@ pub(crate) fn build_list_objects_v2_output( }; if fetch_owner { - obj.owner = Some(Owner { - display_name: Some("rustfs".to_owned()), - id: Some("v0.1".to_owned()), - }); + obj.owner = Some(rustfs_owner()); } obj @@ -482,10 +479,16 @@ mod tests { let contents = output.contents.as_ref().expect("contents should exist"); let common_prefixes = output.common_prefixes.as_ref().expect("common prefixes should exist"); + let expected_owner = rustfs_owner(); assert_eq!(contents[0].key.as_deref(), Some("dir%20a/file%2Bb%25.txt")); assert_eq!(common_prefixes[0].prefix.as_deref(), Some("prefix%20a/sub%2B")); assert!(contents[0].owner.is_some()); + assert_eq!( + contents[0].owner.as_ref().and_then(|owner| owner.display_name.clone()), + expected_owner.display_name + ); + assert_eq!(contents[0].owner.as_ref().and_then(|owner| owner.id.clone()), expected_owner.id); assert_eq!(output.encoding_type.as_ref().map(EncodingType::as_str), Some(EncodingType::URL)); } diff --git a/rustfs/src/storage/s3_api/multipart.rs b/rustfs/src/storage/s3_api/multipart.rs index 6dabf0a70..4f35de70f 100644 --- a/rustfs/src/storage/s3_api/multipart.rs +++ b/rustfs/src/storage/s3_api/multipart.rs @@ -184,6 +184,7 @@ mod tests { build_list_multipart_uploads_output, build_list_parts_output, parse_list_multipart_uploads_params, parse_list_parts_params, }; + use crate::storage::s3_api::common::{rustfs_initiator, rustfs_owner}; use rustfs_ecstore::client::object_api_utils::to_s3s_etag; use rustfs_ecstore::set_disk::MAX_PARTS_COUNT; use rustfs_ecstore::store_api::{ListMultipartsInfo, ListPartsInfo, MultipartInfo, PartInfo}; @@ -226,8 +227,8 @@ mod tests { assert_eq!(parts[0].part_number, Some(1)); assert_eq!(parts[0].size, Some(11)); assert_eq!(parts[0].e_tag, Some(to_s3s_etag("etag-1"))); - assert!(output.owner.is_some()); - assert!(output.initiator.is_some()); + assert_eq!(output.owner, Some(rustfs_owner())); + assert_eq!(output.initiator, Some(rustfs_initiator())); } #[test] diff --git a/rustfs/src/storage/sse.rs b/rustfs/src/storage/sse.rs index 3215cba26..1a0ce86ce 100644 --- a/rustfs/src/storage/sse.rs +++ b/rustfs/src/storage/sse.rs @@ -88,6 +88,7 @@ use rustfs_kms::{ types::{EncryptionMetadata, ObjectEncryptionContext}, }; use rustfs_rio::{DecryptReader, EncryptReader, HardLimitReader, Reader, WarpReader}; +use rustfs_utils::get_env_opt_str; use s3s::S3ErrorCode; use s3s::dto::ServerSideEncryption; use std::collections::HashMap; @@ -1397,8 +1398,8 @@ impl TestSseDekProvider { /// Uses RUSTFS_SSE_S3_MASTER_KEY (base64 32-byte) if set; otherwise a built-in default. /// Allows PUT/GET to work without KMS (backward compatible). pub fn new_for_local_sse() -> Self { - let master_key = match std::env::var("RUSTFS_SSE_S3_MASTER_KEY") { - Ok(v) if !v.trim().is_empty() => match BASE64_STANDARD.decode(v.trim()) { + let master_key = match get_env_opt_str("RUSTFS_SSE_S3_MASTER_KEY") { + Some(v) if !v.trim().is_empty() => match BASE64_STANDARD.decode(v.trim()) { Ok(decoded) if decoded.len() == 32 => { let mut arr = [0u8; 32]; arr.copy_from_slice(&decoded[..32]); diff --git a/scripts/dev_rustfs.env b/scripts/dev_rustfs.env index 59ee523d5..5af66fa3b 100644 --- a/scripts/dev_rustfs.env +++ b/scripts/dev_rustfs.env @@ -5,7 +5,7 @@ RUSTFS_VOLUMES="http://node{1...4}:7000/data/rustfs{0...3} http://node{5...8 RUSTFS_ADDRESS=":7000" RUSTFS_CONSOLE_ENABLE=true RUST_LOG=warn -RUSTFS_OBS_LOG_DIRECTORY="/var/logs/rustfs/" +RUSTFS_OBS_LOG_DIRECTORY="./deploy/logs" RUSTFS_NS_SCANNER_INTERVAL=60 #RUSTFS_SKIP_BACKGROUND_TASK=true -RUSTFS_COMPRESSION_ENABLED=true \ No newline at end of file +RUSTFS_COMPRESSION_ENABLED=true diff --git a/scripts/s3-tests/run.sh b/scripts/s3-tests/run.sh index 29c414394..ced0ecbb6 100755 --- a/scripts/s3-tests/run.sh +++ b/scripts/s3-tests/run.sh @@ -18,7 +18,11 @@ set -euo pipefail # Disable proxy for localhost requests to avoid interference export NO_PROXY="127.0.0.1,localhost,::1" export no_proxy="${NO_PROXY}" -unset http_proxy https_proxy HTTP_PROXY HTTPS_PROXY +# Keep proxy variables for external downloads; NO_PROXY bypasses localhost. + +# Ensure user-level Python scripts are discoverable (awscurl/tox installed via --user) +PYTHON_VERSION=$(python3 -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')" 2>/dev/null || echo "3.14") +export PATH="$HOME/Library/Python/${PYTHON_VERSION}/bin:$HOME/.local/bin:$PATH" # Configuration S3_ACCESS_KEY="${S3_ACCESS_KEY:-rustfsadmin}" diff --git a/scripts/test/decommission_docker.md b/scripts/test/decommission_docker.md new file mode 100644 index 000000000..6e7f2d972 --- /dev/null +++ b/scripts/test/decommission_docker.md @@ -0,0 +1,49 @@ +# Local Docker Decommission Test + +This setup simulates a two-pool RustFS cluster locally with Docker so decommission can be tested without relying on a remote environment. + +## Topology + +- One RustFS container +- Two pools +- Four disks per pool +- S3 API on `http://127.0.0.1:9100` +- Console on `http://127.0.0.1:9101` + +Pool cmdlines used by admin decommission: + +- `/data/pool0/disk{1...4}` +- `/data/pool1/disk{1...4}` + +## Quick Start + +1. Start the local cluster: + `./scripts/test/decommission_docker.sh up` +2. Run the end-to-end smoke flow: + `./scripts/test/decommission_docker.sh smoke` + +## Manual Flow + +1. Start the cluster: + `./scripts/test/decommission_docker.sh up` +2. Prepare test data: + `./scripts/test/decommission_validation.sh prepare rustfs-decom` +3. Start decommission on pool 1: + `./scripts/test/decommission_validation.sh start rustfs-decom '/data/pool1/disk{1...4}'` +4. Wait for completion: + `./scripts/test/decommission_validation.sh wait rustfs-decom '/data/pool1/disk{1...4}' 900` +5. Verify objects and versions: + `./scripts/test/decommission_validation.sh verify rustfs-decom` + +## Cleanup + +- Stop containers: + `./scripts/test/decommission_docker.sh down` +- Remove containers, volumes, generated data, and saved validation state: + `./scripts/test/decommission_docker.sh reset` + +## Notes + +- This simulates pools, not separate physical nodes. +- It is good for validating decommission control flow and object migration behavior. +- It is not a substitute for a real distributed failure-injection test. diff --git a/scripts/test/decommission_docker.sh b/scripts/test/decommission_docker.sh new file mode 100755 index 000000000..b8b205421 --- /dev/null +++ b/scripts/test/decommission_docker.sh @@ -0,0 +1,173 @@ +#!/usr/bin/env bash +set -euo pipefail + +COMPOSE_FILE="${COMPOSE_FILE:-docker-compose.decommission.yml}" +SERVICE_NAME="${SERVICE_NAME:-rustfs-decommission-latest}" +MC_ALIAS_NAME="${MC_ALIAS_NAME:-rustfs-decom}" +S3_ENDPOINT="${S3_ENDPOINT:-http://127.0.0.1:9100}" +ACCESS_KEY="${ACCESS_KEY:-rustfsadmin}" +SECRET_KEY="${SECRET_KEY:-rustfsadmin}" +DOCKERFILE_PATH="${DOCKERFILE_PATH:-Dockerfile.decommission-local}" +DOCKER_IMAGE_NAME="${DOCKER_IMAGE_NAME:-rustfs-local:decommission-latest}" +POOL0_CMDLINE="${POOL0_CMDLINE:-/data/pool0/disk{1...4}}" +POOL1_CMDLINE="${POOL1_CMDLINE:-/data/pool1/disk{1...4}}" +HEALTH_TIMEOUT_SECONDS="${HEALTH_TIMEOUT_SECONDS:-1200}" + +usage() { + cat </dev/null 2>&1; then + echo "missing required binary: $1" >&2 + exit 1 + fi +} + +compose() { + docker compose -f "${COMPOSE_FILE}" "$@" +} + +wait_healthy() { + local elapsed=0 + local container_id + container_id="$(compose ps -q "${SERVICE_NAME}")" + if [[ -z "${container_id}" ]]; then + echo "container not found for service ${SERVICE_NAME}" >&2 + exit 1 + fi + + while (( elapsed < HEALTH_TIMEOUT_SECONDS )); do + local status + status="$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "${container_id}")" + if [[ "${status}" == "healthy" || "${status}" == "running" ]]; then + echo "container is ${status}" + return + fi + sleep 5 + elapsed=$((elapsed + 5)) + done + + echo "container did not become healthy within ${HEALTH_TIMEOUT_SECONDS}s" >&2 + compose logs --tail=200 "${SERVICE_NAME}" || true + exit 1 +} + +configure_alias() { + require_bin mc + mc alias set "${MC_ALIAS_NAME}" "${S3_ENDPOINT}" "${ACCESS_KEY}" "${SECRET_KEY}" >/dev/null + echo "mc alias configured: ${MC_ALIAS_NAME} -> ${S3_ENDPOINT}" +} + +up() { + require_bin docker + require_bin mc + mkdir -p deploy/data/decommission deploy/logs/decommission + compose up -d + wait_healthy + configure_alias + info +} + +build_image() { + require_bin docker + docker build -f "${DOCKERFILE_PATH}" -t "${DOCKER_IMAGE_NAME}" . +} + +down() { + require_bin docker + compose down +} + +reset() { + require_bin docker + compose down -v --remove-orphans || true + rm -rf deploy/data/decommission deploy/logs/decommission .tmp/decommission-validation + echo "local decommission docker environment reset" +} + +info() { + cat <` +2. Start decommission: + `./scripts/test/decommission_validation.sh start ''` +3. Wait for completion: + `./scripts/test/decommission_validation.sh wait '' 900` +4. Verify readable data and version listings: + `./scripts/test/decommission_validation.sh verify ` + +## Optional Cancel Check + +1. Start decommission on a disposable pool. +2. Run: + `./scripts/test/decommission_validation.sh cancel ''` +3. Confirm status shows `canceled=true` and not `complete=true`. + +## Manual Extension For Tiered Objects + +If the cluster already has a configured remote tier and ILM transition rule: + +1. Upload an object that matches the transition rule. +2. Wait until the object is transitioned remotely. +3. Run the decommission flow above. +4. Confirm: + - the object is still readable after decommission + - decommission completes without `failed=true` + - no residual object/version remains on the source pool diff --git a/scripts/test/decommission_validation.sh b/scripts/test/decommission_validation.sh new file mode 100755 index 000000000..086c7e1e1 --- /dev/null +++ b/scripts/test/decommission_validation.sh @@ -0,0 +1,434 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + cat <<'EOF' +Usage: + decommission_validation.sh prepare + decommission_validation.sh start + decommission_validation.sh status + decommission_validation.sh wait [timeout-seconds] + decommission_validation.sh verify + decommission_validation.sh cancel [pool-cmdline] + +Environment: + DECOM_STATE_DIR Directory for generated state files. Default: .tmp/decommission-validation + DECOM_ADMIN_API Admin API mode: auto, mc, rustfs. Default: auto + +Examples: + ./scripts/test/decommission_validation.sh prepare rustfs + ./scripts/test/decommission_validation.sh start rustfs 'http://server{5...8}/disk{1...4}' + ./scripts/test/decommission_validation.sh wait rustfs 'http://server{5...8}/disk{1...4}' 900 + ./scripts/test/decommission_validation.sh verify rustfs +EOF +} + +require_bin() { + if ! command -v "$1" >/dev/null 2>&1; then + echo "missing required binary: $1" >&2 + exit 1 + fi +} + +state_dir() { + printf '%s\n' "${DECOM_STATE_DIR:-.tmp/decommission-validation}" +} + +state_file() { + local alias_name="$1" + printf '%s/%s.env\n' "$(state_dir)" "$alias_name" +} + +load_state() { + local alias_name="$1" + local file + file="$(state_file "$alias_name")" + if [[ ! -f "$file" ]]; then + echo "state file not found: $file" >&2 + exit 1 + fi + # shellcheck disable=SC1090 + source "$file" +} + +admin_alias() { + printf '%s\n' "$1" +} + +admin_api_mode() { + printf '%s\n' "${DECOM_ADMIN_API:-auto}" +} + +mc_alias_field() { + local alias_name="$1" + local field="$2" + local alias_json + + alias_json="$(mc alias list --json 2>/dev/null | grep -F "\"alias\":\"${alias_name}\"" | tail -n 1 || true)" + if [[ -z "$alias_json" ]]; then + return 1 + fi + + if command -v jq >/dev/null 2>&1; then + printf '%s\n' "$alias_json" | jq -r ".${field} // empty" + return + fi + + printf '%s\n' "$alias_json" | sed -n "s/.*\"${field}\":\"\\([^\"]*\\)\".*/\\1/p" +} + +urlencode() { + local raw="$1" + if command -v jq >/dev/null 2>&1; then + jq -rn --arg v "$raw" '$v|@uri' + return + fi + + python3 -c 'import sys, urllib.parse; print(urllib.parse.quote(sys.argv[1], safe=""))' "$raw" +} + +rustfs_admin_base() { + local alias_name="$1" + local url + url="$(mc_alias_field "$alias_name" "URL")" + if [[ -z "$url" ]]; then + echo "unable to resolve mc alias URL for ${alias_name}" >&2 + exit 1 + fi + printf '%s/rustfs/admin/v3' "${url%/}" +} + +rustfs_admin_request() { + local alias_name="$1" + local method="$2" + local path="$3" + local access_key secret_key + + access_key="$(mc_alias_field "$alias_name" "accessKey")" + secret_key="$(mc_alias_field "$alias_name" "secretKey")" + + if [[ -z "$access_key" || -z "$secret_key" ]]; then + echo "unable to resolve mc alias credentials for ${alias_name}" >&2 + exit 1 + fi + + awscurl \ + --fail-with-body \ + --service s3 \ + --region us-east-1 \ + --access_key "$access_key" \ + --secret_key "$secret_key" \ + -X "$method" \ + "$(rustfs_admin_base "$alias_name")${path}" +} + +detect_admin_api() { + local alias_name="$1" + local requested + requested="$(admin_api_mode)" + + case "$requested" in + mc|rustfs) + printf '%s\n' "$requested" + return + ;; + auto) + ;; + *) + echo "unsupported DECOM_ADMIN_API mode: ${requested}" >&2 + exit 1 + ;; + esac + + if ! command -v awscurl >/dev/null 2>&1; then + printf 'mc\n' + return + fi + + if rustfs_admin_request "$alias_name" GET "/pools/list" >/dev/null 2>&1; then + printf 'rustfs\n' + else + printf 'mc\n' + fi +} + +write_state() { + local alias_name="$1" + local run_id="$2" + local basic_bucket="$3" + local lock_bucket="$4" + local state_path + state_path="$(state_file "$alias_name")" + mkdir -p "$(dirname "$state_path")" + cat >"$state_path" <"${workdir}/single.txt" + printf 'tombstone\n' >"${workdir}/delete-marker.txt" + dd if=/dev/zero of="${workdir}/multipart.bin" bs=1M count=20 status=none + + mc mb "${alias_name}/${basic_bucket}" + mc version enable "${alias_name}/${basic_bucket}" + mc cp "${workdir}/single.txt" "${alias_name}/${basic_bucket}/single.txt" + + printf 'alpha-v2\n' >"${workdir}/single.txt" + mc cp "${workdir}/single.txt" "${alias_name}/${basic_bucket}/single.txt" + mc cp "${workdir}/multipart.bin" "${alias_name}/${basic_bucket}/multipart.bin" + mc cp "${workdir}/delete-marker.txt" "${alias_name}/${basic_bucket}/delete-marker.txt" + mc rm "${alias_name}/${basic_bucket}/delete-marker.txt" + + mc mb --with-lock "${alias_name}/${lock_bucket}" + mc retention set --default GOVERNANCE "1d" "${alias_name}/${lock_bucket}" + mc cp "${workdir}/single.txt" "${alias_name}/${lock_bucket}/locked.txt" + + write_state "$alias_name" "$run_id" "$basic_bucket" "$lock_bucket" + + cat <' + ./scripts/test/decommission_validation.sh wait ${alias_name} '' 900 + ./scripts/test/decommission_validation.sh verify ${alias_name} +EOF +} + +start_decommission() { + local alias_name="$1" + local pool_cmdline="$2" + local mode + mode="$(detect_admin_api "$alias_name")" + + case "$mode" in + rustfs) + require_bin awscurl + rustfs_admin_request "$alias_name" POST "/pools/decommission?pool=${pool_cmdline}" + ;; + mc) + require_bin mc + mc admin decommission start "$(admin_alias "$alias_name")" "$pool_cmdline" + ;; + esac +} + +status_decommission() { + local alias_name="$1" + local pool_cmdline="$2" + local mode + mode="$(detect_admin_api "$alias_name")" + + case "$mode" in + rustfs) + require_bin awscurl + rustfs_admin_request "$alias_name" GET "/pools/status?pool=${pool_cmdline}" + ;; + mc) + require_bin mc + mc admin decommission status "$(admin_alias "$alias_name")" "$pool_cmdline" + ;; + esac +} + +parse_status_field() { + local json_input="$1" + local field="$2" + if command -v jq >/dev/null 2>&1; then + printf '%s\n' "$json_input" | jq -r ".. | .${field}? // empty" 2>/dev/null | tail -n 1 + else + printf '%s\n' "$json_input" | grep -Eo "\"${field}\"[[:space:]]*:[[:space:]]*(true|false)" | tail -n 1 | awk -F: '{gsub(/[[:space:]]/, "", $2); print $2}' + fi +} + +wait_decommission() { + local alias_name="$1" + local pool_cmdline="$2" + local timeout_seconds="${3:-900}" + local elapsed=0 + local interval=5 + local mode + + mode="$(detect_admin_api "$alias_name")" + + while (( elapsed < timeout_seconds )); do + local json_output + case "$mode" in + rustfs) + json_output="$(rustfs_admin_request "$alias_name" GET "/pools/status?pool=${pool_cmdline}" 2>/dev/null || true)" + ;; + mc) + require_bin mc + json_output="$(mc admin decommission status --json "$(admin_alias "$alias_name")" "$pool_cmdline" 2>/dev/null || true)" + ;; + esac + + if [[ -n "$json_output" ]]; then + printf '%s\n' "$json_output" + else + status_decommission "$alias_name" "$pool_cmdline" || true + fi + + local complete failed canceled + complete="$(parse_status_field "$json_output" "complete")" + failed="$(parse_status_field "$json_output" "failed")" + canceled="$(parse_status_field "$json_output" "canceled")" + + if [[ "$failed" == "true" ]]; then + echo "decommission finished with failed=true" >&2 + exit 1 + fi + if [[ "$canceled" == "true" ]]; then + echo "decommission finished with canceled=true" >&2 + exit 1 + fi + if [[ "$complete" == "true" ]]; then + echo "decommission completed successfully" + return + fi + + sleep "$interval" + elapsed=$((elapsed + interval)) + done + + echo "timed out waiting for decommission completion after ${timeout_seconds}s" >&2 + exit 1 +} + +verify() { + local alias_name="$1" + require_bin mc + + load_state "$alias_name" + + mc stat "${alias_name}/${BASIC_BUCKET}/single.txt" + mc stat "${alias_name}/${BASIC_BUCKET}/multipart.bin" + mc stat "${alias_name}/${LOCK_BUCKET}/locked.txt" + + echo "version listing for ${BASIC_BUCKET}:" + mc ls --versions "${alias_name}/${BASIC_BUCKET}" + + echo + echo "object lock bucket listing for ${LOCK_BUCKET}:" + mc ls --versions "${alias_name}/${LOCK_BUCKET}" + + cat <}' shows complete=true and failed=false +EOF +} + +cancel_decommission() { + local alias_name="$1" + local pool_cmdline="${2:-}" + local mode + mode="$(detect_admin_api "$alias_name")" + + case "$mode" in + rustfs) + require_bin awscurl + if [[ -n "$pool_cmdline" ]]; then + rustfs_admin_request "$alias_name" POST "/pools/cancel?pool=${pool_cmdline}" + else + rustfs_admin_request "$alias_name" POST "/pools/cancel" + fi + ;; + mc) + require_bin mc + if [[ -n "$pool_cmdline" ]]; then + mc admin decommission cancel "$(admin_alias "$alias_name")" "$pool_cmdline" + else + mc admin decommission cancel "$(admin_alias "$alias_name")" + fi + ;; + esac +} + +main() { + if [[ $# -lt 2 ]]; then + usage + exit 1 + fi + + local cmd="$1" + shift + + case "$cmd" in + prepare) + if [[ $# -ne 1 ]]; then + usage + exit 1 + fi + prepare "$1" + ;; + start) + if [[ $# -ne 2 ]]; then + usage + exit 1 + fi + start_decommission "$1" "$2" + ;; + status) + if [[ $# -ne 2 ]]; then + usage + exit 1 + fi + status_decommission "$1" "$2" + ;; + wait) + if [[ $# -lt 2 || $# -gt 3 ]]; then + usage + exit 1 + fi + wait_decommission "$1" "$2" "${3:-900}" + ;; + verify) + if [[ $# -ne 1 ]]; then + usage + exit 1 + fi + verify "$1" + ;; + cancel) + if [[ $# -lt 1 || $# -gt 2 ]]; then + usage + exit 1 + fi + cancel_decommission "$1" "${2:-}" + ;; + *) + usage + exit 1 + ;; + esac +} + +main "$@"