mirror of
https://github.com/rustfs/rustfs.git
synced 2026-11-16 04:52:36 +00:00
Compare commits
41 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| f4266a4576 | |||
| 00ecde1250 | |||
| e9da1d3096 | |||
| 2308dee4b2 | |||
| b2b4f0c491 | |||
| c5b4a85778 | |||
| 4ae180ea40 | |||
| c0763a43f5 | |||
| d461908bf0 | |||
| 5b8876ad0e | |||
| 75b293f094 | |||
| fb3e2818c7 | |||
| 4294c41ccf | |||
| 3cb0aa137e | |||
| a8cf62f8f3 | |||
| d0fd6425e4 | |||
| 92050cdca6 | |||
| 26db9b19af | |||
| 292974baff | |||
| 993464fbbb | |||
| cfdd4b2686 | |||
| 6f93ac5f47 | |||
| 620fdc286a | |||
| 58310c569b | |||
| e36c7452ac | |||
| 48e97253b5 | |||
| ce5bd8b117 | |||
| 93947f22ce | |||
| 8f361e3a6f | |||
| 3cee9c154b | |||
| 504cc06d0c | |||
| 44145ef327 | |||
| ba9e8342f8 | |||
| a41019802d | |||
| 4313812e24 | |||
| 5c58a56c4c | |||
| 8196603a76 | |||
| 1e62cbaeaf | |||
| da1817ae01 | |||
| c2b5fbb138 | |||
| 97a7639f59 |
@@ -31,3 +31,14 @@
|
||||
# the flag, so the two can no longer drift apart unnoticed.
|
||||
#
|
||||
# For CPU profiling, add `-C force-frame-pointers=yes` to that RUSTFLAGS value.
|
||||
|
||||
# Exact internode HTTP/2 PING ACK timeout accounting is also deliberately
|
||||
# default-off. Hyper exposes the exact timeout only through its unstable tracing
|
||||
# event, so diagnostic builds must opt in to both the feature and cfg:
|
||||
#
|
||||
# RUSTFLAGS="--cfg hyper_unstable_tracing" \
|
||||
# cargo build -p rustfs --features diagnostic-internode-http2-ping
|
||||
#
|
||||
# `crates/protos/build.rs` and `crates/obs/build.rs` fail fast if the feature and
|
||||
# cfg drift apart. Do not move this cfg into global rustflags: doing so would
|
||||
# recompile every dependency under an unstable, non-semver build contract.
|
||||
|
||||
@@ -1,2 +1,2 @@
|
||||
sha256-darwin=8fef326a3fddd8a548e2cbcc6ea8159f831113595fb003eeebbd9aeb4ac23e48
|
||||
sha256-linux=8b4381578377704ac8b1b5156128b83f036ec4aedc9d863fff25df15112420fc
|
||||
sha256-darwin=24a8cfaa1b9125bd69d7d4a7ff752a8ab8ed350c761d90ae8472ef52393e2686
|
||||
sha256-linux=014257197fbbfbb48cab2864da9e10d96ee877d478fa29d119f8186532ea293d
|
||||
|
||||
@@ -1 +1 @@
|
||||
sha256=061d0a697967d2dc8f472ff885886e069b1cf4aeaa5f230e177b7236017c23b6
|
||||
sha256=fd5683273bba1eae3b4402a4e854a5d168ace845bd0e5fc801f8e336485a4394
|
||||
|
||||
@@ -39,6 +39,8 @@ script-tests: ## Run shell script tests
|
||||
./scripts/test_entrypoint_directories.sh
|
||||
./scripts/test_internode_grpc_ab_bench.sh
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_internode_grpc_window.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_internode_msgpack_peer_canary.py
|
||||
./scripts/test_internode_msgpack_peer_canary_runner.sh
|
||||
./scripts/test_object_batch_bench_enhanced.sh
|
||||
./scripts/test_package_service_scripts.sh
|
||||
./scripts/test_hotpath_warp_ab_gate.sh
|
||||
|
||||
@@ -141,8 +141,9 @@ setup = 'ecstore-large-stack'
|
||||
filter = '(package(rustfs) & test(/^(admin::handlers::replication::target_repair_tests::[a-z_]+|admin::handlers::plugin_attempt::v2_tests::delegation_v2_persists_and_vends_with_real_parent_intersection|admin::handlers::site_replication::tests::durable_iam_state_apply_iam_item_rejects_a_stale_recreate_after_a_replicated_delete|admin::service::federated_identity::tests::(binding_keeps_explicit_iam_mapping_strict|markerless_legacy_oidc_session_crosses_auth_and_iam_deletion_boundaries)|storage::scanner_dirty_journal::tests::(clear_callback_keeps_unknown_coverage_until_all_pending_work_is_confirmed|owned_journals_keep_peer_records_after_interleaved_writes_and_clear))$/)) | (package(rustfs-heal) & ((binary(heal_b5_versioned_regression_test) & test(/^serial_tests::(test_heal_unversioned_bucket_e2e|test_historical_null_recursive_heal_restores_physical_versions|test_recursive_deep_heal_reports_stale_delete_marker_repair)$/)) | (binary(usage_observation_read_repair_test) & test(=usage_observation_read_repair_survives_snapshot_cleanup)))) | (package(rustfs-test-utils) & test(=data_usage_snapshot_tests::data_usage_snapshot_recovery_wires_primary_backup_and_negative_cache))'
|
||||
setup = 'ecstore-base-stack'
|
||||
|
||||
# The multipart refusal fixture composes real-store setup, abort, quota and legal-hold futures.
|
||||
[[profile.default.scripts]]
|
||||
filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::complete::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|app::object::restore::tests::execute_restore_object_maps_failures_to_typed_s3_errors|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))'
|
||||
filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::complete::tests::completion_absent_binding_refusal_order_baseline|app::multipart_usecase::complete::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|app::object::restore::tests::execute_restore_object_maps_failures_to_typed_s3_errors|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))'
|
||||
setup = 'ecstore-base-stack'
|
||||
|
||||
# These handler/store fixtures and the relocated-pool GET compose deep storage
|
||||
@@ -465,7 +466,7 @@ filter = '(package(rustfs) & test(/^(admin::handlers::replication::target_repair
|
||||
setup = 'ecstore-base-stack'
|
||||
|
||||
[[profile.ci.scripts]]
|
||||
filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::complete::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|app::object::restore::tests::execute_restore_object_maps_failures_to_typed_s3_errors|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))'
|
||||
filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::complete::tests::completion_absent_binding_refusal_order_baseline|app::multipart_usecase::complete::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|app::object::restore::tests::execute_restore_object_maps_failures_to_typed_s3_errors|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))'
|
||||
setup = 'ecstore-base-stack'
|
||||
|
||||
[[profile.ci.scripts]]
|
||||
|
||||
@@ -35,6 +35,7 @@ services:
|
||||
- RUSTFS_LOCK_ACQUIRE_TIMEOUT=${RUSTFS_LOCK_ACQUIRE_TIMEOUT:-5}
|
||||
- RUSTFS_UNSAFE_BYPASS_DISK_CHECK=${RUSTFS_UNSAFE_BYPASS_DISK_CHECK:-true}
|
||||
- RUSTFS_DURABILITY_MODE=${RUSTFS_DURABILITY_MODE:-strict}
|
||||
- RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE=${RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE:-false}
|
||||
- RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_ENABLE=${RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_ENABLE:-false}
|
||||
- RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS=${RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS:-0}
|
||||
# Internode gRPC optimization knobs (grpc-optimization P0-P3). Defaults match the binary
|
||||
@@ -46,6 +47,8 @@ services:
|
||||
- RUSTFS_INTERNODE_CHANNEL_ISOLATION=${RUSTFS_INTERNODE_CHANNEL_ISOLATION:-false}
|
||||
- RUSTFS_INTERNODE_BULK_CHANNELS=${RUSTFS_INTERNODE_BULK_CHANNELS:-2}
|
||||
- RUSTFS_INTERNODE_RPC_MSGPACK_ONLY=${RUSTFS_INTERNODE_RPC_MSGPACK_ONLY:-false}
|
||||
- RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED=${RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED:-false}
|
||||
- RUSTFS_OBS_PUT_STAGE_METRICS_ENABLED=${RUSTFS_OBS_PUT_STAGE_METRICS_ENABLED:-false}
|
||||
- RUSTFS_INTERNODE_PREWARM=${RUSTFS_INTERNODE_PREWARM:-false}
|
||||
- RUSTFS_INTERNODE_OFFLINE_BYPASS=${RUSTFS_INTERNODE_OFFLINE_BYPASS:-false}
|
||||
- RUSTFS_INTERNODE_OFFLINE_REPROBE_SECS=${RUSTFS_INTERNODE_OFFLINE_REPROBE_SECS:-5}
|
||||
@@ -59,7 +62,7 @@ services:
|
||||
- node1_data_2:/data/rustfs2
|
||||
- node1_data_3:/data/rustfs3
|
||||
ports:
|
||||
- "9000:9000"
|
||||
- "${RUSTFS_NODE1_PORT:-9000}:9000"
|
||||
networks:
|
||||
- rustfs-cluster-net
|
||||
|
||||
@@ -85,6 +88,7 @@ services:
|
||||
- RUSTFS_LOCK_ACQUIRE_TIMEOUT=${RUSTFS_LOCK_ACQUIRE_TIMEOUT:-5}
|
||||
- RUSTFS_UNSAFE_BYPASS_DISK_CHECK=${RUSTFS_UNSAFE_BYPASS_DISK_CHECK:-true}
|
||||
- RUSTFS_DURABILITY_MODE=${RUSTFS_DURABILITY_MODE:-strict}
|
||||
- RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE=${RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE:-false}
|
||||
- RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_ENABLE=${RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_ENABLE:-false}
|
||||
- RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS=${RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS:-0}
|
||||
# Internode gRPC optimization knobs (grpc-optimization P0-P3). Defaults match the binary
|
||||
@@ -96,6 +100,8 @@ services:
|
||||
- RUSTFS_INTERNODE_CHANNEL_ISOLATION=${RUSTFS_INTERNODE_CHANNEL_ISOLATION:-false}
|
||||
- RUSTFS_INTERNODE_BULK_CHANNELS=${RUSTFS_INTERNODE_BULK_CHANNELS:-2}
|
||||
- RUSTFS_INTERNODE_RPC_MSGPACK_ONLY=${RUSTFS_INTERNODE_RPC_MSGPACK_ONLY:-false}
|
||||
- RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED=${RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED:-false}
|
||||
- RUSTFS_OBS_PUT_STAGE_METRICS_ENABLED=${RUSTFS_OBS_PUT_STAGE_METRICS_ENABLED:-false}
|
||||
- RUSTFS_INTERNODE_PREWARM=${RUSTFS_INTERNODE_PREWARM:-false}
|
||||
- RUSTFS_INTERNODE_OFFLINE_BYPASS=${RUSTFS_INTERNODE_OFFLINE_BYPASS:-false}
|
||||
- RUSTFS_INTERNODE_OFFLINE_REPROBE_SECS=${RUSTFS_INTERNODE_OFFLINE_REPROBE_SECS:-5}
|
||||
@@ -109,7 +115,7 @@ services:
|
||||
- node2_data_2:/data/rustfs2
|
||||
- node2_data_3:/data/rustfs3
|
||||
ports:
|
||||
- "9001:9000"
|
||||
- "${RUSTFS_NODE2_PORT:-9001}:9000"
|
||||
networks:
|
||||
- rustfs-cluster-net
|
||||
|
||||
@@ -135,6 +141,7 @@ services:
|
||||
- RUSTFS_LOCK_ACQUIRE_TIMEOUT=${RUSTFS_LOCK_ACQUIRE_TIMEOUT:-5}
|
||||
- RUSTFS_UNSAFE_BYPASS_DISK_CHECK=${RUSTFS_UNSAFE_BYPASS_DISK_CHECK:-true}
|
||||
- RUSTFS_DURABILITY_MODE=${RUSTFS_DURABILITY_MODE:-strict}
|
||||
- RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE=${RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE:-false}
|
||||
- RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_ENABLE=${RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_ENABLE:-false}
|
||||
- RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS=${RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS:-0}
|
||||
# Internode gRPC optimization knobs (grpc-optimization P0-P3). Defaults match the binary
|
||||
@@ -146,6 +153,8 @@ services:
|
||||
- RUSTFS_INTERNODE_CHANNEL_ISOLATION=${RUSTFS_INTERNODE_CHANNEL_ISOLATION:-false}
|
||||
- RUSTFS_INTERNODE_BULK_CHANNELS=${RUSTFS_INTERNODE_BULK_CHANNELS:-2}
|
||||
- RUSTFS_INTERNODE_RPC_MSGPACK_ONLY=${RUSTFS_INTERNODE_RPC_MSGPACK_ONLY:-false}
|
||||
- RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED=${RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED:-false}
|
||||
- RUSTFS_OBS_PUT_STAGE_METRICS_ENABLED=${RUSTFS_OBS_PUT_STAGE_METRICS_ENABLED:-false}
|
||||
- RUSTFS_INTERNODE_PREWARM=${RUSTFS_INTERNODE_PREWARM:-false}
|
||||
- RUSTFS_INTERNODE_OFFLINE_BYPASS=${RUSTFS_INTERNODE_OFFLINE_BYPASS:-false}
|
||||
- RUSTFS_INTERNODE_OFFLINE_REPROBE_SECS=${RUSTFS_INTERNODE_OFFLINE_REPROBE_SECS:-5}
|
||||
@@ -159,7 +168,7 @@ services:
|
||||
- node3_data_2:/data/rustfs2
|
||||
- node3_data_3:/data/rustfs3
|
||||
ports:
|
||||
- "9002:9000"
|
||||
- "${RUSTFS_NODE3_PORT:-9002}:9000"
|
||||
networks:
|
||||
- rustfs-cluster-net
|
||||
|
||||
@@ -185,6 +194,7 @@ services:
|
||||
- RUSTFS_LOCK_ACQUIRE_TIMEOUT=${RUSTFS_LOCK_ACQUIRE_TIMEOUT:-5}
|
||||
- RUSTFS_UNSAFE_BYPASS_DISK_CHECK=${RUSTFS_UNSAFE_BYPASS_DISK_CHECK:-true}
|
||||
- RUSTFS_DURABILITY_MODE=${RUSTFS_DURABILITY_MODE:-strict}
|
||||
- RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE=${RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE:-false}
|
||||
- RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_ENABLE=${RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_ENABLE:-false}
|
||||
- RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS=${RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS:-0}
|
||||
# Internode gRPC optimization knobs (grpc-optimization P0-P3). Defaults match the binary
|
||||
@@ -196,6 +206,8 @@ services:
|
||||
- RUSTFS_INTERNODE_CHANNEL_ISOLATION=${RUSTFS_INTERNODE_CHANNEL_ISOLATION:-false}
|
||||
- RUSTFS_INTERNODE_BULK_CHANNELS=${RUSTFS_INTERNODE_BULK_CHANNELS:-2}
|
||||
- RUSTFS_INTERNODE_RPC_MSGPACK_ONLY=${RUSTFS_INTERNODE_RPC_MSGPACK_ONLY:-false}
|
||||
- RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED=${RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED:-false}
|
||||
- RUSTFS_OBS_PUT_STAGE_METRICS_ENABLED=${RUSTFS_OBS_PUT_STAGE_METRICS_ENABLED:-false}
|
||||
- RUSTFS_INTERNODE_PREWARM=${RUSTFS_INTERNODE_PREWARM:-false}
|
||||
- RUSTFS_INTERNODE_OFFLINE_BYPASS=${RUSTFS_INTERNODE_OFFLINE_BYPASS:-false}
|
||||
- RUSTFS_INTERNODE_OFFLINE_REPROBE_SECS=${RUSTFS_INTERNODE_OFFLINE_REPROBE_SECS:-5}
|
||||
@@ -209,7 +221,7 @@ services:
|
||||
- node4_data_2:/data/rustfs2
|
||||
- node4_data_3:/data/rustfs3
|
||||
ports:
|
||||
- "9003:9000"
|
||||
- "${RUSTFS_NODE4_PORT:-9003}:9000"
|
||||
networks:
|
||||
- rustfs-cluster-net
|
||||
|
||||
|
||||
Generated
+60
-33
@@ -1618,15 +1618,6 @@ dependencies = [
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bincode"
|
||||
version = "1.3.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad"
|
||||
dependencies = [
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bindgen"
|
||||
version = "0.71.1"
|
||||
@@ -4142,7 +4133,7 @@ dependencies = [
|
||||
"libc",
|
||||
"option-ext",
|
||||
"redox_users",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.59.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -4456,7 +4447,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e35d6e78f4cebb90e51d543db8fe41528851d6deeef6ef21ebd4bfdc1e784776"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.59.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -5596,6 +5587,7 @@ dependencies = [
|
||||
"pin-project-lite",
|
||||
"smallvec",
|
||||
"tokio",
|
||||
"tracing",
|
||||
"want",
|
||||
]
|
||||
|
||||
@@ -5852,7 +5844,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "20fd6de4ccfcc187e38bc21cfa543cb5a302cb86a8b114eb7f0bf0dc9f8ac00f"
|
||||
dependencies = [
|
||||
"io-lifetimes 3.0.1",
|
||||
"windows-sys 0.60.2",
|
||||
"windows-sys 0.59.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -7017,11 +7009,10 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "nixl-sys"
|
||||
version = "1.5.2"
|
||||
source = "git+https://github.com/ai-dynamo/nixl?tag=v1.5.2#e72c52c78478d2e1ca9e9c57b872114e113ad767"
|
||||
version = "1.6.0"
|
||||
source = "git+https://github.com/houseme/nixl?rev=cc636c83f13d692e85b67e55aed092413d16fb41#cc636c83f13d692e85b67e55aed092413d16fb41"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"bincode",
|
||||
"bindgen 0.71.1",
|
||||
"cc",
|
||||
"libc",
|
||||
@@ -7030,6 +7021,7 @@ dependencies = [
|
||||
"serde",
|
||||
"thiserror 2.0.21",
|
||||
"tracing",
|
||||
"wincode",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -7081,7 +7073,7 @@ version = "0.50.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5"
|
||||
dependencies = [
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.59.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -7974,6 +7966,12 @@ dependencies = [
|
||||
"phc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pastey"
|
||||
version = "0.2.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2ee67f1008b1ba2321834326597b8e186293b049a023cdef258527550b9935b4"
|
||||
|
||||
[[package]]
|
||||
name = "path-absolutize"
|
||||
version = "4.0.1"
|
||||
@@ -9028,7 +9026,7 @@ dependencies = [
|
||||
"once_cell",
|
||||
"socket2",
|
||||
"tracing",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.59.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -10189,6 +10187,7 @@ dependencies = [
|
||||
"serial_test",
|
||||
"sha2 0.11.1",
|
||||
"shadow-rs",
|
||||
"slab",
|
||||
"smallvec",
|
||||
"temp-env",
|
||||
"tempfile",
|
||||
@@ -10267,8 +10266,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-gateway"
|
||||
version = "0.63.0"
|
||||
source = "git+https://github.com/rustfs/gateway.git?rev=2a6b8d1f7663db978d8784ab3ea331cade19237b#2a6b8d1f7663db978d8784ab3ea331cade19237b"
|
||||
version = "0.66.0"
|
||||
source = "git+https://github.com/rustfs/gateway.git?rev=44f9a2422d745ffba952347ec91b0332ef2667a4#44f9a2422d745ffba952347ec91b0332ef2667a4"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"bytes",
|
||||
@@ -10314,8 +10313,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-gateway-core"
|
||||
version = "0.60.0"
|
||||
source = "git+https://github.com/rustfs/gateway.git?rev=2a6b8d1f7663db978d8784ab3ea331cade19237b#2a6b8d1f7663db978d8784ab3ea331cade19237b"
|
||||
version = "0.63.0"
|
||||
source = "git+https://github.com/rustfs/gateway.git?rev=44f9a2422d745ffba952347ec91b0332ef2667a4#44f9a2422d745ffba952347ec91b0332ef2667a4"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"futures-core",
|
||||
@@ -10336,7 +10335,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "rustfs-gateway-http"
|
||||
version = "0.1.4"
|
||||
source = "git+https://github.com/rustfs/gateway.git?rev=2a6b8d1f7663db978d8784ab3ea331cade19237b#2a6b8d1f7663db978d8784ab3ea331cade19237b"
|
||||
source = "git+https://github.com/rustfs/gateway.git?rev=44f9a2422d745ffba952347ec91b0332ef2667a4#44f9a2422d745ffba952347ec91b0332ef2667a4"
|
||||
dependencies = [
|
||||
"hmac 0.13.0",
|
||||
"http 1.5.0",
|
||||
@@ -10354,7 +10353,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "rustfs-gateway-macros"
|
||||
version = "0.1.4"
|
||||
source = "git+https://github.com/rustfs/gateway.git?rev=2a6b8d1f7663db978d8784ab3ea331cade19237b#2a6b8d1f7663db978d8784ab3ea331cade19237b"
|
||||
source = "git+https://github.com/rustfs/gateway.git?rev=44f9a2422d745ffba952347ec91b0332ef2667a4#44f9a2422d745ffba952347ec91b0332ef2667a4"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -10364,7 +10363,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "rustfs-gateway-sig"
|
||||
version = "0.15.0"
|
||||
source = "git+https://github.com/rustfs/gateway.git?rev=2a6b8d1f7663db978d8784ab3ea331cade19237b#2a6b8d1f7663db978d8784ab3ea331cade19237b"
|
||||
source = "git+https://github.com/rustfs/gateway.git?rev=44f9a2422d745ffba952347ec91b0332ef2667a4#44f9a2422d745ffba952347ec91b0332ef2667a4"
|
||||
dependencies = [
|
||||
"hmac 0.13.0",
|
||||
"http 1.5.0",
|
||||
@@ -10379,7 +10378,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "rustfs-gateway-stream"
|
||||
version = "0.2.0"
|
||||
source = "git+https://github.com/rustfs/gateway.git?rev=2a6b8d1f7663db978d8784ab3ea331cade19237b#2a6b8d1f7663db978d8784ab3ea331cade19237b"
|
||||
source = "git+https://github.com/rustfs/gateway.git?rev=44f9a2422d745ffba952347ec91b0332ef2667a4#44f9a2422d745ffba952347ec91b0332ef2667a4"
|
||||
dependencies = [
|
||||
"bitflags 2.13.2",
|
||||
"bytes",
|
||||
@@ -10389,8 +10388,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-gateway-types"
|
||||
version = "0.40.0+aws.2026-10-02"
|
||||
source = "git+https://github.com/rustfs/gateway.git?rev=2a6b8d1f7663db978d8784ab3ea331cade19237b#2a6b8d1f7663db978d8784ab3ea331cade19237b"
|
||||
version = "0.42.0+aws.2026-10-02"
|
||||
source = "git+https://github.com/rustfs/gateway.git?rev=44f9a2422d745ffba952347ec91b0332ef2667a4#44f9a2422d745ffba952347ec91b0332ef2667a4"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"crc-fast",
|
||||
@@ -10413,7 +10412,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "rustfs-gateway-xml"
|
||||
version = "0.4.0"
|
||||
source = "git+https://github.com/rustfs/gateway.git?rev=2a6b8d1f7663db978d8784ab3ea331cade19237b#2a6b8d1f7663db978d8784ab3ea331cade19237b"
|
||||
source = "git+https://github.com/rustfs/gateway.git?rev=44f9a2422d745ffba952347ec91b0332ef2667a4#44f9a2422d745ffba952347ec91b0332ef2667a4"
|
||||
dependencies = [
|
||||
"quick-xml 0.41.0",
|
||||
]
|
||||
@@ -11030,6 +11029,8 @@ version = "1.0.5"
|
||||
dependencies = [
|
||||
"flatbuffers",
|
||||
"hotpath",
|
||||
"hyper",
|
||||
"hyper-util",
|
||||
"prost 0.14.6",
|
||||
"rcgen",
|
||||
"rmp-serde",
|
||||
@@ -11048,6 +11049,7 @@ dependencies = [
|
||||
"tonic-prost",
|
||||
"tonic-prost-build",
|
||||
"tracing",
|
||||
"tracing-subscriber",
|
||||
"uuid",
|
||||
]
|
||||
|
||||
@@ -11640,7 +11642,7 @@ dependencies = [
|
||||
"errno",
|
||||
"libc",
|
||||
"linux-raw-sys",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.59.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -11722,7 +11724,7 @@ dependencies = [
|
||||
"security-framework",
|
||||
"security-framework-sys",
|
||||
"webpki-root-certs",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.59.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -12998,10 +13000,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
|
||||
dependencies = [
|
||||
"fastrand",
|
||||
"getrandom 0.4.3",
|
||||
"getrandom 0.3.4",
|
||||
"once_cell",
|
||||
"rustix",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.59.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -14116,7 +14118,7 @@ version = "0.1.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
|
||||
dependencies = [
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.59.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -14125,6 +14127,31 @@ version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
|
||||
|
||||
[[package]]
|
||||
name = "wincode"
|
||||
version = "0.6.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "063b5affe63fb4e584a33101d8808f01f2348b4f533db6223bf27263a0fdb9e9"
|
||||
dependencies = [
|
||||
"pastey",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"thiserror 2.0.21",
|
||||
"wincode-derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wincode-derive"
|
||||
version = "0.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "11d582d9fc9d813264407a9e16bfa16846ccf15ef032f9bbcd700741bdc00255"
|
||||
dependencies = [
|
||||
"darling 0.23.0",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows"
|
||||
version = "0.62.2"
|
||||
|
||||
+6
-4
@@ -151,14 +151,14 @@ rustfs-zip = { path = "./crates/zip", version = "1.0.5" }
|
||||
# engine crates move onto from s3s (rustfs/backlog#2745). The one pin of the
|
||||
# gateway in this workspace; no crate may enable a `compat-s3s*` feature except
|
||||
# the s3s-facing edge of the rustfs binary.
|
||||
rustfs-gateway-types = { git = "https://github.com/rustfs/gateway.git", rev = "2a6b8d1f7663db978d8784ab3ea331cade19237b", version = "0.40.0", default-features = false }
|
||||
rustfs-gateway-stream = { git = "https://github.com/rustfs/gateway.git", rev = "2a6b8d1f7663db978d8784ab3ea331cade19237b", version = "0.2.0", default-features = false }
|
||||
rustfs-gateway-types = { git = "https://github.com/rustfs/gateway.git", rev = "44f9a2422d745ffba952347ec91b0332ef2667a4", version = "0.42.0", default-features = false }
|
||||
rustfs-gateway-stream = { git = "https://github.com/rustfs/gateway.git", rev = "44f9a2422d745ffba952347ec91b0332ef2667a4", version = "0.2.0", default-features = false }
|
||||
# The gateway framework itself, consumed for its extension traits by the ring-2
|
||||
# adapter. `default-features = false` keeps the `server` feature, and with it the
|
||||
# server crate, out of the build.
|
||||
rustfs-gateway = { git = "https://github.com/rustfs/gateway.git", rev = "2a6b8d1f7663db978d8784ab3ea331cade19237b", version = "0.63.0", default-features = false }
|
||||
rustfs-gateway = { git = "https://github.com/rustfs/gateway.git", rev = "44f9a2422d745ffba952347ec91b0332ef2667a4", version = "0.66.0", default-features = false }
|
||||
# Test-only: builds the `EffectiveHost` a host-resolution test queries with.
|
||||
rustfs-gateway-http = { git = "https://github.com/rustfs/gateway.git", rev = "2a6b8d1f7663db978d8784ab3ea331cade19237b", default-features = false }
|
||||
rustfs-gateway-http = { git = "https://github.com/rustfs/gateway.git", rev = "44f9a2422d745ffba952347ec91b0332ef2667a4", default-features = false }
|
||||
|
||||
# Async Runtime and Networking
|
||||
async-channel = "2.5.0"
|
||||
@@ -186,6 +186,7 @@ memchr = "2.8.3"
|
||||
minlz = "1.2.3"
|
||||
reqwest = "0.13.5"
|
||||
rustfs-kafka-async = { version = "1.4.1" }
|
||||
slab = { version = "0.4.12" }
|
||||
socket2 = { version = "0.6.5" }
|
||||
tokio = { version = "1.54.1" }
|
||||
tokio-rustls = { default-features = false, version = "0.26.6" }
|
||||
@@ -313,6 +314,7 @@ libsystemd = "0.7.2"
|
||||
local-ip-address = "0.6.13"
|
||||
log = "0.4"
|
||||
memmap2 = "0.9.12"
|
||||
nixl-sys = { git = "https://github.com/houseme/nixl", rev = "cc636c83f13d692e85b67e55aed092413d16fb41" }
|
||||
lz4 = "1.28.1"
|
||||
lz4_flex = { version = "=0.14.0", default-features = false, features = ["std", "frame"] }
|
||||
matchit = "0.9.2"
|
||||
|
||||
@@ -23,7 +23,80 @@ pub static GLOBAL_LOCAL_NODE_NAME: LazyLock<RwLock<String>> = LazyLock::new(|| R
|
||||
pub static GLOBAL_RUSTFS_HOST: LazyLock<RwLock<String>> = LazyLock::new(|| RwLock::new("".to_string()));
|
||||
pub static GLOBAL_RUSTFS_PORT: LazyLock<RwLock<String>> = LazyLock::new(|| RwLock::new("9000".to_string()));
|
||||
pub static GLOBAL_RUSTFS_ADDR: LazyLock<RwLock<String>> = LazyLock::new(|| RwLock::new("".to_string()));
|
||||
static NEXT_GRPC_CHANNEL_ID: AtomicU64 = AtomicU64::new(1);
|
||||
static NEXT_GRPC_CONNECTION_GENERATION: AtomicU64 = AtomicU64::new(1);
|
||||
|
||||
#[derive(Debug)]
|
||||
struct GrpcConnectionStateInner {
|
||||
channel_id: u64,
|
||||
generation: AtomicU64,
|
||||
}
|
||||
|
||||
/// Lock-free identity and physical-connection generation shared by every clone of one gRPC channel.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct GrpcConnectionState(Arc<GrpcConnectionStateInner>);
|
||||
|
||||
/// A point-in-time identity for the physical connection currently used by a gRPC channel.
|
||||
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
|
||||
pub struct GrpcConnectionSnapshot {
|
||||
pub channel_id: u64,
|
||||
pub generation: u64,
|
||||
}
|
||||
|
||||
impl GrpcConnectionState {
|
||||
pub fn new() -> Self {
|
||||
Self(Arc::new(GrpcConnectionStateInner {
|
||||
channel_id: next_nonzero_id(&NEXT_GRPC_CHANNEL_ID),
|
||||
generation: AtomicU64::new(0),
|
||||
}))
|
||||
}
|
||||
|
||||
/// Publish a successfully opened physical transport before tonic makes it request-ready.
|
||||
pub fn record_connected(&self) -> GrpcConnectionSnapshot {
|
||||
let generation = next_nonzero_id(&NEXT_GRPC_CONNECTION_GENERATION);
|
||||
self.0.generation.store(generation, Ordering::Release);
|
||||
GrpcConnectionSnapshot {
|
||||
channel_id: self.0.channel_id,
|
||||
generation,
|
||||
}
|
||||
}
|
||||
|
||||
/// Return `None` until the first physical transport has connected successfully.
|
||||
pub fn snapshot(&self) -> Option<GrpcConnectionSnapshot> {
|
||||
let generation = self.0.generation.load(Ordering::Acquire);
|
||||
(generation != 0).then_some(GrpcConnectionSnapshot {
|
||||
channel_id: self.0.channel_id,
|
||||
generation,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for GrpcConnectionState {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
fn next_nonzero_id(counter: &AtomicU64) -> u64 {
|
||||
loop {
|
||||
let id = counter.fetch_add(1, Ordering::Relaxed);
|
||||
if id != 0 {
|
||||
return id;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub static GLOBAL_CONN_MAP: LazyLock<RwLock<HashMap<String, Channel>>> = LazyLock::new(|| RwLock::new(HashMap::new()));
|
||||
|
||||
#[derive(Clone)]
|
||||
struct TrackedGrpcConnection {
|
||||
channel: Channel,
|
||||
connection_state: GrpcConnectionState,
|
||||
}
|
||||
|
||||
static GLOBAL_TRACKED_CONN_MAP: LazyLock<RwLock<HashMap<String, TrackedGrpcConnection>>> =
|
||||
LazyLock::new(|| RwLock::new(HashMap::new()));
|
||||
static GLOBAL_CONN_CACHE_WRITER: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(());
|
||||
pub static GLOBAL_ROOT_CERT: LazyLock<RwLock<Option<Vec<u8>>>> = LazyLock::new(|| RwLock::new(None));
|
||||
pub static GLOBAL_MTLS_IDENTITY: LazyLock<RwLock<Option<MtlsIdentityPem>>> = LazyLock::new(|| RwLock::new(None));
|
||||
pub static GLOBAL_OUTBOUND_TLS_GENERATION: LazyLock<AtomicU64> = LazyLock::new(|| AtomicU64::new(0));
|
||||
@@ -231,6 +304,16 @@ pub async fn evict_connection(addr: &str) {
|
||||
|
||||
/// Evict a stale/dead connection from the global connection cache with an explicit log level.
|
||||
pub async fn evict_connection_with_log_level(addr: &str, log_level: ConnectionEvictionLogLevel) {
|
||||
let _ = remove_connection_with_log_level(addr, log_level).await;
|
||||
}
|
||||
|
||||
/// Remove a cached connection and report whether it existed. Transport owners use this to
|
||||
/// attach a rebuild reason only to real cache transitions.
|
||||
pub async fn remove_connection_with_log_level(addr: &str, log_level: ConnectionEvictionLogLevel) -> bool {
|
||||
// Writes are rare and serialized so the optimized tracked cache and public compatibility map
|
||||
// cannot publish different channel clones for one key.
|
||||
let _writer = GLOBAL_CONN_CACHE_WRITER.lock().await;
|
||||
GLOBAL_TRACKED_CONN_MAP.write().await.remove(addr);
|
||||
let removed = GLOBAL_CONN_MAP.write().await.remove(addr);
|
||||
if removed.is_some() {
|
||||
match log_level {
|
||||
@@ -254,6 +337,7 @@ pub async fn evict_connection_with_log_level(addr: &str, log_level: ConnectionEv
|
||||
}
|
||||
}
|
||||
}
|
||||
removed.is_some()
|
||||
}
|
||||
|
||||
/// Get a cached gRPC connection for the given address.
|
||||
@@ -261,8 +345,37 @@ pub async fn cached_connection(addr: &str) -> Option<Channel> {
|
||||
GLOBAL_CONN_MAP.read().await.get(addr).cloned()
|
||||
}
|
||||
|
||||
/// Get a cached channel together with its shared physical-connection state, when tracked.
|
||||
pub async fn cached_connection_with_state(addr: &str) -> Option<(Channel, Option<GrpcConnectionState>)> {
|
||||
if let Some(cached) = GLOBAL_TRACKED_CONN_MAP.read().await.get(addr).cloned() {
|
||||
return Some((cached.channel, Some(cached.connection_state)));
|
||||
}
|
||||
// A miss may race a low-frequency publish/evict. Serialize only this fallback, recheck the
|
||||
// tracked map, then consult legacy entries injected through the compatibility API.
|
||||
let _writer = GLOBAL_CONN_CACHE_WRITER.lock().await;
|
||||
if let Some(cached) = GLOBAL_TRACKED_CONN_MAP.read().await.get(addr).cloned() {
|
||||
return Some((cached.channel, Some(cached.connection_state)));
|
||||
}
|
||||
cached_connection(addr).await.map(|channel| (channel, None))
|
||||
}
|
||||
|
||||
/// Cache a gRPC connection for the given address.
|
||||
pub async fn cache_connection(addr: String, channel: Channel) {
|
||||
let _writer = GLOBAL_CONN_CACHE_WRITER.lock().await;
|
||||
GLOBAL_TRACKED_CONN_MAP.write().await.remove(&addr);
|
||||
GLOBAL_CONN_MAP.write().await.insert(addr, channel);
|
||||
}
|
||||
|
||||
/// Cache a gRPC channel and its physical-connection state as one coherent entry.
|
||||
pub async fn cache_connection_with_state(addr: String, channel: Channel, connection_state: GrpcConnectionState) {
|
||||
let _writer = GLOBAL_CONN_CACHE_WRITER.lock().await;
|
||||
GLOBAL_TRACKED_CONN_MAP.write().await.insert(
|
||||
addr.clone(),
|
||||
TrackedGrpcConnection {
|
||||
channel: channel.clone(),
|
||||
connection_state,
|
||||
},
|
||||
);
|
||||
GLOBAL_CONN_MAP.write().await.insert(addr, channel);
|
||||
}
|
||||
|
||||
@@ -279,6 +392,8 @@ pub async fn has_cached_connection(addr: &str) -> bool {
|
||||
|
||||
/// Clear all cached connections. Useful for full cluster reset/recovery.
|
||||
pub async fn clear_all_connections() {
|
||||
let _writer = GLOBAL_CONN_CACHE_WRITER.lock().await;
|
||||
GLOBAL_TRACKED_CONN_MAP.write().await.clear();
|
||||
let mut map = GLOBAL_CONN_MAP.write().await;
|
||||
let count = map.len();
|
||||
map.clear();
|
||||
@@ -363,4 +478,46 @@ mod tls_snapshot_tests {
|
||||
assert!(!debug.contains("PRIVATE-KEY-MARKER"));
|
||||
assert!(debug.contains("[REDACTED]"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn conditional_connection_removal_preserves_legacy_evict_api() {
|
||||
let key = "http://conditional-eviction-test.invalid";
|
||||
cache_connection(
|
||||
key.to_owned(),
|
||||
tonic::transport::Endpoint::from_static("http://127.0.0.1:1").connect_lazy(),
|
||||
)
|
||||
.await;
|
||||
assert!(remove_connection_with_log_level(key, ConnectionEvictionLogLevel::Debug).await);
|
||||
assert!(!remove_connection_with_log_level(key, ConnectionEvictionLogLevel::Debug).await);
|
||||
let _: () = evict_connection_with_log_level(key, ConnectionEvictionLogLevel::Debug).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cached_connection_keeps_channel_and_transport_state_coherent() {
|
||||
let key = "http://tracked-connection-test.invalid";
|
||||
let channel = tonic::transport::Endpoint::from_static("http://127.0.0.1:1").connect_lazy();
|
||||
let state = GrpcConnectionState::new();
|
||||
assert!(state.snapshot().is_none());
|
||||
let first = state.record_connected();
|
||||
cache_connection_with_state(key.to_owned(), channel, state.clone()).await;
|
||||
|
||||
let (_, cached_state) = cached_connection_with_state(key).await.expect("tracked cache entry");
|
||||
let cached_state = cached_state.expect("tracked cache state");
|
||||
assert_eq!(cached_state.snapshot(), Some(first));
|
||||
let second = state.record_connected();
|
||||
assert_eq!(cached_state.snapshot(), Some(second));
|
||||
assert_eq!(first.channel_id, second.channel_id);
|
||||
assert_ne!(first.generation, second.generation);
|
||||
|
||||
cache_connection(
|
||||
key.to_owned(),
|
||||
tonic::transport::Endpoint::from_static("http://127.0.0.1:2").connect_lazy(),
|
||||
)
|
||||
.await;
|
||||
let (_, replacement_state) = cached_connection_with_state(key).await.expect("legacy replacement entry");
|
||||
assert!(replacement_state.is_none(), "legacy replacement must remove stale tracked state");
|
||||
|
||||
assert!(remove_connection_with_log_level(key, ConnectionEvictionLogLevel::Debug).await);
|
||||
assert!(cached_connection_with_state(key).await.is_none());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -302,6 +302,8 @@ Drive timeout profile preset:
|
||||
|
||||
RustFS production guidance remains direct-attached local POSIX filesystems. Network-mounted filesystems (for example `nfs`, `cifs`, `smb2`, and `fuse.*`) are treated as unsupported by this startup guard. On Windows, endpoints on network drives (UNC shares such as `\\server\share` and mapped drive letters) are treated the same way, even though Windows reports the server's filesystem (usually `NTFS`) for them.
|
||||
|
||||
A local POSIX filesystem bind-mounted into a gVisor sandbox with directfs is a supported storage path. gVisor presents that bind mount as `9p`/`V9FS`, so the host filesystem type is not visible; the startup guard recognizes mountinfo options `trans=fd` and `directfs` and does not treat the mount as unsupported. Genuine 9p, virtio, and NFS shares remain unsupported.
|
||||
|
||||
## 📄 License
|
||||
|
||||
This project is licensed under the Apache License 2.0 - see the [LICENSE](../../LICENSE) file for details.
|
||||
|
||||
@@ -121,9 +121,17 @@ pub const DEFAULT_INTERNODE_RPC_MSGPACK_ONLY: bool = false;
|
||||
pub const ENV_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED: &str = "RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED";
|
||||
pub const DEFAULT_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED: bool = false;
|
||||
|
||||
/// Permit current per-peer capability proof to replace fleet-wide confirmation.
|
||||
///
|
||||
/// This remains ineffective unless [`ENV_INTERNODE_RPC_MSGPACK_ONLY`] is also enabled. Unknown,
|
||||
/// unsupported, revoked, or stale connection generations keep dual-writing JSON.
|
||||
pub const ENV_INTERNODE_RPC_MSGPACK_PEER_CAPABILITY: &str = "RUSTFS_INTERNODE_RPC_MSGPACK_PEER_CAPABILITY";
|
||||
pub const DEFAULT_INTERNODE_RPC_MSGPACK_PEER_CAPABILITY: bool = false;
|
||||
|
||||
// Compile-time invariants: dual-write by default so the base build is byte-for-byte legacy behavior.
|
||||
const _: () = assert!(!DEFAULT_INTERNODE_RPC_MSGPACK_ONLY);
|
||||
const _: () = assert!(!DEFAULT_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED);
|
||||
const _: () = assert!(!DEFAULT_INTERNODE_RPC_MSGPACK_PEER_CAPABILITY);
|
||||
|
||||
/// Require target-bound v2 signatures on every internode gRPC request, rejecting the legacy
|
||||
/// constant-target fallback instead of accepting it (<https://github.com/rustfs/backlog/issues/1327>).
|
||||
@@ -294,6 +302,14 @@ pub const DEFAULT_INTERNODE_BULK_CONCURRENCY_LIMIT: usize = 64;
|
||||
pub const ENV_INTERNODE_BULK_BUFFER_SIZE: &str = "RUSTFS_INTERNODE_BULK_BUFFER_SIZE";
|
||||
pub const DEFAULT_INTERNODE_BULK_BUFFER_SIZE: usize = 1024;
|
||||
|
||||
/// Per-peer limit held through complete bulk-unary response decoding and validation. Zero keeps
|
||||
/// the compatibility behavior with no additional admission gate.
|
||||
pub const ENV_INTERNODE_BULK_BODY_CONCURRENCY_LIMIT: &str = "RUSTFS_INTERNODE_BULK_BODY_CONCURRENCY_LIMIT";
|
||||
pub const DEFAULT_INTERNODE_BULK_BODY_CONCURRENCY_LIMIT: usize = 0;
|
||||
pub const ENV_INTERNODE_BULK_BODY_WAIT_TIMEOUT_MS: &str = "RUSTFS_INTERNODE_BULK_BODY_WAIT_TIMEOUT_MS";
|
||||
pub const DEFAULT_INTERNODE_BULK_BODY_WAIT_TIMEOUT_MS: u64 = 1_000;
|
||||
const _: () = assert!(DEFAULT_INTERNODE_BULK_BODY_CONCURRENCY_LIMIT == 0);
|
||||
|
||||
/// Profile selector for conservative internode HTTP data-plane client tuning.
|
||||
pub const ENV_INTERNODE_HTTP_TUNING_PROFILE: &str = "RUSTFS_INTERNODE_HTTP_TUNING_PROFILE";
|
||||
pub const DEFAULT_INTERNODE_HTTP_TUNING_PROFILE: &str = "legacy";
|
||||
@@ -501,8 +517,12 @@ mod tests {
|
||||
fn internode_channel_isolation_defaults_and_env_names() {
|
||||
// The isolation-off default is asserted at compile time next to the constant definition.
|
||||
assert_eq!(DEFAULT_INTERNODE_BULK_CHANNELS, 2);
|
||||
assert_eq!(DEFAULT_INTERNODE_BULK_BODY_CONCURRENCY_LIMIT, 0);
|
||||
assert_eq!(DEFAULT_INTERNODE_BULK_BODY_WAIT_TIMEOUT_MS, 1_000);
|
||||
assert_eq!(ENV_INTERNODE_CHANNEL_ISOLATION, "RUSTFS_INTERNODE_CHANNEL_ISOLATION");
|
||||
assert_eq!(ENV_INTERNODE_BULK_CHANNELS, "RUSTFS_INTERNODE_BULK_CHANNELS");
|
||||
assert_eq!(ENV_INTERNODE_BULK_BODY_CONCURRENCY_LIMIT, "RUSTFS_INTERNODE_BULK_BODY_CONCURRENCY_LIMIT");
|
||||
assert_eq!(ENV_INTERNODE_BULK_BODY_WAIT_TIMEOUT_MS, "RUSTFS_INTERNODE_BULK_BODY_WAIT_TIMEOUT_MS");
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -513,6 +533,7 @@ mod tests {
|
||||
ENV_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED,
|
||||
"RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED"
|
||||
);
|
||||
assert_eq!(ENV_INTERNODE_RPC_MSGPACK_PEER_CAPABILITY, "RUSTFS_INTERNODE_RPC_MSGPACK_PEER_CAPABILITY");
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -146,7 +146,11 @@ async fn create_service_account_for(
|
||||
Some(serde_json::json!({ "targetUser": parent }).to_string()),
|
||||
)
|
||||
.await?;
|
||||
let response: serde_json::Value = serde_json::from_str(&response)?;
|
||||
service_account_keys(&response)
|
||||
}
|
||||
|
||||
fn service_account_keys(response: &str) -> Result<(String, String), Box<dyn Error + Send + Sync>> {
|
||||
let response: serde_json::Value = serde_json::from_str(response)?;
|
||||
let access_key = response["credentials"]["accessKey"]
|
||||
.as_str()
|
||||
.ok_or("service account response should contain credentials.accessKey")?
|
||||
@@ -344,6 +348,134 @@ async fn test_update_service_account_enforces_owner_and_parent_scope() -> TestRe
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_delegated_service_account_creation_through_console_sessions() -> TestResult {
|
||||
init_logging();
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server_without_cleanup(vec![]).await?;
|
||||
let bucket = "delegated-backups";
|
||||
let target = "backup-user";
|
||||
let operator = "backup-operator";
|
||||
let operator_secret = "operatorSecret123";
|
||||
let wildcard_admin = "backup-admin";
|
||||
let wildcard_secret = "adminSecret123";
|
||||
let s3 = env.create_s3_client();
|
||||
s3.create_bucket().bucket(bucket).send().await?;
|
||||
for (name, secret) in [
|
||||
(target, "backupSecret123"),
|
||||
(operator, operator_secret),
|
||||
(wildcard_admin, wildcard_secret),
|
||||
("other-user", "otherUserSecret123"),
|
||||
] {
|
||||
admin_ok(
|
||||
&env,
|
||||
http::Method::PUT,
|
||||
&format!("/rustfs/admin/v3/add-user?accessKey={name}"),
|
||||
Some(serde_json::json!({"secretKey":secret,"status":"enabled"}).to_string()),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
let delegation = serde_json::json!({
|
||||
"Version":"2012-10-17",
|
||||
"Statement":[
|
||||
{"Effect":"Allow","Action":"admin:CreateServiceAccount"},
|
||||
{"Effect":"Allow","Action":"sts:AssumeRole","Resource":"arn:aws:s3:::*"},
|
||||
{"Effect":"Allow","Action":"admin:CreateServiceAccountForUser","Condition":{"StringEquals":{"svc:TargetUser":target}}}
|
||||
]
|
||||
});
|
||||
let wildcard = serde_json::json!({
|
||||
"Version":"2012-10-17",
|
||||
"Statement":[
|
||||
{"Effect":"Allow","Action":"admin:*"},
|
||||
{"Effect":"Allow","Action":"sts:AssumeRole","Resource":"arn:aws:s3:::*"}
|
||||
]
|
||||
});
|
||||
for (user, name, policy) in [
|
||||
(wildcard_admin, "backup-admin-wildcard", wildcard.to_string()),
|
||||
(target, "backup-writer", bucket_rw_policy(bucket)),
|
||||
(operator, "backup-key-manager", delegation.to_string()),
|
||||
] {
|
||||
admin_ok(
|
||||
&env,
|
||||
http::Method::PUT,
|
||||
&format!("/rustfs/admin/v3/add-canned-policy?name={name}"),
|
||||
Some(policy),
|
||||
)
|
||||
.await?;
|
||||
admin_ok(
|
||||
&env,
|
||||
http::Method::POST,
|
||||
"/rustfs/admin/v3/idp/builtin/policy/attach",
|
||||
Some(serde_json::json!({"policies":[name],"user":user}).to_string()),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
for (name, secret) in [
|
||||
(operator, operator_secret),
|
||||
(wildcard_admin, wildcard_secret),
|
||||
(env.access_key.as_str(), env.secret_key.as_str()),
|
||||
] {
|
||||
let session = build_test_sts_client(&env.url, name, secret, None, "delegated-create")
|
||||
.assume_role()
|
||||
.role_arn("arn:aws:iam::123456789012:role/console")
|
||||
.role_session_name("console")
|
||||
.send()
|
||||
.await?;
|
||||
let caller = session.credentials().ok_or("missing console session")?;
|
||||
let create = |user: &str| {
|
||||
admin_request_with_session_token(
|
||||
&env.url,
|
||||
http::Method::PUT,
|
||||
"/rustfs/admin/v3/add-service-account",
|
||||
Some(serde_json::json!({"targetUser":user}).to_string()),
|
||||
caller.access_key_id(),
|
||||
caller.secret_access_key(),
|
||||
Some(caller.session_token()),
|
||||
)
|
||||
};
|
||||
let (status, response) = create(target).await?;
|
||||
assert_eq!(status, StatusCode::OK, "create for target as {name}");
|
||||
let (key, secret) = service_account_keys(&response)?;
|
||||
let client = build_s3_client(&env.url, &key, &secret);
|
||||
wait_for_s3_put(&client, bucket, "backup", Duration::from_secs(10)).await?;
|
||||
let denied = client
|
||||
.create_bucket()
|
||||
.bucket("outside-backup-policy")
|
||||
.send()
|
||||
.await
|
||||
.expect_err("target cannot create unrelated buckets");
|
||||
assert_eq!(denied.as_service_error().and_then(|e| e.code()), Some("AccessDenied"));
|
||||
if name != env.access_key {
|
||||
let (status, _) = create(&env.access_key).await?;
|
||||
assert_eq!(status, StatusCode::FORBIDDEN, "delegated targets exclude root");
|
||||
}
|
||||
if name == operator {
|
||||
let (status, _) = create("other-user").await?;
|
||||
assert_eq!(status, StatusCode::FORBIDDEN, "outside delegated target");
|
||||
}
|
||||
}
|
||||
let restricted = build_test_sts_client(&env.url, &env.access_key, &env.secret_key, None, "restricted-root")
|
||||
.assume_role()
|
||||
.role_arn("arn:aws:iam::123456789012:role/console")
|
||||
.role_session_name("restricted-console")
|
||||
.policy(r#"{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"admin:CreateServiceAccount"}]}"#)
|
||||
.send()
|
||||
.await?;
|
||||
let caller = restricted.credentials().ok_or("missing restricted root session")?;
|
||||
let (status, _) = admin_request_with_session_token(
|
||||
&env.url,
|
||||
http::Method::PUT,
|
||||
"/rustfs/admin/v3/add-service-account",
|
||||
Some(serde_json::json!({"targetUser":target}).to_string()),
|
||||
caller.access_key_id(),
|
||||
caller.secret_access_key(),
|
||||
Some(caller.session_token()),
|
||||
)
|
||||
.await?;
|
||||
assert_eq!(status, StatusCode::FORBIDDEN, "restricted root session requires delegated permission");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Full user -> policy -> service-account lifecycle, proving each management
|
||||
/// call takes effect on the data plane, not just that the endpoint answers 200.
|
||||
#[tokio::test]
|
||||
|
||||
@@ -126,6 +126,7 @@ test-util = []
|
||||
e2e-test-hooks = []
|
||||
|
||||
[dependencies]
|
||||
slab.workspace = true
|
||||
hashbrown.workspace = true
|
||||
hotpath.workspace = true
|
||||
rustfs-filemeta.workspace = true
|
||||
|
||||
@@ -708,13 +708,14 @@ pub mod rpc {
|
||||
sign_ns_scanner_capability_with_checkpoint_coverage, sign_ns_scanner_capability_with_durable_progress,
|
||||
sign_ns_scanner_capability_with_tier_registry_generation, sign_ns_scanner_capability_with_usage_capabilities,
|
||||
sign_ns_scanner_usage_watch_capability, sign_put_file_capability, sign_tonic_rpc_response_proof,
|
||||
tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason,
|
||||
verify_ns_scanner_capability, verify_ns_scanner_capability_with_checkpoint_coverage,
|
||||
verify_ns_scanner_capability_with_durable_progress, verify_ns_scanner_capability_with_tier_registry_generation,
|
||||
verify_ns_scanner_capability_with_usage_capabilities, verify_ns_scanner_usage_watch_capability,
|
||||
verify_put_file_auth_trailer, verify_put_file_capability, verify_rpc_signature, verify_tonic_boot_epoch_response,
|
||||
verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest, verify_tonic_mutation_body_digest_reject_unsigned,
|
||||
verify_tonic_rpc_response_proof, verify_tonic_rpc_signature, verify_tonic_rpc_signature_with_bootstrap,
|
||||
tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, tonic_boot_epoch_response_headers_with_msgpack_capability,
|
||||
tonic_msgpack_manifest_capability_requested, tonic_rpc_auth_failure_reason, verify_ns_scanner_capability,
|
||||
verify_ns_scanner_capability_with_checkpoint_coverage, verify_ns_scanner_capability_with_durable_progress,
|
||||
verify_ns_scanner_capability_with_tier_registry_generation, verify_ns_scanner_capability_with_usage_capabilities,
|
||||
verify_ns_scanner_usage_watch_capability, verify_put_file_auth_trailer, verify_put_file_capability, verify_rpc_signature,
|
||||
verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest,
|
||||
verify_tonic_mutation_body_digest_reject_unsigned, verify_tonic_rpc_response_proof, verify_tonic_rpc_signature,
|
||||
verify_tonic_rpc_signature_with_bootstrap,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -78,6 +78,9 @@ pub const RPC_BOOT_EPOCH_CHALLENGE_HEADER: &str = "x-rustfs-rpc-boot-epoch-chall
|
||||
pub const RPC_BOOT_EPOCH_PROOF_HEADER: &str = "x-rustfs-rpc-boot-epoch-proof";
|
||||
pub(crate) const RPC_REPLAY_CACHE_CAPABILITY_HEADER: &str = "x-rustfs-rpc-replay-cache-capability";
|
||||
pub(crate) const RPC_REPLAY_CACHE_CAPABILITY_PROOF_HEADER: &str = "x-rustfs-rpc-replay-cache-capability-proof";
|
||||
pub(crate) const RPC_MSGPACK_MANIFEST_CAPABILITY_REQUEST_HEADER: &str = "x-rustfs-rpc-msgpack-manifest-request";
|
||||
pub(crate) const RPC_MSGPACK_MANIFEST_CAPABILITY_HEADER: &str = "x-rustfs-rpc-msgpack-manifest-capability";
|
||||
pub(crate) const RPC_MSGPACK_MANIFEST_CAPABILITY_PROOF_HEADER: &str = "x-rustfs-rpc-msgpack-manifest-capability-proof";
|
||||
const RPC_REPLAY_SCOPE_VERSION_V3: &str = "3";
|
||||
const RPC_RESPONSE_PROOF_DOMAIN: &[u8] = b"rustfs-rpc-response-proof-v1\0";
|
||||
|
||||
@@ -124,6 +127,8 @@ const RPC_REPLAY_SCOPE_DOMAIN: &[u8] = b"rustfs-rpc-replay-scope-v3\0";
|
||||
const RPC_BOOT_EPOCH_PROOF_DOMAIN: &[u8] = b"rustfs-rpc-boot-epoch-proof-v1\0";
|
||||
const RPC_REPLAY_CACHE_CAPABILITY_PROOF_DOMAIN: &[u8] = b"rustfs-rpc-replay-cache-capability-proof-v1\0";
|
||||
const RPC_REPLAY_CACHE_CAPABILITY_V1: &str = "dynamic-replay-cache-v1";
|
||||
const RPC_MSGPACK_MANIFEST_CAPABILITY_PROOF_DOMAIN: &[u8] = b"rustfs-rpc-msgpack-manifest-capability-proof-v1\0";
|
||||
pub(crate) const RPC_MSGPACK_MANIFEST_CAPABILITY_V1: &str = "msgpack-compat-manifest-v1";
|
||||
const HTTP_PUT_FILE_AUTH_DOMAIN: &[u8] = b"rustfs-http-put-file-auth-v1\0";
|
||||
const HTTP_PUT_FILE_CAPABILITY_AUTH_DOMAIN: &[u8] = b"rustfs-http-put-file-capability-v1\0";
|
||||
const UNSIGNED_PAYLOAD: &str = "UNSIGNED-PAYLOAD";
|
||||
@@ -1238,6 +1243,50 @@ fn verify_replay_cache_capability_proof(
|
||||
.map_err(|_| std::io::Error::new(std::io::ErrorKind::PermissionDenied, "Invalid RPC replay cache capability proof"))
|
||||
}
|
||||
|
||||
fn update_msgpack_manifest_capability_proof(mac: &mut HmacSha256, audience: &str, challenge: Uuid, boot_epoch: Uuid) {
|
||||
mac.update(RPC_MSGPACK_MANIFEST_CAPABILITY_PROOF_DOMAIN);
|
||||
for part in [
|
||||
audience.as_bytes(),
|
||||
b"|",
|
||||
challenge.as_bytes(),
|
||||
b"|",
|
||||
boot_epoch.as_bytes(),
|
||||
b"|",
|
||||
RPC_MSGPACK_MANIFEST_CAPABILITY_V1.as_bytes(),
|
||||
] {
|
||||
mac.update(part);
|
||||
}
|
||||
}
|
||||
|
||||
fn generate_msgpack_manifest_capability_proof(
|
||||
secret: &str,
|
||||
audience: &str,
|
||||
challenge: Uuid,
|
||||
boot_epoch: Uuid,
|
||||
) -> std::io::Result<String> {
|
||||
let mut mac =
|
||||
<HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
|
||||
update_msgpack_manifest_capability_proof(&mut mac, audience, challenge, boot_epoch);
|
||||
Ok(base64_simd::STANDARD.encode_to_string(mac.finalize().into_bytes()))
|
||||
}
|
||||
|
||||
fn verify_msgpack_manifest_capability_proof(
|
||||
secret: &str,
|
||||
audience: &str,
|
||||
challenge: Uuid,
|
||||
boot_epoch: Uuid,
|
||||
proof: &str,
|
||||
) -> std::io::Result<()> {
|
||||
let proof = base64_simd::STANDARD
|
||||
.decode_to_vec(proof)
|
||||
.map_err(|_| std::io::Error::other("Invalid RPC msgpack manifest capability proof"))?;
|
||||
let mut mac =
|
||||
<HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
|
||||
update_msgpack_manifest_capability_proof(&mut mac, audience, challenge, boot_epoch);
|
||||
mac.verify_slice(&proof)
|
||||
.map_err(|_| std::io::Error::new(std::io::ErrorKind::PermissionDenied, "Invalid RPC msgpack manifest capability proof"))
|
||||
}
|
||||
|
||||
fn non_nil_uuid(value: &str, name: &str) -> std::io::Result<Uuid> {
|
||||
let value = Uuid::parse_str(value).map_err(|_| std::io::Error::other(format!("Invalid {name}")))?;
|
||||
(!value.is_nil())
|
||||
@@ -1337,8 +1386,24 @@ pub fn tonic_boot_epoch_challenge(headers: &HeaderMap) -> std::io::Result<Option
|
||||
.transpose()
|
||||
}
|
||||
|
||||
/// Whether the request opts into the optional msgpack manifest capability proof.
|
||||
pub fn tonic_msgpack_manifest_capability_requested(headers: &HeaderMap) -> bool {
|
||||
headers
|
||||
.get(RPC_MSGPACK_MANIFEST_CAPABILITY_REQUEST_HEADER)
|
||||
.is_some_and(|value| value == RPC_MSGPACK_MANIFEST_CAPABILITY_V1)
|
||||
}
|
||||
|
||||
/// Build the authenticated response headers for a client boot-epoch challenge.
|
||||
pub fn tonic_boot_epoch_response_headers(audience: &str, challenge: Uuid) -> std::io::Result<HeaderMap> {
|
||||
tonic_boot_epoch_response_headers_with_msgpack_capability(audience, challenge, false)
|
||||
}
|
||||
|
||||
/// Build authenticated boot-epoch headers and, when requested, the msgpack manifest capability.
|
||||
pub fn tonic_boot_epoch_response_headers_with_msgpack_capability(
|
||||
audience: &str,
|
||||
challenge: Uuid,
|
||||
include_msgpack_manifest: bool,
|
||||
) -> std::io::Result<HeaderMap> {
|
||||
let boot_epoch = tonic_rpc_boot_epoch();
|
||||
let secret = get_shared_secret()?;
|
||||
let proof = generate_boot_epoch_proof(&secret, audience, challenge, boot_epoch)?;
|
||||
@@ -1354,6 +1419,17 @@ pub fn tonic_boot_epoch_response_headers(audience: &str, challenge: Uuid) -> std
|
||||
RPC_REPLAY_CACHE_CAPABILITY_PROOF_HEADER,
|
||||
header_value(&capability_proof, RPC_REPLAY_CACHE_CAPABILITY_PROOF_HEADER)?,
|
||||
);
|
||||
if include_msgpack_manifest {
|
||||
let msgpack_capability_proof = generate_msgpack_manifest_capability_proof(&secret, audience, challenge, boot_epoch)?;
|
||||
headers.insert(
|
||||
RPC_MSGPACK_MANIFEST_CAPABILITY_HEADER,
|
||||
HeaderValue::from_static(RPC_MSGPACK_MANIFEST_CAPABILITY_V1),
|
||||
);
|
||||
headers.insert(
|
||||
RPC_MSGPACK_MANIFEST_CAPABILITY_PROOF_HEADER,
|
||||
header_value(&msgpack_capability_proof, RPC_MSGPACK_MANIFEST_CAPABILITY_PROOF_HEADER)?,
|
||||
);
|
||||
}
|
||||
Ok(headers)
|
||||
}
|
||||
|
||||
@@ -1387,6 +1463,12 @@ pub(crate) struct AuthenticatedPeerReplayCapabilities {
|
||||
pub(crate) dynamic_replay_cache: bool,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub(crate) struct AuthenticatedPeerMsgpackCapabilities {
|
||||
pub(crate) boot_epoch: Uuid,
|
||||
pub(crate) msgpack_compat_manifest: bool,
|
||||
}
|
||||
|
||||
pub(crate) fn verify_tonic_peer_replay_capabilities_response(
|
||||
audience: &str,
|
||||
challenge: Uuid,
|
||||
@@ -1418,6 +1500,57 @@ pub(crate) fn verify_tonic_peer_replay_capabilities_response(
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn verify_tonic_peer_msgpack_capabilities_response(
|
||||
audience: &str,
|
||||
challenge: Uuid,
|
||||
headers: &HeaderMap,
|
||||
) -> std::io::Result<AuthenticatedPeerMsgpackCapabilities> {
|
||||
let secret = get_shared_secret()?;
|
||||
let boot_epoch = verify_tonic_boot_epoch_response_with_secret(&secret, audience, challenge, headers)?;
|
||||
verify_tonic_peer_msgpack_capability_for_epoch_with_secret(&secret, audience, challenge, boot_epoch, headers)
|
||||
}
|
||||
|
||||
pub(crate) fn verify_tonic_peer_msgpack_capability_for_authenticated_epoch(
|
||||
audience: &str,
|
||||
challenge: Uuid,
|
||||
boot_epoch: Uuid,
|
||||
headers: &HeaderMap,
|
||||
) -> std::io::Result<AuthenticatedPeerMsgpackCapabilities> {
|
||||
verify_tonic_peer_msgpack_capability_for_epoch_with_secret(&get_shared_secret()?, audience, challenge, boot_epoch, headers)
|
||||
}
|
||||
|
||||
fn verify_tonic_peer_msgpack_capability_for_epoch_with_secret(
|
||||
secret: &str,
|
||||
audience: &str,
|
||||
challenge: Uuid,
|
||||
boot_epoch: Uuid,
|
||||
headers: &HeaderMap,
|
||||
) -> std::io::Result<AuthenticatedPeerMsgpackCapabilities> {
|
||||
let capability = headers.get(RPC_MSGPACK_MANIFEST_CAPABILITY_HEADER);
|
||||
let proof = headers.get(RPC_MSGPACK_MANIFEST_CAPABILITY_PROOF_HEADER);
|
||||
if capability.is_none() && proof.is_none() {
|
||||
return Ok(AuthenticatedPeerMsgpackCapabilities {
|
||||
boot_epoch,
|
||||
msgpack_compat_manifest: false,
|
||||
});
|
||||
}
|
||||
let capability = capability
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.ok_or_else(|| std::io::Error::other("Missing RPC msgpack manifest capability"))?;
|
||||
if capability != RPC_MSGPACK_MANIFEST_CAPABILITY_V1 {
|
||||
return Err(std::io::Error::other("Unsupported RPC msgpack manifest capability"));
|
||||
}
|
||||
let proof = proof
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.ok_or_else(|| std::io::Error::other("Missing RPC msgpack manifest capability proof"))?;
|
||||
verify_msgpack_manifest_capability_proof(secret, audience, challenge, boot_epoch, proof)?;
|
||||
Ok(AuthenticatedPeerMsgpackCapabilities {
|
||||
boot_epoch,
|
||||
msgpack_compat_manifest: true,
|
||||
})
|
||||
}
|
||||
|
||||
fn valid_content_sha256(value: &str) -> bool {
|
||||
value == UNSIGNED_PAYLOAD
|
||||
|| (value.len() == 64
|
||||
@@ -3147,6 +3280,55 @@ mod tests {
|
||||
assert!(verify_tonic_peer_replay_capabilities_response("node-a:9000", challenge, &changed_capability).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn msgpack_manifest_capability_is_requested_explicitly_and_proved_for_one_epoch() {
|
||||
ensure_test_rpc_secret();
|
||||
let challenge = Uuid::new_v4();
|
||||
let mut request_headers = HeaderMap::new();
|
||||
assert!(!tonic_msgpack_manifest_capability_requested(&request_headers));
|
||||
request_headers.insert(
|
||||
RPC_MSGPACK_MANIFEST_CAPABILITY_REQUEST_HEADER,
|
||||
HeaderValue::from_static(RPC_MSGPACK_MANIFEST_CAPABILITY_V1),
|
||||
);
|
||||
assert!(tonic_msgpack_manifest_capability_requested(&request_headers));
|
||||
request_headers.insert(
|
||||
RPC_MSGPACK_MANIFEST_CAPABILITY_REQUEST_HEADER,
|
||||
HeaderValue::from_static("msgpack-compat-manifest-v2"),
|
||||
);
|
||||
assert!(!tonic_msgpack_manifest_capability_requested(&request_headers));
|
||||
|
||||
let legacy_headers = tonic_boot_epoch_response_headers("node-a:9000", challenge).expect("legacy capability headers");
|
||||
let legacy = verify_tonic_peer_msgpack_capabilities_response("node-a:9000", challenge, &legacy_headers)
|
||||
.expect("authenticated response without msgpack capability");
|
||||
assert_eq!(legacy.boot_epoch, tonic_rpc_boot_epoch());
|
||||
assert!(!legacy.msgpack_compat_manifest);
|
||||
|
||||
let headers = tonic_boot_epoch_response_headers_with_msgpack_capability("node-a:9000", challenge, true)
|
||||
.expect("msgpack capability headers");
|
||||
let capability = verify_tonic_peer_msgpack_capabilities_response("node-a:9000", challenge, &headers)
|
||||
.expect("matching msgpack capability proof");
|
||||
assert_eq!(capability.boot_epoch, tonic_rpc_boot_epoch());
|
||||
assert!(capability.msgpack_compat_manifest);
|
||||
assert!(verify_tonic_peer_msgpack_capabilities_response("node-b:9000", challenge, &headers).is_err());
|
||||
assert!(verify_tonic_peer_msgpack_capabilities_response("node-a:9000", Uuid::new_v4(), &headers).is_err());
|
||||
|
||||
let mut changed_capability = headers.clone();
|
||||
changed_capability.insert(
|
||||
RPC_MSGPACK_MANIFEST_CAPABILITY_HEADER,
|
||||
HeaderValue::from_static("msgpack-compat-manifest-v2"),
|
||||
);
|
||||
assert!(verify_tonic_peer_msgpack_capabilities_response("node-a:9000", challenge, &changed_capability).is_err());
|
||||
let mut forged_proof = headers;
|
||||
forged_proof.insert(RPC_MSGPACK_MANIFEST_CAPABILITY_PROOF_HEADER, HeaderValue::from_static("forged"));
|
||||
assert!(verify_tonic_peer_msgpack_capabilities_response("node-a:9000", challenge, &forged_proof).is_err());
|
||||
let mut missing_proof = forged_proof.clone();
|
||||
missing_proof.remove(RPC_MSGPACK_MANIFEST_CAPABILITY_PROOF_HEADER);
|
||||
assert!(verify_tonic_peer_msgpack_capabilities_response("node-a:9000", challenge, &missing_proof).is_err());
|
||||
let mut missing_value = forged_proof;
|
||||
missing_value.remove(RPC_MSGPACK_MANIFEST_CAPABILITY_HEADER);
|
||||
assert!(verify_tonic_peer_msgpack_capabilities_response("node-a:9000", challenge, &missing_value).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tonic_rpc_auth_failure_reason_maps_security_relevant_errors() {
|
||||
for (message, reason) in [
|
||||
|
||||
@@ -43,8 +43,9 @@ pub use http_auth::{
|
||||
set_tonic_rolling_mutation_body_digest, sign_ns_scanner_capability, sign_ns_scanner_capability_with_checkpoint_coverage,
|
||||
sign_ns_scanner_capability_with_durable_progress, sign_ns_scanner_capability_with_tier_registry_generation,
|
||||
sign_ns_scanner_capability_with_usage_capabilities, sign_ns_scanner_usage_watch_capability, sign_put_file_capability,
|
||||
sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason,
|
||||
verify_ns_scanner_capability, verify_ns_scanner_capability_with_checkpoint_coverage,
|
||||
sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers,
|
||||
tonic_boot_epoch_response_headers_with_msgpack_capability, tonic_msgpack_manifest_capability_requested,
|
||||
tonic_rpc_auth_failure_reason, verify_ns_scanner_capability, verify_ns_scanner_capability_with_checkpoint_coverage,
|
||||
verify_ns_scanner_capability_with_durable_progress, verify_ns_scanner_capability_with_tier_registry_generation,
|
||||
verify_ns_scanner_capability_with_usage_capabilities, verify_ns_scanner_usage_watch_capability, verify_put_file_auth_trailer,
|
||||
verify_put_file_capability, verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest,
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -29,7 +29,7 @@ use rustfs_protos::proto_gen::node_service::{
|
||||
LockAcquireStatus, LockReleaseStatus, PingRequest,
|
||||
};
|
||||
use rustfs_protos::{
|
||||
ConnectionEvictionLogLevel, evict_failed_connection_with_log_level, models::PingBodyBuilder,
|
||||
ChannelRebuildReason, ConnectionEvictionLogLevel, evict_failed_connection_with_reason, models::PingBodyBuilder,
|
||||
proto_gen::node_service::node_service_client::NodeServiceClient,
|
||||
};
|
||||
use std::collections::HashMap;
|
||||
@@ -498,7 +498,7 @@ impl RemoteClient {
|
||||
))
|
||||
}
|
||||
|
||||
async fn evict_connection(&self, op: &'static str, reason: &str, resource_summary: &str) {
|
||||
async fn evict_connection(&self, op: &'static str, reason: &str, resource_summary: &str, trigger: EvictionTrigger) {
|
||||
let log_level = if Self::is_scanner_leader_lock(resource_summary) {
|
||||
debug!(
|
||||
addr = %self.addr,
|
||||
@@ -518,7 +518,11 @@ impl RemoteClient {
|
||||
);
|
||||
ConnectionEvictionLogLevel::Warn
|
||||
};
|
||||
evict_failed_connection_with_log_level(&self.addr, log_level).await;
|
||||
let rebuild_reason = match trigger {
|
||||
EvictionTrigger::Timeout => ChannelRebuildReason::LockTimeout,
|
||||
EvictionTrigger::Transport => ChannelRebuildReason::LockTransport,
|
||||
};
|
||||
evict_failed_connection_with_reason(&self.addr, rebuild_reason, log_level).await;
|
||||
}
|
||||
|
||||
fn summarize_resources(requests: &[LockRequest]) -> String {
|
||||
@@ -772,7 +776,7 @@ impl RemoteClient {
|
||||
});
|
||||
if verdict == EvictionVerdict::Evict {
|
||||
rustfs_io_metrics::lock_metrics::record_remote_lock_channel_eviction(&self.addr, trigger.as_str());
|
||||
self.evict_connection(op, reason, resource_summary).await;
|
||||
self.evict_connection(op, reason, resource_summary, trigger).await;
|
||||
return;
|
||||
}
|
||||
rustfs_io_metrics::lock_metrics::record_remote_lock_channel_eviction_suppressed(&self.addr, verdict.as_str());
|
||||
|
||||
@@ -11083,6 +11083,104 @@ impl LocalDisk {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
if !recursive
|
||||
&& staging_cleanup.is_none()
|
||||
&& delete_path.file_name().and_then(|n| n.to_str()) == Some(STORAGE_FORMAT_FILE_BACKUP)
|
||||
&& self.multipart_delete_check(delete_path, false)?.is_none()
|
||||
{
|
||||
let target_path = delete_path.clone();
|
||||
let root_base = base_path.to_path_buf();
|
||||
let prune_op = move |is_cancelled: &dyn Fn() -> bool, check_cancel: bool| -> Result<()> {
|
||||
if check_cancel && is_cancelled() {
|
||||
return Err(DiskError::Io(std::io::Error::other("blocking namespace operation cancelled")));
|
||||
}
|
||||
#[cfg(all(any(test, feature = "test-util"), not(windows)))]
|
||||
os::prepared_publication_test_hooks::run(os::prepared_publication_test_hooks::Stage::Remove, &target_path);
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
let start_as_dir = target_path.is_dir();
|
||||
#[cfg(target_os = "linux")]
|
||||
let start_as_dir = false;
|
||||
let mut current = if start_as_dir {
|
||||
Some(target_path.as_path())
|
||||
} else {
|
||||
match std::fs::remove_file(&target_path) {
|
||||
Ok(()) => target_path.parent(),
|
||||
Err(err) if err.kind() == ErrorKind::NotFound => target_path.parent(),
|
||||
Err(err)
|
||||
if err.kind() == ErrorKind::IsADirectory
|
||||
|| (cfg!(not(target_os = "linux"))
|
||||
&& err.kind() == ErrorKind::PermissionDenied
|
||||
&& target_path.is_dir()) =>
|
||||
{
|
||||
Some(target_path.as_path())
|
||||
}
|
||||
Err(err) => {
|
||||
warn!(
|
||||
event = EVENT_DISK_LOCAL_DELETE_FAILED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_DISK_LOCAL,
|
||||
path = ?target_path,
|
||||
operation = "remove_file",
|
||||
error = ?err,
|
||||
"Disk local delete failed"
|
||||
);
|
||||
return Err(Error::other(FileAccessDeniedWithContext {
|
||||
path: target_path.clone(),
|
||||
source: err,
|
||||
}));
|
||||
}
|
||||
}
|
||||
};
|
||||
let first_dir = current;
|
||||
while let Some(dir_path) = current.filter(|parent| *parent != root_base && parent.starts_with(&root_base)) {
|
||||
if Some(dir_path) != first_dir && std::fs::symlink_metadata(dir_path.join(STORAGE_FORMAT_FILE)).is_ok() {
|
||||
return Ok(());
|
||||
}
|
||||
if dir_path != target_path.as_path() {
|
||||
#[cfg(all(any(test, feature = "test-util"), not(windows)))]
|
||||
os::prepared_publication_test_hooks::run(
|
||||
os::prepared_publication_test_hooks::Stage::Remove,
|
||||
dir_path,
|
||||
);
|
||||
}
|
||||
match std::fs::remove_dir(dir_path) {
|
||||
Ok(()) => current = dir_path.parent(),
|
||||
Err(err) if is_dir_not_empty_error(&err) => return Ok(()),
|
||||
Err(err) if is_benign_object_rmdir_error(&err) => current = dir_path.parent(),
|
||||
Err(err) => {
|
||||
warn!(
|
||||
event = EVENT_DISK_LOCAL_DELETE_FAILED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_DISK_LOCAL,
|
||||
path = ?dir_path,
|
||||
operation = "remove_dir",
|
||||
error_kind = %err.kind(),
|
||||
"Disk local delete failed"
|
||||
);
|
||||
return Err(Error::other(FileAccessDeniedWithContext {
|
||||
path: dir_path.to_path_buf(),
|
||||
source: err,
|
||||
}));
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
};
|
||||
return match namespace_owner {
|
||||
Some(owner) => {
|
||||
let lease = os::acquire_namespace_mutation_lease_with_owner(delete_path, Some(owner)).await;
|
||||
os::run_blocking_namespace_operation(lease, move || Ok(prune_op(&|| false, false)))
|
||||
.await
|
||||
.map_err(to_file_error)?
|
||||
}
|
||||
None => self
|
||||
.blocking_queue
|
||||
.run_cancellable(move |is_cancelled| Ok(prune_op(is_cancelled, true)))
|
||||
.await
|
||||
.map_err(to_file_error)?,
|
||||
};
|
||||
}
|
||||
|
||||
if recursive {
|
||||
self.move_to_trash_with_namespace_owner(delete_path, recursive, immediate_purge, namespace_owner.clone())
|
||||
.await?;
|
||||
@@ -24563,6 +24661,20 @@ mod test {
|
||||
assert!(!DurabilityMode::LegacyOff.syncs_recovery_protocol_state());
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(file_fdatasync_group_commit)]
|
||||
fn test_file_fdatasync_grouping_only_blocks_non_inline_fused_commit() {
|
||||
let _grouping = os::set_file_fdatasync_group_commit_for_test(true);
|
||||
assert!(
|
||||
!commit::file_fdatasync_grouping_requires_async_shard_sync(false),
|
||||
"inline metadata has no shard directory and must keep the fused path"
|
||||
);
|
||||
assert!(
|
||||
commit::file_fdatasync_grouping_requires_async_shard_sync(true),
|
||||
"non-inline shard sync must remain on the grouping-aware async path"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_system_critical_volume_classification() {
|
||||
// System namespaces are pinned.
|
||||
@@ -25395,14 +25507,223 @@ mod test {
|
||||
.join(STORAGE_FORMAT_FILE_BACKUP)
|
||||
.is_file()
|
||||
);
|
||||
use std::os::unix::fs::MetadataExt;
|
||||
let v3_meta_bytes = fs::read(dst_object_dir.join(STORAGE_FORMAT_FILE))
|
||||
.await
|
||||
.expect("read v3 xl.meta");
|
||||
let v3_meta_ino = std::fs::metadata(dst_object_dir.join(STORAGE_FORMAT_FILE))
|
||||
.expect("stat v3 xl.meta")
|
||||
.ino();
|
||||
let synthetic_rollback = inline_metadata_rollback_dir(version_id, &FileMeta::new());
|
||||
let synthetic_backup_parent = dst_object_dir.join(synthetic_rollback.to_string());
|
||||
let synthetic_backup_path = synthetic_backup_parent.join(STORAGE_FORMAT_FILE_BACKUP);
|
||||
fs::create_dir_all(&synthetic_backup_parent)
|
||||
.await
|
||||
.expect("create stale synthetic dir");
|
||||
fs::write(&synthetic_backup_path, b"stale-bkp")
|
||||
.await
|
||||
.expect("write stale xl.meta.bkp");
|
||||
|
||||
let fi_v4 = test_file_info(object, version_id, None, Some(Bytes::from_static(b"v4-inline")));
|
||||
let resp4 = disk
|
||||
.rename_data(RUSTFS_META_TMP_BUCKET, "tmp-v4", fi_v4, bucket, object)
|
||||
.await
|
||||
.expect("fused inline overwrite v4 over inline v3");
|
||||
assert_eq!(
|
||||
(resp4.old_data_dir, resp4.rollback_data_dir, resp4.cleanup_data_dir),
|
||||
(None, Some(synthetic_rollback), None)
|
||||
);
|
||||
assert_eq!(fs::read(&synthetic_backup_path).await.expect("read v3 synthetic backup"), v3_meta_bytes);
|
||||
assert_eq!(
|
||||
std::fs::metadata(&synthetic_backup_path).expect("stat v3 backup").ino(),
|
||||
v3_meta_ino,
|
||||
"linkat path must hardlink the previous xl.meta inode"
|
||||
);
|
||||
assert!(os::fsync_dir_recorder::was_fsynced(&synthetic_backup_parent));
|
||||
assert!(os::fsync_dir_recorder::was_limited(&synthetic_backup_parent));
|
||||
disk.delete(
|
||||
bucket,
|
||||
&format!("{object}/{synthetic_rollback}/{STORAGE_FORMAT_FILE_BACKUP}"),
|
||||
DeleteOptions::default(),
|
||||
)
|
||||
.await
|
||||
.expect("reclaim synthetic inline rollback backup");
|
||||
assert!(!synthetic_backup_parent.exists(), "empty synthetic rollback dir must be pruned");
|
||||
assert!(dst_object_dir.join(STORAGE_FORMAT_FILE).is_file());
|
||||
|
||||
let open_dst_meta = disk
|
||||
.io_get_object_open_path(bucket, &format!("{object}/{STORAGE_FORMAT_FILE}"))
|
||||
.expect("open dst meta path");
|
||||
let v4_meta_bytes = fs::read(dst_object_dir.join(STORAGE_FORMAT_FILE))
|
||||
.await
|
||||
.expect("read v4 xl.meta");
|
||||
let v4_meta_ino = std::fs::metadata(dst_object_dir.join(STORAGE_FORMAT_FILE))
|
||||
.expect("stat v4 xl.meta")
|
||||
.ino();
|
||||
set_local_inline_rollback_hardlink_failure(&open_dst_meta);
|
||||
fs::create_dir_all(&synthetic_backup_parent)
|
||||
.await
|
||||
.expect("create synthetic dir with child");
|
||||
fs::write(synthetic_backup_parent.join("child-marker"), b"keep")
|
||||
.await
|
||||
.expect("write child marker");
|
||||
let fi_v5 = test_file_info(object, version_id, None, Some(Bytes::from_static(b"v5-inline")));
|
||||
let resp5 = disk
|
||||
.rename_data(RUSTFS_META_TMP_BUCKET, "tmp-v5", fi_v5, bucket, object)
|
||||
.await
|
||||
.expect("fused inline overwrite v5 copy fallback");
|
||||
assert_eq!(resp5.rollback_data_dir, Some(synthetic_rollback));
|
||||
assert_eq!(
|
||||
fs::read(&synthetic_backup_path).await.expect("read v4 copy-fallback backup"),
|
||||
v4_meta_bytes
|
||||
);
|
||||
assert_ne!(
|
||||
std::fs::metadata(&synthetic_backup_path).expect("stat v4 backup").ino(),
|
||||
v4_meta_ino,
|
||||
"copy-fallback path must allocate a new backup inode"
|
||||
);
|
||||
disk.delete(
|
||||
bucket,
|
||||
&format!("{object}/{synthetic_rollback}/{STORAGE_FORMAT_FILE_BACKUP}"),
|
||||
DeleteOptions::default(),
|
||||
)
|
||||
.await
|
||||
.expect("reclaim synthetic backup while child exists");
|
||||
assert!(!synthetic_backup_path.exists());
|
||||
assert!(
|
||||
synthetic_backup_parent.join("child-marker").is_file(),
|
||||
"non-empty synthetic rollback dir must preserve child entries"
|
||||
);
|
||||
fs::remove_file(synthetic_backup_parent.join("child-marker"))
|
||||
.await
|
||||
.expect("remove child marker");
|
||||
fs::remove_dir(&synthetic_backup_parent)
|
||||
.await
|
||||
.expect("remove empty synthetic dir");
|
||||
|
||||
let legacy_key = "parent/xl.meta.bkp";
|
||||
let legacy_fi = test_file_info(legacy_key, version_id, None, Some(Bytes::from_static(b"legacy-inline")));
|
||||
let legacy_dir = disk.io_get_object_path(bucket, legacy_key).expect("legacy object dir");
|
||||
fs::create_dir_all(&legacy_dir).await.expect("create legacy dir");
|
||||
fs::write(legacy_dir.join(STORAGE_FORMAT_FILE), test_meta(legacy_fi.clone()))
|
||||
.await
|
||||
.expect("write legacy xl.meta");
|
||||
disk.delete_version(bucket, legacy_key, legacy_fi, false, DeleteOptions::default())
|
||||
.await
|
||||
.expect("delete legacy key named xl.meta.bkp");
|
||||
assert!(!disk.io_get_object_path(bucket, "parent").expect("parent path").exists());
|
||||
|
||||
let dst_meta_path = dst_object_dir.join(STORAGE_FORMAT_FILE);
|
||||
let v5_meta_bytes = fs::read(&dst_meta_path).await.expect("read v5 xl.meta");
|
||||
let dst_vol_dir = disk.io_get_bucket_path(bucket).expect("bucket path");
|
||||
let lease = os::acquire_namespace_mutation_lease_with_owner(&dst_meta_path, None).await;
|
||||
let guard =
|
||||
commit::lock_rename_commit_directories(&dst_object_dir, &dst_object_dir, &dst_vol_dir, &disk.publication_root, lease)
|
||||
.await
|
||||
.expect("lock commit guard");
|
||||
fs::create_dir_all(synthetic_backup_path.join("nested-child"))
|
||||
.await
|
||||
.expect("create conflicting directory at xl.meta.bkp");
|
||||
assert!(
|
||||
os::stage_inline_rollback_backup_with_commit_guard_std(
|
||||
&dst_meta_path,
|
||||
&synthetic_backup_parent,
|
||||
&synthetic_backup_path,
|
||||
&v5_meta_bytes,
|
||||
true,
|
||||
true,
|
||||
&guard,
|
||||
)
|
||||
.is_err(),
|
||||
"directory at xl.meta.bkp must fail staging without removing child"
|
||||
);
|
||||
assert!(synthetic_backup_path.join("nested-child").is_dir());
|
||||
fs::remove_dir_all(&synthetic_backup_parent)
|
||||
.await
|
||||
.expect("remove conflicting synthetic dir");
|
||||
|
||||
os::fsync_dir_recorder::set_failure(&synthetic_backup_parent, std::io::ErrorKind::Other);
|
||||
assert!(
|
||||
os::stage_inline_rollback_backup_with_commit_guard_std(
|
||||
&dst_meta_path,
|
||||
&synthetic_backup_parent,
|
||||
&synthetic_backup_path,
|
||||
&v5_meta_bytes,
|
||||
true,
|
||||
true,
|
||||
&guard,
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
assert!(!synthetic_backup_parent.exists(), "staging fsync failure must clean up created dir");
|
||||
|
||||
os::stage_inline_rollback_backup_with_commit_guard_std(
|
||||
&dst_meta_path,
|
||||
&synthetic_backup_parent,
|
||||
&synthetic_backup_path,
|
||||
&v5_meta_bytes,
|
||||
true,
|
||||
true,
|
||||
&guard,
|
||||
)
|
||||
.expect("stage direct inline rollback backup");
|
||||
assert!(synthetic_backup_path.is_file());
|
||||
let displaced_object_dir = dst_object_dir.with_file_name("fused-object-displaced");
|
||||
fs::rename(&dst_object_dir, &displaced_object_dir)
|
||||
.await
|
||||
.expect("displace dst_object_dir");
|
||||
fs::create_dir_all(&synthetic_backup_parent)
|
||||
.await
|
||||
.expect("create replacement synthetic dir");
|
||||
fs::write(&synthetic_backup_path, b"replacement-bkp")
|
||||
.await
|
||||
.expect("write replacement backup");
|
||||
commit::discard_staged_inline_rollback_backup_std(synthetic_rollback, None, &guard);
|
||||
assert_eq!(
|
||||
fs::read(&synthetic_backup_path)
|
||||
.await
|
||||
.expect("replacement backup must survive"),
|
||||
b"replacement-bkp",
|
||||
"discard must operate on the pinned guard directory, not the replacement path"
|
||||
);
|
||||
assert!(
|
||||
!displaced_object_dir.join(synthetic_rollback.to_string()).exists(),
|
||||
"discard must remove the backup and synthetic dir from the pinned directory"
|
||||
);
|
||||
fs::remove_dir_all(&dst_object_dir)
|
||||
.await
|
||||
.expect("remove replacement object dir");
|
||||
fs::rename(&displaced_object_dir, &dst_object_dir)
|
||||
.await
|
||||
.expect("restore original object dir");
|
||||
|
||||
os::stage_inline_rollback_backup_with_commit_guard_std(
|
||||
&dst_meta_path,
|
||||
&synthetic_backup_parent,
|
||||
&synthetic_backup_path,
|
||||
&v5_meta_bytes,
|
||||
true,
|
||||
true,
|
||||
&guard,
|
||||
)
|
||||
.expect("stage direct inline rollback backup for fsync rollback");
|
||||
let staged_new_meta = dst_object_dir.join("staged-new.xl.meta");
|
||||
fs::write(&staged_new_meta, b"uncommitted-meta")
|
||||
.await
|
||||
.expect("write staged new meta");
|
||||
fs::rename(&staged_new_meta, &dst_meta_path)
|
||||
.await
|
||||
.expect("replace dst_meta_path before fsync failure");
|
||||
os::fsync_dir_recorder::set_failure(&dst_object_dir, std::io::ErrorKind::Other);
|
||||
assert!(
|
||||
commit::fsync_inline_commit_std(&dst_meta_path, &dst_vol_dir, false, Some(synthetic_rollback), None, None, &guard,)
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
!synthetic_backup_parent.exists(),
|
||||
"fsync rollback must remove consumed synthetic rollback dir"
|
||||
);
|
||||
assert_eq!(fs::read(&dst_meta_path).await.expect("restored v5"), v5_meta_bytes);
|
||||
let replacement_dir = dst_object_dir.join(Uuid::new_v4().to_string());
|
||||
fs::create_dir_all(&replacement_dir).await.expect("create replacement_dir");
|
||||
fs::write(replacement_dir.join("part.1"), b"live").await.expect("write part");
|
||||
@@ -37574,12 +37895,12 @@ mod test {
|
||||
.expect("truncate behind cached descriptor");
|
||||
assert!(read().await.is_err(), "native short data must not be accepted");
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
while backend.driver.stats().in_flight != 0 {
|
||||
while backend.driver.stats().in_flight != 0 || memory.snapshot().used != 0 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("failed native read retired");
|
||||
.expect("failed native read retired and released its memory");
|
||||
assert!(backend.driver.stats().submitted > submitted, "short read must reach the native driver");
|
||||
assert_eq!(memory.snapshot().used, 0, "errors free native and assembled storage");
|
||||
std::fs::write(dir.path().join("bucket/part.1"), &content).expect("restore fixture");
|
||||
|
||||
@@ -231,14 +231,26 @@ fn rollback_metadata_std(destination_metadata: &Path, rollback_data_dir: Option<
|
||||
let Some(dst_parent) = destination_metadata.parent() else {
|
||||
return Err(std::io::Error::new(ErrorKind::InvalidInput, "missing object metadata parent"));
|
||||
};
|
||||
let backup_path = dst_parent.join(old_data_dir.to_string()).join(STORAGE_FORMAT_FILE_BACKUP);
|
||||
std::fs::rename(backup_path, destination_metadata)?;
|
||||
let rollback_parent = dst_parent.join(old_data_dir.to_string());
|
||||
let backup_path = rollback_parent.join(STORAGE_FORMAT_FILE_BACKUP);
|
||||
std::fs::rename(&backup_path, destination_metadata)?;
|
||||
let _ = remove_file_if_exists(&backup_path);
|
||||
let _ = std::fs::remove_dir(rollback_parent);
|
||||
} else {
|
||||
remove_file_if_exists(destination_metadata)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
pub(super) fn discard_staged_inline_rollback_backup_std(
|
||||
old_data_dir: Uuid,
|
||||
has_old_data_dir: Option<Uuid>,
|
||||
commit_guard: &os::RenameCommitGuard,
|
||||
) {
|
||||
os::discard_inline_rollback_backup_with_commit_guard_std(&old_data_dir.to_string(), has_old_data_dir.is_none(), commit_guard);
|
||||
}
|
||||
|
||||
fn rollback_inline_metadata_std(
|
||||
destination_metadata: &Path,
|
||||
rollback_data_dir: Option<Uuid>,
|
||||
@@ -412,6 +424,14 @@ enum FusedCommitAttempt {
|
||||
ResumePanic(Box<dyn std::any::Any + Send + 'static>),
|
||||
}
|
||||
|
||||
/// File fdatasync grouping only owns staged shard files. Inline commits have
|
||||
/// no shard directory, so the switch must not evict them from the flat/fused
|
||||
/// metadata path where grouping can never run.
|
||||
#[cfg(any(target_os = "linux", test))]
|
||||
pub(super) fn file_fdatasync_grouping_requires_async_shard_sync(no_inline: bool) -> bool {
|
||||
no_inline && os::file_fdatasync_group_commit_enabled()
|
||||
}
|
||||
|
||||
async fn prepare_rename_commit_and_destination_metadata(
|
||||
create_source_parent: bool,
|
||||
source_parent: &Path,
|
||||
@@ -750,7 +770,6 @@ impl LocalDisk {
|
||||
&& pool_meta_condition.is_none()
|
||||
&& !fi_healing
|
||||
&& !os::dst_dir_fsync_group_commit_enabled()
|
||||
&& !os::file_fdatasync_group_commit_enabled()
|
||||
&& !has_rename_data_test_failpoints(dst_path);
|
||||
let src_file_path = match multipart_publication.as_ref() {
|
||||
Some(publication) => publication.canonical_head(),
|
||||
@@ -838,7 +857,7 @@ impl LocalDisk {
|
||||
&& pool_meta_condition.is_none()
|
||||
&& !fi_healing
|
||||
&& !os::dst_dir_fsync_group_commit_enabled()
|
||||
&& !os::file_fdatasync_group_commit_enabled()
|
||||
&& !file_fdatasync_grouping_requires_async_shard_sync(no_inline)
|
||||
{
|
||||
#[cfg(test)]
|
||||
if !no_inline {
|
||||
@@ -961,9 +980,9 @@ impl LocalDisk {
|
||||
}
|
||||
|
||||
let mut staged_rollback_path = None;
|
||||
if (!no_inline || rollback_data_dir.is_none())
|
||||
if rollback_data_dir.is_none()
|
||||
&& let Some(ref old_metadata) = has_dst_buf
|
||||
&& (rollback_data_dir.is_some() || cfg!(test))
|
||||
&& cfg!(test)
|
||||
{
|
||||
let backup_path = match create_local_inline_rollback_backup(
|
||||
&dst_file_path_owned,
|
||||
@@ -1117,25 +1136,32 @@ impl LocalDisk {
|
||||
if crash_inject::should_crash_at(CrashPoint::RenameAfterBackupBeforeMetaCommit, &dst_path_str) {
|
||||
return Ok(FusedCommitAttempt::CommitErr(DiskError::Unexpected));
|
||||
}
|
||||
} else if let (Some(old_data_dir), Some(staged_backup)) =
|
||||
(rollback_data_dir, staged_rollback_path.as_ref().map(StagedRollbackBackup::path))
|
||||
{
|
||||
} else if let (Some(old_data_dir), Some(dst_buf)) = (rollback_data_dir, has_dst_buf.as_ref()) {
|
||||
let backup_parent = dst_parent_owned.join(old_data_dir.to_string());
|
||||
let backup_path = backup_parent.join(STORAGE_FORMAT_FILE_BACKUP);
|
||||
if let Err(err) = os::rename_all_with_precondition_and_parent_sync_std(
|
||||
staged_backup,
|
||||
if let Err(err) = check_path_length(backup_path.to_string_lossy().as_ref()) {
|
||||
drop(prepared_metadata_source);
|
||||
cleanup_flat_tmp();
|
||||
return Ok(FusedCommitAttempt::CommitErr(err));
|
||||
}
|
||||
#[cfg(test)]
|
||||
run_owned_file_before_open(&backup_path);
|
||||
if let Err(err) = os::stage_inline_rollback_backup_with_commit_guard_std(
|
||||
&dst_file_path_owned,
|
||||
&backup_parent,
|
||||
&backup_path,
|
||||
&skip_parent,
|
||||
&publication_root_owned,
|
||||
None,
|
||||
dst_buf.as_ref(),
|
||||
sync_recovery_protocol,
|
||||
!should_fail_local_inline_rollback_hardlink(&dst_file_path_owned),
|
||||
&guard,
|
||||
) {
|
||||
drop(prepared_metadata_source);
|
||||
cleanup_flat_tmp();
|
||||
let _ = remove_file_if_exists(staged_backup);
|
||||
return Ok(FusedCommitAttempt::CommitErr(to_file_error(err).into()));
|
||||
}
|
||||
staged_rollback_path = None;
|
||||
if crash_inject::should_crash_at(CrashPoint::RenameAfterBackupBeforeMetaCommit, &dst_path_str) {
|
||||
return Ok(FusedCommitAttempt::CommitErr(DiskError::Unexpected));
|
||||
}
|
||||
}
|
||||
|
||||
let rename_res = if should_fail_commit_rename(&dst_path_str) {
|
||||
@@ -1159,6 +1185,9 @@ impl LocalDisk {
|
||||
if no_inline {
|
||||
return Ok(FusedCommitAttempt::RestoreDataSource(guard, err));
|
||||
}
|
||||
if let Some(old_data_dir) = rollback_data_dir {
|
||||
discard_staged_inline_rollback_backup_std(old_data_dir, has_old_data_dir, &guard);
|
||||
}
|
||||
return Ok(FusedCommitAttempt::CommitErr(err));
|
||||
}
|
||||
|
||||
|
||||
+754
-75
File diff suppressed because it is too large
Load Diff
@@ -1388,6 +1388,10 @@ pub struct ObjectOptions {
|
||||
#[doc(hidden)]
|
||||
pub multipart_completion_authorization:
|
||||
Option<Arc<dyn crate::storage_api_contracts::multipart::MultipartCompletionAuthorization<Error = Error>>>,
|
||||
/// In-process authorization against the selected upload, while Abort holds its write lock.
|
||||
#[doc(hidden)]
|
||||
pub multipart_abort_authorization:
|
||||
Option<Arc<dyn crate::storage_api_contracts::multipart::MultipartAbortAuthorization<Error = Error>>>,
|
||||
pub eval_metadata: Option<HashMap<String, String>>,
|
||||
/// Internal compare-and-set condition for replication workers publishing
|
||||
/// terminal status after remote I/O. Storage validates it while holding
|
||||
|
||||
@@ -4009,6 +4009,7 @@ impl SetDisks {
|
||||
file_info.validate_for_erasure_write()?;
|
||||
}
|
||||
}
|
||||
let file_infos = Arc::new(file_infos);
|
||||
|
||||
let disk_count = disks.len();
|
||||
let fanout_disks = disks.to_vec();
|
||||
@@ -4031,19 +4032,16 @@ impl SetDisks {
|
||||
let successful_rename_completion_rank =
|
||||
rustfs_io_metrics::put_stage_metrics_enabled().then(|| Arc::new(AtomicUsize::new(0)));
|
||||
let mut tasks = JoinSet::new();
|
||||
for (i, ((disk, file_info), scanner_publication_lease_token)) in fanout_disks
|
||||
.into_iter()
|
||||
.zip(file_infos.iter())
|
||||
.zip(fanout_fence_tokens.into_iter())
|
||||
.enumerate()
|
||||
for (i, (disk, scanner_publication_lease_token)) in
|
||||
fanout_disks.into_iter().zip(fanout_fence_tokens.into_iter()).enumerate()
|
||||
{
|
||||
let src_bucket = fanout_src_bucket.clone();
|
||||
let src_object = fanout_src_object.clone();
|
||||
let dst_bucket = fanout_dst_bucket.clone();
|
||||
let dst_object = fanout_dst_object.clone();
|
||||
let file_info = file_info.clone();
|
||||
let successful_rename_completion_rank = successful_rename_completion_rank.clone();
|
||||
let namespace_commit_guard = namespace_commit_guard.clone();
|
||||
let file_infos = file_infos.clone();
|
||||
tasks.spawn(async move {
|
||||
let _namespace_commit_guard = namespace_commit_guard.clone();
|
||||
let mut dispatch_state = RenameDispatchState::NotDispatched;
|
||||
@@ -4060,12 +4058,16 @@ impl SetDisks {
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DISK_WAIT_REMOTE
|
||||
};
|
||||
|
||||
let file_info = &file_infos[i];
|
||||
let is_delete_marker = file_info.is_canonical_delete_marker();
|
||||
// Clone FileInfo and set erasure.index for this disk
|
||||
let mut file_info = file_info.clone();
|
||||
if file_info.erasure.index == 0 {
|
||||
file_info.erasure.index = i + 1;
|
||||
}
|
||||
let mut indexed_file_info;
|
||||
let file_info = if file_info.erasure.index == 0 {
|
||||
indexed_file_info = file_info.clone();
|
||||
indexed_file_info.erasure.index = i + 1;
|
||||
&indexed_file_info
|
||||
} else {
|
||||
file_info
|
||||
};
|
||||
if file_info.erasure.index == 0 || (!is_delete_marker && !file_info.has_valid_erasure_geometry()) {
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
@@ -4083,7 +4085,7 @@ impl SetDisks {
|
||||
.rename_data_borrowed_with_fence_observed(
|
||||
&src_bucket,
|
||||
&src_object,
|
||||
&file_info,
|
||||
file_info,
|
||||
&dst_bucket,
|
||||
&dst_object,
|
||||
disk::RenameDataGuards {
|
||||
@@ -4191,7 +4193,7 @@ impl SetDisks {
|
||||
let snapshot_commit = Self::rename_data_commit_from_observations(
|
||||
RenameDataObservations {
|
||||
disks: &coordinator_disks,
|
||||
file_infos: &file_infos,
|
||||
file_infos: file_infos.as_slice(),
|
||||
disk_versions: &disk_versions,
|
||||
errs: &errs,
|
||||
cleanup_data_dirs: &cleanup_data_dirs,
|
||||
@@ -4236,6 +4238,7 @@ impl SetDisks {
|
||||
if !sent_commit {
|
||||
let ret_err =
|
||||
reduce_write_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, write_quorum).unwrap_or(DiskError::Unexpected);
|
||||
let file_infos = Arc::try_unwrap(file_infos).unwrap_or_else(|file_infos| file_infos.as_ref().clone());
|
||||
rollback_failed_rename(
|
||||
&coordinator_disks,
|
||||
file_infos,
|
||||
|
||||
@@ -83,6 +83,7 @@ use crate::set_disk::core::multipart_source::{
|
||||
use crate::set_disk::mem;
|
||||
use crate::set_disk::metadata_sys;
|
||||
use crate::set_disk::runtime_sources;
|
||||
use crate::storage_api_contracts::multipart::ListPartsOwner;
|
||||
#[cfg(test)]
|
||||
use crate::storage_api_contracts::multipart::MultipartOperations;
|
||||
#[cfg(test)]
|
||||
@@ -683,6 +684,22 @@ fn fence_commit_on_lock_loss(guard: Option<&ObjectLockDiagGuard>, mode: &'static
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn recorded_multipart_owner(
|
||||
metadata: &HashMap<String, String>,
|
||||
bucket: &str,
|
||||
encoded_object: &str,
|
||||
) -> Option<crate::storage_api_contracts::multipart::MultipartOwner> {
|
||||
let recorded_bucket = get_consistent_str(metadata, "multipart-bucket")?;
|
||||
let recorded_object = get_consistent_str(metadata, "multipart-object")?;
|
||||
if recorded_bucket != bucket || recorded_object != encoded_object {
|
||||
return None;
|
||||
}
|
||||
Some(crate::storage_api_contracts::multipart::MultipartOwner {
|
||||
bucket: recorded_bucket.to_owned(),
|
||||
object: decode_dir_object(recorded_object),
|
||||
})
|
||||
}
|
||||
|
||||
fn multipart_bucket_incarnation_id(metadata: &HashMap<String, String>) -> Result<Option<Uuid>> {
|
||||
let Some(value) = rustfs_utils::http::metadata_compat::get_consistent_str(metadata, SUFFIX_BUCKET_INCARNATION_ID) else {
|
||||
if rustfs_utils::http::metadata_compat::contains_key_str(metadata, SUFFIX_BUCKET_INCARNATION_ID) {
|
||||
@@ -3190,7 +3207,21 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
.cloned()
|
||||
.unwrap_or_else(|| storageclass::STANDARD.to_string());
|
||||
|
||||
// Use the same metadata snapshot that produced the page.
|
||||
let recorded_owner = match (
|
||||
get_consistent_str(&fi.metadata, "multipart-bucket"),
|
||||
get_consistent_str(&fi.metadata, "multipart-object"),
|
||||
) {
|
||||
(Some(recorded_bucket), Some(recorded_object)) if recorded_bucket == bucket && recorded_object == object => {
|
||||
Some(ListPartsOwner {
|
||||
bucket: recorded_bucket.to_owned(),
|
||||
object: decode_dir_object(recorded_object),
|
||||
})
|
||||
}
|
||||
_ => None,
|
||||
};
|
||||
let mut ret = ListPartsInfo {
|
||||
recorded_owner,
|
||||
bucket: bucket.to_owned(),
|
||||
object: decode_dir_object(object),
|
||||
upload_id: upload_id.to_owned(),
|
||||
@@ -3597,6 +3628,10 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
ensure_multipart_bucket_incarnation(&self.ctx, &fi, bucket, object, upload_id, opts.expected_bucket_incarnation_id)
|
||||
.await?;
|
||||
ensure_multipart_bucket_lifecycle_lock_held(bucket, object, opts)?;
|
||||
if let Some(authorization) = &opts.multipart_abort_authorization {
|
||||
let owner = recorded_multipart_owner(&fi.metadata, bucket, object);
|
||||
authorization.authorize(owner.as_ref())?;
|
||||
}
|
||||
let disks = self.get_disks_internal().await;
|
||||
let cleanup = prepare_source_cleanup(
|
||||
&disks,
|
||||
@@ -6434,6 +6469,8 @@ mod tests {
|
||||
assert!(empty.parts.is_empty());
|
||||
assert!(!empty.is_truncated);
|
||||
assert_eq!(empty.next_part_number_marker, None);
|
||||
let owner = empty.recorded_owner.expect("the empty page has the recorded owner");
|
||||
assert_eq!((owner.bucket.as_str(), owner.object.as_str()), (bucket, object));
|
||||
|
||||
for part_number in [10, 1, 3] {
|
||||
put_test_part(&set_disks, bucket, object, &upload.upload_id, part_number, b"part", 4).await;
|
||||
@@ -6460,6 +6497,66 @@ mod tests {
|
||||
assert_eq!(page.parts.iter().map(|part| part.part_num).collect::<Vec<_>>(), expected_parts);
|
||||
assert_eq!(page.is_truncated, next_marker.is_some());
|
||||
assert_eq!(page.next_part_number_marker, next_marker);
|
||||
assert_eq!(page.recorded_owner.as_ref(), Some(&owner));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn list_parts_baseline_legacy_owner_metadata() {
|
||||
let (_roots, disks, set) = hermetic_set_disks(4).await;
|
||||
let bucket = "list-parts-owner-observation";
|
||||
let object = "object";
|
||||
let opts = ObjectOptions::default();
|
||||
make_bucket_on_all(&disks, bucket).await;
|
||||
let upload = set.new_multipart_upload(bucket, object, &opts).await.expect("real upload");
|
||||
let path = SetDisks::get_upload_id_dir(bucket, object, &upload.upload_id);
|
||||
let owner_keys = [
|
||||
RUSTFS_MULTIPART_BUCKET_KEY,
|
||||
"x-minio-internal-multipart-bucket",
|
||||
RUSTFS_MULTIPART_OBJECT_KEY,
|
||||
"x-minio-internal-multipart-object",
|
||||
];
|
||||
for (label, values) in [
|
||||
("rustfs", [Some(bucket), None, Some(object), None]),
|
||||
("minio", [None, Some(bucket), None, Some(object)]),
|
||||
("both", [Some(bucket), Some(bucket), Some(object), Some(object)]),
|
||||
("no bucket", [None, None, Some(object), None]),
|
||||
("no object", [Some(bucket), None, None, None]),
|
||||
("other bucket", [Some("other-bucket"), None, Some(object), None]),
|
||||
("other object", [Some(bucket), None, Some("other-object"), None]),
|
||||
("bucket aliases differ", [Some(bucket), Some("other-bucket"), Some(object), None]),
|
||||
("object aliases differ", [Some(bucket), None, Some(object), Some("other-object")]),
|
||||
] {
|
||||
for disk in &disks {
|
||||
let mut fi = disk
|
||||
.read_version("", RUSTFS_META_MULTIPART_BUCKET, &path, "", &ReadOptions::default())
|
||||
.await
|
||||
.expect("read actual upload metadata");
|
||||
for (key, value) in owner_keys.into_iter().zip(values) {
|
||||
if let Some(value) = value {
|
||||
fi.metadata.insert(key.to_owned(), value.to_owned());
|
||||
} else {
|
||||
fi.metadata.remove(key);
|
||||
}
|
||||
}
|
||||
disk.write_metadata(bucket, RUSTFS_META_MULTIPART_BUCKET, &path, fi)
|
||||
.await
|
||||
.expect("write the synthetic owner metadata");
|
||||
}
|
||||
let page = set
|
||||
.list_object_parts(bucket, object, &upload.upload_id, None, 1, &opts)
|
||||
.await
|
||||
.unwrap_or_else(|error| panic!("{label}: {error:?}"));
|
||||
assert_eq!((page.bucket.as_str(), page.object.as_str()), (bucket, object), "{label}");
|
||||
assert!(page.parts.is_empty(), "{label}");
|
||||
let expected = matches!(label, "rustfs" | "minio" | "both").then_some((bucket, object));
|
||||
assert_eq!(
|
||||
page.recorded_owner
|
||||
.as_ref()
|
||||
.map(|owner| (owner.bucket.as_str(), owner.object.as_str())),
|
||||
expected,
|
||||
"{label}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1029,3 +1029,229 @@ mod native_abort_closure_tests {
|
||||
include!("multipart_initial_probe_tests.rs");
|
||||
}
|
||||
}
|
||||
|
||||
mod abort_owner_tests {
|
||||
use super::*;
|
||||
use crate::storage_api_contracts::multipart::{MultipartAbortAuthorization, MultipartOwner};
|
||||
|
||||
#[test]
|
||||
fn recorded_multipart_owner_requires_unambiguous_metadata_for_the_selected_namespace() {
|
||||
const BUCKET: &str = "abort-owner";
|
||||
const ENCODED: &str = "directory__XLDIR__";
|
||||
const RUSTFS: &str = RUSTFS_MULTIPART_BUCKET_KEY;
|
||||
const MINIO: &str = "x-minio-internal-multipart-bucket";
|
||||
const RUSTFS_OBJECT: &str = RUSTFS_MULTIPART_OBJECT_KEY;
|
||||
const MINIO_OBJECT: &str = "x-minio-internal-multipart-object";
|
||||
|
||||
let expected = MultipartOwner {
|
||||
bucket: BUCKET.to_owned(),
|
||||
object: "directory/".to_owned(),
|
||||
};
|
||||
let rustfs = [(RUSTFS, BUCKET), (RUSTFS_OBJECT, ENCODED)];
|
||||
let minio = [(MINIO, BUCKET), (MINIO_OBJECT, ENCODED)];
|
||||
for fields in [rustfs.to_vec(), minio.to_vec(), [rustfs, minio].concat()] {
|
||||
let metadata = fields
|
||||
.into_iter()
|
||||
.map(|(key, value)| (key.to_owned(), value.to_owned()))
|
||||
.collect();
|
||||
assert_eq!(recorded_multipart_owner(&metadata, BUCKET, ENCODED), Some(expected.clone()));
|
||||
}
|
||||
for fields in [
|
||||
vec![(RUSTFS, BUCKET)],
|
||||
vec![(RUSTFS_OBJECT, ENCODED)],
|
||||
vec![(RUSTFS, ""), (RUSTFS_OBJECT, ENCODED)],
|
||||
vec![(RUSTFS, BUCKET), (RUSTFS_OBJECT, "")],
|
||||
vec![(RUSTFS, "foreign"), (RUSTFS_OBJECT, ENCODED)],
|
||||
vec![(RUSTFS, BUCKET), (RUSTFS_OBJECT, "directory/")],
|
||||
[rustfs, [(MINIO, "foreign"), (MINIO_OBJECT, ENCODED)]].concat(),
|
||||
[rustfs, [(MINIO, BUCKET), ("X-MINIO-INTERNAL-MULTIPART-OBJECT", "other")]].concat(),
|
||||
] {
|
||||
let metadata = fields
|
||||
.iter()
|
||||
.map(|(key, value)| (key.to_string(), value.to_string()))
|
||||
.collect();
|
||||
assert_eq!(recorded_multipart_owner(&metadata, BUCKET, ENCODED), None, "{fields:?}");
|
||||
}
|
||||
}
|
||||
|
||||
struct AbortGate {
|
||||
expected: Option<MultipartOwner>,
|
||||
accepted: bool,
|
||||
calls: AtomicUsize,
|
||||
}
|
||||
|
||||
impl MultipartAbortAuthorization for AbortGate {
|
||||
type Error = StorageError;
|
||||
|
||||
fn authorize(&self, owner: Option<&MultipartOwner>) -> Result<()> {
|
||||
self.calls.fetch_add(1, Ordering::Relaxed);
|
||||
assert_eq!(owner, self.expected.as_ref(), "gate sees the actual selected upload owner");
|
||||
if self.accepted {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(StorageError::InvalidUploadID(String::new(), String::new(), String::new()))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn abort_owner_callback_controls_physical_deletion() {
|
||||
let (_dirs, disks, set) = hermetic_set_disks(4).await;
|
||||
let bucket = "abort-locked-owner";
|
||||
let key = "staged-object";
|
||||
make_bucket_on_all(&disks, bucket).await;
|
||||
let (id, _) = stage_upload_with_create_opts(&set, bucket, key, b"staged part", &ObjectOptions::default()).await;
|
||||
let path = SetDisks::get_upload_id_dir(bucket, key, &id);
|
||||
let (fi, _) = set
|
||||
.check_upload_id_exists(bucket, key, &id, true)
|
||||
.await
|
||||
.expect("staged upload");
|
||||
let part = format!("{path}/{}/part.1", fi.data_dir.expect("staged data directory"));
|
||||
let expected = MultipartOwner {
|
||||
bucket: bucket.to_owned(),
|
||||
object: key.to_owned(),
|
||||
};
|
||||
let mut before = Vec::new();
|
||||
for disk in &disks {
|
||||
before.push(
|
||||
disk.read_all(RUSTFS_META_MULTIPART_BUCKET, &part)
|
||||
.await
|
||||
.expect("staged shard"),
|
||||
);
|
||||
}
|
||||
let rejected = Arc::new(AbortGate {
|
||||
expected: Some(expected.clone()),
|
||||
accepted: false,
|
||||
calls: AtomicUsize::new(0),
|
||||
});
|
||||
let opts = ObjectOptions {
|
||||
multipart_abort_authorization: Some(rejected.clone()),
|
||||
..Default::default()
|
||||
};
|
||||
let result = set.abort_multipart_upload(bucket, key, &id, &opts).await;
|
||||
for (disk, staged) in disks.iter().zip(before) {
|
||||
let after = disk
|
||||
.read_all(RUSTFS_META_MULTIPART_BUCKET, &part)
|
||||
.await
|
||||
.expect("denied Abort preserves the shard");
|
||||
assert_eq!(staged, after);
|
||||
}
|
||||
assert!(matches!(result, Err(StorageError::InvalidUploadID(..))));
|
||||
assert_eq!(rejected.calls.load(Ordering::Relaxed), 1);
|
||||
|
||||
let allowed = Arc::new(AbortGate {
|
||||
expected: Some(expected),
|
||||
accepted: true,
|
||||
calls: AtomicUsize::new(0),
|
||||
});
|
||||
let opts = ObjectOptions {
|
||||
multipart_abort_authorization: Some(allowed.clone()),
|
||||
..Default::default()
|
||||
};
|
||||
set.abort_multipart_upload(bucket, key, &id, &opts)
|
||||
.await
|
||||
.expect("owner permits Abort");
|
||||
assert_eq!(allowed.calls.load(Ordering::Relaxed), 1);
|
||||
for disk in &disks {
|
||||
assert!(matches!(
|
||||
disk.read_all(RUSTFS_META_MULTIPART_BUCKET, &part).await,
|
||||
Err(DiskError::FileNotFound)
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn abort_owner_comes_from_the_selected_metadata_on_all_disks() {
|
||||
use rustfs_utils::http::{get_str, remove_str};
|
||||
|
||||
let (_dirs, disks, set) = hermetic_set_disks(4).await;
|
||||
let bucket = "abort-owner-metadata";
|
||||
make_bucket_on_all(&disks, bucket).await;
|
||||
for (key, bucket_fields, object_fields, accepted) in [
|
||||
("rustfs", [Some(bucket), None], [Some("rustfs"), None], true),
|
||||
("minio", [None, Some(bucket)], [None, Some("minio")], true),
|
||||
("matching", [Some(bucket), Some(bucket)], [Some("matching"), Some("matching")], true),
|
||||
("empty", [Some(bucket), Some(bucket)], [Some("empty"), Some("")], false),
|
||||
("missing", [None, None], [Some("missing"), None], false),
|
||||
("foreign-bucket", [Some("foreign"), None], [Some("foreign-bucket"), None], false),
|
||||
("foreign-key", [Some(bucket), None], [Some("unrelated"), None], false),
|
||||
("conflict", [Some(bucket), Some("foreign")], [Some("conflict"), Some("conflict")], false),
|
||||
] {
|
||||
let (id, _) = stage_upload_with_create_opts(&set, bucket, key, b"staged shard data", &ObjectOptions::default()).await;
|
||||
let path = SetDisks::get_upload_id_dir(bucket, key, &id);
|
||||
pin_historical_unmarked_upload(&disks, bucket, &path).await;
|
||||
let mut part_path = None;
|
||||
let mut staged = Vec::new();
|
||||
let fields = [
|
||||
(RUSTFS_MULTIPART_BUCKET_KEY, bucket_fields[0]),
|
||||
("x-minio-internal-multipart-bucket", bucket_fields[1]),
|
||||
(RUSTFS_MULTIPART_OBJECT_KEY, object_fields[0]),
|
||||
("x-minio-internal-multipart-object", object_fields[1]),
|
||||
];
|
||||
for disk in &disks {
|
||||
let mut head = disk
|
||||
.read_version("", RUSTFS_META_MULTIPART_BUCKET, &path, "", &ReadOptions::default())
|
||||
.await
|
||||
.expect("read the staged metadata on each disk");
|
||||
let part = part_path.get_or_insert_with(|| format!("{path}/{}/part.1", head.data_dir.expect("part directory")));
|
||||
staged.push(
|
||||
disk.read_all(RUSTFS_META_MULTIPART_BUCKET, part)
|
||||
.await
|
||||
.expect("read staged part before Abort"),
|
||||
);
|
||||
assert_eq!(get_str(&head.metadata, "multipart-bucket").as_deref(), Some(bucket));
|
||||
remove_str(&mut head.metadata, "multipart-bucket");
|
||||
remove_str(&mut head.metadata, "multipart-object");
|
||||
for (name, value) in fields {
|
||||
if let Some(value) = value {
|
||||
head.metadata.insert(name.to_owned(), value.to_owned());
|
||||
}
|
||||
}
|
||||
disk.write_metadata(bucket, RUSTFS_META_MULTIPART_BUCKET, &path, head)
|
||||
.await
|
||||
.expect("write the test owner on each disk");
|
||||
let persisted = disk
|
||||
.read_version("", RUSTFS_META_MULTIPART_BUCKET, &path, "", &ReadOptions::default())
|
||||
.await
|
||||
.expect("verify the actual persisted metadata");
|
||||
for (name, value) in fields {
|
||||
assert_eq!(persisted.metadata.get(name).map(String::as_str), value);
|
||||
}
|
||||
}
|
||||
let gate = Arc::new(AbortGate {
|
||||
expected: accepted.then(|| MultipartOwner {
|
||||
bucket: bucket.to_owned(),
|
||||
object: key.to_owned(),
|
||||
}),
|
||||
accepted,
|
||||
calls: AtomicUsize::new(0),
|
||||
});
|
||||
let opts = ObjectOptions {
|
||||
multipart_abort_authorization: Some(gate.clone()),
|
||||
..Default::default()
|
||||
};
|
||||
let result = set.abort_multipart_upload(bucket, key, &id, &opts).await;
|
||||
for (disk, previous) in disks.iter().zip(staged) {
|
||||
let after = disk
|
||||
.read_all(RUSTFS_META_MULTIPART_BUCKET, part_path.as_deref().expect("part path"))
|
||||
.await;
|
||||
if accepted {
|
||||
assert!(matches!(after, Err(DiskError::FileNotFound)), "{key}: authorized data removed");
|
||||
} else {
|
||||
assert_eq!(after.expect("refused Abort preserves the staged part"), previous, "{key}");
|
||||
}
|
||||
}
|
||||
let calls = gate.calls.load(Ordering::Relaxed);
|
||||
if accepted {
|
||||
assert_eq!(calls, 1, "{key}");
|
||||
result.expect("Abort with agreeing persisted owner");
|
||||
} else {
|
||||
assert!(result.is_err(), "{key}: invalid metadata must not authorize Abort");
|
||||
if key == "empty" {
|
||||
// The existing source inspection refuses this head before selecting an owner.
|
||||
assert_eq!(calls, 0, "malformed physical owner was refused before the callback");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -52,9 +52,10 @@ pub(crate) mod list {
|
||||
}
|
||||
|
||||
pub(crate) mod multipart {
|
||||
pub(crate) use rustfs_storage_api::multipart::ListPartsOwner;
|
||||
pub(crate) use rustfs_storage_api::{
|
||||
CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartCompletionAuthorization, MultipartInfo, MultipartOperations,
|
||||
MultipartUploadInspection, MultipartUploadResult, PartInfo,
|
||||
CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartAbortAuthorization, MultipartCompletionAuthorization,
|
||||
MultipartInfo, MultipartOperations, MultipartOwner, MultipartUploadInspection, MultipartUploadResult, PartInfo,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -6825,6 +6825,28 @@ mod tests {
|
||||
tokio::fs::remove_file(conflicting_metadata)
|
||||
.await
|
||||
.expect("remove conflicting fixture");
|
||||
struct RequiresOwner;
|
||||
impl crate::storage_api_contracts::multipart::MultipartAbortAuthorization for RequiresOwner {
|
||||
type Error = StorageError;
|
||||
|
||||
fn authorize(&self, _: Option<&crate::storage_api_contracts::multipart::MultipartOwner>) -> Result<()> {
|
||||
panic!("metadata-absent recovery cannot observe an upload owner");
|
||||
}
|
||||
}
|
||||
let native_opts = ObjectOptions {
|
||||
multipart_abort_authorization: Some(Arc::new(RequiresOwner)),
|
||||
..Default::default()
|
||||
};
|
||||
let rejected = store
|
||||
.abort_multipart_upload(&bucket, object, &upload.upload_id, &native_opts)
|
||||
.await;
|
||||
for disk_index in 0..3 {
|
||||
assert!(
|
||||
staging_dir(0, disk_index).join("part.1").exists(),
|
||||
"owner-required Abort must preserve late data"
|
||||
);
|
||||
}
|
||||
assert!(matches!(rejected, Err(StorageError::InvalidUploadID(..))));
|
||||
let (paused, release) = crate::store::multipart::pause_orphan_multipart_cleanup_for_test(&bucket);
|
||||
let recovery_store = store.clone();
|
||||
let recovery_bucket = bucket.clone();
|
||||
|
||||
@@ -966,7 +966,8 @@ impl ECStore {
|
||||
bucket_lifecycle_guard: Option<super::BucketIncarnationFenceGuard>,
|
||||
) -> Result<()> {
|
||||
let invalid_upload = || StorageError::InvalidUploadID(bucket.to_owned(), object.to_owned(), upload_id.to_owned());
|
||||
if self.ctx.lock_manager().is_disabled() {
|
||||
// Without upload metadata this path has no recorded owner to authorize.
|
||||
if opts.multipart_abort_authorization.is_some() || self.ctx.lock_manager().is_disabled() {
|
||||
return Err(invalid_upload());
|
||||
}
|
||||
let Some(bucket_lifecycle_guard) = bucket_lifecycle_guard else {
|
||||
|
||||
@@ -866,6 +866,74 @@ where
|
||||
}
|
||||
|
||||
let _mutation_guard = self.cache.service_account_mutation_lock().lock().await;
|
||||
self.save_new_service_account_locked(cred, updated_at).await
|
||||
}
|
||||
|
||||
/// Order a cross-user issue with changes to either identity. Parent locks
|
||||
/// are acquired by name, then the service-account lock, then the caller's
|
||||
/// STS lock, matching user updates. Hold them through durable publication.
|
||||
pub(crate) async fn add_delegated_service_account(&self, cred: &Credentials, caller: &Credentials) -> Result<OffsetDateTime> {
|
||||
use subtle::ConstantTimeEq;
|
||||
|
||||
let target = cred.parent_user.as_str();
|
||||
let principal = if caller.parent_user.is_empty() {
|
||||
caller.access_key.as_str()
|
||||
} else {
|
||||
caller.parent_user.as_str()
|
||||
};
|
||||
if target == principal || target == caller.access_key || crate::is_root_access_key(target) {
|
||||
return Err(Error::IAMActionNotAllowed);
|
||||
}
|
||||
let mut parents = [principal, target];
|
||||
parents.sort_unstable();
|
||||
let _first = self.cache.parent_mutation_lock(parents[0]).lock_owned().await;
|
||||
let _second = self.cache.parent_mutation_lock(parents[1]).lock_owned().await;
|
||||
let _service = self.cache.service_account_mutation_lock().lock().await;
|
||||
let _session = if !caller.session_token.is_empty() && !caller.is_service_account() {
|
||||
Some(self.cache.sts_account_mutation_lock(&caller.access_key).lock_owned().await)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let active_local_user = |user: &UserIdentity| {
|
||||
user.credentials.is_valid() && user.credentials.parent_user.is_empty() && user.credentials.session_token.is_empty()
|
||||
};
|
||||
let snapshot = self.cache.snapshot();
|
||||
if !snapshot.users.get(target).is_some_and(active_local_user) {
|
||||
return Err(Error::IAMActionNotAllowed);
|
||||
}
|
||||
let root = crate::root_credentials::credentials().ok_or(Error::IAMActionNotAllowed)?;
|
||||
let current = if caller.access_key == root.access_key {
|
||||
&root
|
||||
} else {
|
||||
&snapshot
|
||||
.users
|
||||
.get(&caller.access_key)
|
||||
.or_else(|| snapshot.sts_accounts.get(&caller.access_key))
|
||||
.ok_or(Error::IAMActionNotAllowed)?
|
||||
.credentials
|
||||
};
|
||||
if !current.is_valid()
|
||||
|| current.parent_user != caller.parent_user
|
||||
|| !bool::from(current.secret_key.as_bytes().ct_eq(caller.secret_key.as_bytes()))
|
||||
|| !bool::from(current.session_token.as_bytes().ct_eq(caller.session_token.as_bytes()))
|
||||
{
|
||||
return Err(Error::IAMActionNotAllowed);
|
||||
}
|
||||
if !caller.parent_user.is_empty()
|
||||
&& principal != root.access_key
|
||||
&& !crate::sys::is_rustfs_oidc_claims(caller.claims_or_empty())
|
||||
&& !snapshot.users.get(principal).is_some_and(active_local_user)
|
||||
{
|
||||
return Err(Error::IAMActionNotAllowed);
|
||||
}
|
||||
drop(snapshot);
|
||||
self.save_new_service_account_locked(cred.clone(), OffsetDateTime::now_utc())
|
||||
.await
|
||||
}
|
||||
|
||||
/// The caller holds the service-account mutation lock through cache publication.
|
||||
async fn save_new_service_account_locked(&self, cred: Credentials, updated_at: OffsetDateTime) -> Result<OffsetDateTime> {
|
||||
let cache = self.cache.snapshot();
|
||||
if cache.users.contains_key(&cred.access_key) || cache.sts_accounts.contains_key(&cred.access_key) {
|
||||
return Err(Error::AccessKeyAlreadyExists);
|
||||
|
||||
+321
-8
@@ -960,6 +960,19 @@ impl<T: Store> IamSys<T> {
|
||||
opts: NewServiceAccountOpts,
|
||||
updated_at: OffsetDateTime,
|
||||
) -> Result<(Credentials, OffsetDateTime)> {
|
||||
let cred = Self::service_account_credentials(parent_user, groups, opts)?;
|
||||
let create_at = self.store.add_service_account_at(cred.clone(), updated_at).await?;
|
||||
|
||||
self.notify_for_service_account(&cred.access_key).await;
|
||||
|
||||
Ok((cred, create_at))
|
||||
}
|
||||
|
||||
fn service_account_credentials(
|
||||
parent_user: &str,
|
||||
groups: Option<Vec<String>>,
|
||||
opts: NewServiceAccountOpts,
|
||||
) -> Result<Credentials> {
|
||||
if !opts.access_key.is_empty()
|
||||
&& (opts.access_key.trim() == "."
|
||||
|| opts.access_key.trim() == ".."
|
||||
@@ -1046,11 +1059,7 @@ impl<T: Store> IamSys<T> {
|
||||
cred.name = opts.name;
|
||||
cred.description = opts.description;
|
||||
|
||||
let create_at = self.store.add_service_account_at(cred.clone(), updated_at).await?;
|
||||
|
||||
self.notify_for_service_account(&cred.access_key).await;
|
||||
|
||||
Ok((cred, create_at))
|
||||
Ok(cred)
|
||||
}
|
||||
|
||||
pub async fn update_service_account(&self, name: &str, opts: UpdateServiceAccountOpts) -> Result<OffsetDateTime> {
|
||||
@@ -1689,6 +1698,80 @@ impl<T: Store> IamSys<T> {
|
||||
Ok(Some(verified_claims))
|
||||
}
|
||||
|
||||
/// Issue a credential for a different local user. Both actions must allow the
|
||||
/// authenticated caller and the same resolved target. Once admitted, the
|
||||
/// write continues if the caller stops waiting.
|
||||
///
|
||||
/// Returns an error if the caller lacks either grant, the target is missing
|
||||
/// or ineligible, the caller has been revoked, or the key cannot be saved.
|
||||
pub async fn new_service_account_for_user(
|
||||
&self,
|
||||
target: &str,
|
||||
opts: NewServiceAccountOpts,
|
||||
caller: &Credentials,
|
||||
conditions: &HashMap<String, Vec<String>>,
|
||||
authenticated_owner: bool,
|
||||
) -> Result<(Credentials, OffsetDateTime, HashMap<String, Value>)> {
|
||||
use rustfs_policy::policy::{
|
||||
SVC_TARGET_USER,
|
||||
action::{Action, AdminAction},
|
||||
};
|
||||
|
||||
if target.is_empty() || crate::is_root_access_key(target) || target == caller.access_key || target == caller.parent_user {
|
||||
return Err(IamError::IAMActionNotAllowed);
|
||||
}
|
||||
// Bind the policy decision to the identity that will be written.
|
||||
if !conditions
|
||||
.get(SVC_TARGET_USER)
|
||||
.is_some_and(|values| values.len() == 1 && values[0] == target)
|
||||
{
|
||||
return Err(IamError::IAMActionNotAllowed);
|
||||
}
|
||||
let mut args = Args {
|
||||
account: &caller.access_key,
|
||||
groups: &caller.groups,
|
||||
action: Action::AdminAction(AdminAction::CreateServiceAccountAdminAction),
|
||||
bucket: "",
|
||||
conditions,
|
||||
is_owner: authenticated_owner,
|
||||
object: "",
|
||||
claims: caller.claims_or_empty(),
|
||||
deny_only: false,
|
||||
};
|
||||
// Both built-in decisions must use the same identity and policy revision.
|
||||
let prepared = self.prepare_auth(&args).await;
|
||||
if !self.eval_prepared(&prepared, &args).await {
|
||||
return Err(IamError::IAMActionNotAllowed);
|
||||
}
|
||||
args.action = Action::AdminAction(AdminAction::CreateServiceAccountForUserAdminAction);
|
||||
if !self.eval_prepared(&prepared, &args).await {
|
||||
return Err(IamError::IAMActionNotAllowed);
|
||||
}
|
||||
if self.check_key(target).await?.0.is_none() {
|
||||
return Err(IamError::NoSuchUser(target.to_owned()));
|
||||
}
|
||||
|
||||
// A delegated credential belongs entirely to its local target. Only
|
||||
// its requested session policy and expiration constrain that identity.
|
||||
let opts = NewServiceAccountOpts {
|
||||
claims: None,
|
||||
allow_site_replicator_account: false,
|
||||
..opts
|
||||
};
|
||||
let cred = Self::service_account_credentials(target, None, opts)?;
|
||||
let iam = self.clone();
|
||||
let caller = caller.clone();
|
||||
let runtime = tokio::runtime::Handle::try_current().map_err(Error::other)?;
|
||||
runtime
|
||||
.spawn(async move {
|
||||
let updated_at = iam.store.add_delegated_service_account(&cred, &caller).await?;
|
||||
iam.notify_for_service_account(&cred.access_key).await;
|
||||
Ok((cred, updated_at, HashMap::new()))
|
||||
})
|
||||
.await
|
||||
.map_err(Error::other)?
|
||||
}
|
||||
|
||||
/// Create within the authenticated caller's scope and current parent revision.
|
||||
/// An admitted write owns its parent lock until publication even if the caller
|
||||
/// stops waiting; errors never publish a credential from a changed parent.
|
||||
@@ -1719,8 +1802,7 @@ impl<T: Store> IamSys<T> {
|
||||
mut opts: NewServiceAccountOpts,
|
||||
caller: &Credentials,
|
||||
) -> Result<(Credentials, OffsetDateTime, HashMap<String, Value>)> {
|
||||
if parent_user != caller.access_key && parent_user != caller.parent_user && !crate::is_root_access_key(&caller.access_key)
|
||||
{
|
||||
if parent_user != caller.access_key && parent_user != caller.parent_user {
|
||||
return Err(IamError::IAMActionNotAllowed);
|
||||
}
|
||||
|
||||
@@ -3347,6 +3429,7 @@ mod tests {
|
||||
/// When true, parent user has no groups and no mapped policies (empty `policy_db_get`).
|
||||
empty_policies: bool,
|
||||
bucket_tags: Arc<Mutex<Result<HashMap<String, String>>>>,
|
||||
policy_load_barrier: Option<Arc<tokio::sync::Barrier>>,
|
||||
saved_sts_users: Arc<Mutex<HashMap<String, UserIdentity>>>,
|
||||
saved_service_account_count: Arc<Mutex<usize>>,
|
||||
fail_delete: Arc<std::sync::atomic::AtomicBool>,
|
||||
@@ -3369,6 +3452,7 @@ mod tests {
|
||||
fn new(empty_policies: bool) -> Self {
|
||||
Self {
|
||||
empty_policies,
|
||||
policy_load_barrier: None,
|
||||
bucket_tags: Arc::new(Mutex::new(Err(Error::other("bucket metadata unavailable")))),
|
||||
saved_sts_users: Arc::new(Mutex::new(HashMap::new())),
|
||||
saved_service_account_count: Arc::new(Mutex::new(0)),
|
||||
@@ -3561,7 +3645,14 @@ mod tests {
|
||||
Err(Error::InvalidArgument)
|
||||
}
|
||||
|
||||
async fn load_policy_doc(&self, _name: &str, _m: &mut HashMap<String, rustfs_policy::policy::PolicyDoc>) -> Result<()> {
|
||||
async fn load_policy_doc(&self, name: &str, docs: &mut HashMap<String, PolicyDoc>) -> Result<()> {
|
||||
if let Some(barrier) = &self.policy_load_barrier {
|
||||
// merge_policies has already captured the cached policy documents.
|
||||
barrier.wait().await;
|
||||
barrier.wait().await;
|
||||
docs.insert(name.to_owned(), PolicyDoc::new(Policy::default()));
|
||||
return Ok(());
|
||||
}
|
||||
Err(Error::InvalidArgument)
|
||||
}
|
||||
|
||||
@@ -4391,6 +4482,67 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delegated_service_account_holds_both_parents_and_session_through_publication() {
|
||||
use std::sync::atomic::Ordering;
|
||||
ensure_test_global_credentials();
|
||||
let store = StsTestMockStore::new(false);
|
||||
let iam = IamSys::new(IamCache::new(store.clone()).await.expect("initialize IAM"));
|
||||
let caller_parent = "operator";
|
||||
let target = "backup";
|
||||
for user in [caller_parent, target] {
|
||||
iam.create_user(
|
||||
user,
|
||||
&AddOrUpdateUserReq {
|
||||
secret_key: "originalSecret123".into(),
|
||||
status: rustfs_madmin::AccountStatus::Enabled,
|
||||
policy: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("create local user");
|
||||
}
|
||||
let session = rotation_test_session(caller_parent, "operator-console-session");
|
||||
iam.set_temp_user(&session.access_key, &session, None)
|
||||
.await
|
||||
.expect("publish STS session");
|
||||
store.block_derived_save.store(true, Ordering::SeqCst);
|
||||
let pending = IamSys::<StsTestMockStore>::service_account_credentials(target, None, NewServiceAccountOpts::default())
|
||||
.expect("prepare credential");
|
||||
let creation = {
|
||||
let iam = iam.clone();
|
||||
let session = session.clone();
|
||||
tokio::spawn(async move { iam.store.add_delegated_service_account(&pending, &session).await })
|
||||
};
|
||||
tokio::time::timeout(std::time::Duration::from_secs(5), store.derived_save_started.notified())
|
||||
.await
|
||||
.expect("credential write started");
|
||||
for principal in [caller_parent, target] {
|
||||
assert!(iam.store.cache.parent_mutation_lock(principal).try_lock().is_err());
|
||||
}
|
||||
assert!(
|
||||
iam.store
|
||||
.cache
|
||||
.sts_account_mutation_lock(&session.access_key)
|
||||
.try_lock()
|
||||
.is_err()
|
||||
);
|
||||
store.block_derived_save.store(false, Ordering::SeqCst);
|
||||
store.release_derived_save.notify_one();
|
||||
creation.await.expect("join writer").expect("publish credential");
|
||||
|
||||
iam.set_user_status(target, rustfs_madmin::AccountStatus::Disabled)
|
||||
.await
|
||||
.expect("disable target");
|
||||
let pending = IamSys::<StsTestMockStore>::service_account_credentials(target, None, NewServiceAccountOpts::default())
|
||||
.expect("prepare credential for disabled user");
|
||||
assert!(matches!(
|
||||
iam.store.add_delegated_service_account(&pending, &session).await,
|
||||
Err(Error::IAMActionNotAllowed)
|
||||
));
|
||||
assert_eq!(store.saved_service_account_count(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn revoked_sts_cannot_mint_service_account_between_rotation_and_sweep() {
|
||||
ensure_test_global_credentials();
|
||||
@@ -5332,6 +5484,167 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delegated_service_account_requires_both_actions_in_one_policy_revision() {
|
||||
use rustfs_policy::policy::SVC_TARGET_USER;
|
||||
|
||||
ensure_test_global_credentials();
|
||||
let barrier = Arc::new(tokio::sync::Barrier::new(2));
|
||||
let mut store = StsTestMockStore::new(true);
|
||||
store.policy_load_barrier = Some(barrier.clone());
|
||||
let cache = temp_env::async_with_vars([("RUSTFS_SKIP_BACKGROUND_TASK", Some("1"))], IamCache::new(store.clone()))
|
||||
.await
|
||||
.expect("initialize IAM");
|
||||
let iam = IamSys::new(cache);
|
||||
let operator = "backup-operator";
|
||||
let target = "backup-user";
|
||||
for name in [operator, target] {
|
||||
iam.create_user(
|
||||
name,
|
||||
&AddOrUpdateUserReq {
|
||||
secret_key: "testSecret123".into(),
|
||||
status: rustfs_madmin::AccountStatus::Enabled,
|
||||
policy: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("create local user");
|
||||
}
|
||||
let caller = iam.get_user(operator).await.expect("operator exists").credentials;
|
||||
let set_grant = |action: &str| {
|
||||
let policy = serde_json::json!({"Version": "2012-10-17", "Statement": [{"Effect": "Allow", "Action": action}]});
|
||||
let policy = Policy::parse_config(policy.to_string().as_bytes()).expect("parse administrator policy");
|
||||
iam.store
|
||||
.cache
|
||||
.add_or_update_policy_doc("operator-grant", &PolicyDoc::new(policy), OffsetDateTime::now_utc());
|
||||
};
|
||||
set_grant("admin:CreateServiceAccount");
|
||||
iam.store.cache.add_or_update_user_policy(
|
||||
operator,
|
||||
&MappedPolicy::new("operator-grant,uncached-policy"),
|
||||
OffsetDateTime::now_utc(),
|
||||
);
|
||||
let conditions = HashMap::from([(SVC_TARGET_USER.to_owned(), vec![target.to_owned()])]);
|
||||
let create = iam.new_service_account_for_user(target, NewServiceAccountOpts::default(), &caller, &conditions, false);
|
||||
let replace = async {
|
||||
barrier.wait().await;
|
||||
set_grant("admin:CreateServiceAccountForUser");
|
||||
barrier.wait().await;
|
||||
};
|
||||
let (result, ()) = tokio::time::timeout(std::time::Duration::from_secs(5), async { tokio::join!(create, replace) })
|
||||
.await
|
||||
.expect("finish lookup and concurrent policy replacement");
|
||||
assert!(matches!(result, Err(Error::IAMActionNotAllowed)));
|
||||
assert_eq!(store.saved_service_account_count(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delegated_service_account_distinguishes_missing_target_from_storage_failure() {
|
||||
use rustfs_policy::policy::SVC_TARGET_USER;
|
||||
|
||||
ensure_test_global_credentials();
|
||||
let store = StsTestMockStore::new(true);
|
||||
let iam = IamSys::new(IamCache::new(store.clone()).await.expect("initialize IAM"));
|
||||
let caller = crate::root_credentials::credentials().expect("root credentials");
|
||||
for target in ["missing-target", "load-failure-user"] {
|
||||
let conditions = HashMap::from([(SVC_TARGET_USER.to_owned(), vec![target.to_owned()])]);
|
||||
let error = iam
|
||||
.new_service_account_for_user(target, NewServiceAccountOpts::default(), &caller, &conditions, true)
|
||||
.await
|
||||
.expect_err("cannot publish without a target");
|
||||
match target {
|
||||
"missing-target" => assert!(matches!(error, Error::NoSuchUser(name) if name == target)),
|
||||
_ => assert!(matches!(error, Error::Io(_))),
|
||||
}
|
||||
}
|
||||
assert_eq!(store.saved_service_account_count(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delegated_service_account_uses_local_target_permissions_and_requires_live_oidc_session() {
|
||||
use rustfs_policy::policy::{
|
||||
SVC_TARGET_USER,
|
||||
action::{Action, AdminAction, S3Action},
|
||||
};
|
||||
|
||||
ensure_test_global_credentials();
|
||||
// This fixture seeds a policy mapping in memory. Its mock reload only
|
||||
// persists identities and would replace the mapping with its seed data.
|
||||
let cache =
|
||||
temp_env::async_with_vars([("RUSTFS_SKIP_BACKGROUND_TASK", Some("1"))], IamCache::new(StsTestMockStore::new(true)))
|
||||
.await
|
||||
.expect("initialize IAM");
|
||||
let iam = IamSys::new(cache);
|
||||
let target = "backup-user";
|
||||
iam.create_user(
|
||||
target,
|
||||
&AddOrUpdateUserReq {
|
||||
secret_key: "backupUserSecret123".into(),
|
||||
status: rustfs_madmin::AccountStatus::Enabled,
|
||||
policy: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("create backup user");
|
||||
iam.store
|
||||
.cache
|
||||
.add_or_update_user_policy(target, &MappedPolicy::new("readonly"), OffsetDateTime::now_utc());
|
||||
|
||||
let mut caller = rotation_test_session("oidc-admin-parent", "oidc-console-session");
|
||||
caller.claims = Some(HashMap::from([
|
||||
("iss".into(), Value::from("rustfs-oidc")),
|
||||
("oidc_provider".into(), Value::from("default")),
|
||||
("sub".into(), Value::from("operator")),
|
||||
("parent".into(), Value::from(caller.parent_user.clone())),
|
||||
(OIDC_VIRTUAL_PARENT_CLAIM.into(), Value::from(caller.parent_user.clone())),
|
||||
(POLICYNAME.into(), Value::from("consoleAdmin")),
|
||||
]));
|
||||
iam.set_temp_user(&caller.access_key, &caller, None)
|
||||
.await
|
||||
.expect("publish OIDC session");
|
||||
let conditions = HashMap::from([(SVC_TARGET_USER.to_owned(), vec![target.to_owned()])]);
|
||||
let (created, _, replicated_claims) = iam
|
||||
.new_service_account_for_user(target, NewServiceAccountOpts::default(), &caller, &conditions, false)
|
||||
.await
|
||||
.expect("delegate to local user");
|
||||
assert_eq!(created.parent_user, target);
|
||||
assert!(replicated_claims.is_empty());
|
||||
let key = iam
|
||||
.get_user(&created.access_key)
|
||||
.await
|
||||
.expect("published service account")
|
||||
.credentials;
|
||||
assert!(!is_rustfs_oidc_claims(key.claims_or_empty()));
|
||||
let mut args = Args {
|
||||
account: &key.access_key,
|
||||
groups: &key.groups,
|
||||
action: Action::S3Action(S3Action::GetObjectAction),
|
||||
bucket: "backups",
|
||||
object: "archive",
|
||||
conditions: &HashMap::new(),
|
||||
claims: key.claims_or_empty(),
|
||||
is_owner: false,
|
||||
deny_only: false,
|
||||
};
|
||||
assert!(iam.is_allowed(&args).await);
|
||||
args.action = Action::S3Action(S3Action::PutObjectAction);
|
||||
assert!(!iam.is_allowed(&args).await);
|
||||
args.action = Action::AdminAction(AdminAction::CreateServiceAccountForUserAdminAction);
|
||||
assert!(!iam.is_allowed(&args).await);
|
||||
|
||||
iam.store
|
||||
.delete_user(&caller.access_key, UserType::Sts)
|
||||
.await
|
||||
.expect("revoke session");
|
||||
let pending = IamSys::<StsTestMockStore>::service_account_credentials(target, None, NewServiceAccountOpts::default())
|
||||
.expect("prepare credential");
|
||||
assert!(matches!(
|
||||
iam.store.add_delegated_service_account(&pending, &caller).await,
|
||||
Err(Error::IAMActionNotAllowed)
|
||||
));
|
||||
assert!(iam.store.get_user(&pending.access_key).await.is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn oidc_service_account_uses_verified_policy_and_persisted_boundary() {
|
||||
ensure_test_global_credentials();
|
||||
|
||||
@@ -30,6 +30,7 @@ harness = false
|
||||
|
||||
[features]
|
||||
default = []
|
||||
diagnostic-internode-http2-ping = []
|
||||
diagnostic-read-version-caller = []
|
||||
hotpath = [
|
||||
"hotpath/hotpath",
|
||||
|
||||
@@ -433,6 +433,13 @@ const INTERNODE_OPERATION_DURATION_MS: &str = "rustfs_system_network_internode_o
|
||||
const INTERNODE_OPERATION_SERVER_DURATION_MS: &str = "rustfs_system_network_internode_operation_server_duration_ms";
|
||||
const INTERNODE_RPC_ACTIVE: &str = "rustfs_system_network_internode_rpc_active";
|
||||
const INTERNODE_RPC_COMPLETIONS_TOTAL: &str = "rustfs_system_network_internode_rpc_completions_total";
|
||||
const INTERNODE_CHANNEL_REBUILD_ATTEMPTS_TOTAL: &str = "rustfs_system_network_internode_channel_rebuild_attempts_total";
|
||||
#[cfg(feature = "diagnostic-internode-http2-ping")]
|
||||
const INTERNODE_HTTP2_PING_TIMEOUTS_TOTAL: &str = "rustfs_system_network_internode_http2_ping_timeouts_total";
|
||||
const INTERNODE_BULK_BODY_WAIT_DURATION_MS: &str = "rustfs_system_network_internode_bulk_body_wait_duration_ms";
|
||||
const INTERNODE_BULK_BODY_WAITERS: &str = "rustfs_system_network_internode_bulk_body_waiters";
|
||||
const INTERNODE_BULK_BODY_ACTIVE: &str = "rustfs_system_network_internode_bulk_body_active";
|
||||
const INTERNODE_BULK_BODY_ADMISSIONS_TOTAL: &str = "rustfs_system_network_internode_bulk_body_admissions_total";
|
||||
const INTERNODE_OPERATION_STAGE_DURATION_MS: &str = "rustfs_system_network_internode_operation_stage_duration_ms";
|
||||
const INTERNODE_OPERATION_CLASSIFIED_ERRORS_TOTAL: &str = "rustfs_system_network_internode_operation_classified_errors_total";
|
||||
const INTERNODE_OPERATION_RETRIES_TOTAL: &str = "rustfs_system_network_internode_operation_retries_total";
|
||||
@@ -447,8 +454,26 @@ const INTERNODE_OPERATION_LARGE_PAYLOADS_TOTAL: &str = "rustfs_system_network_in
|
||||
const INTERNODE_MSGPACK_JSON_DECODE_TOTAL: &str = "rustfs_system_network_internode_msgpack_json_decode_total";
|
||||
const INTERNODE_MSGPACK_JSON_FALLBACK_TOTAL: &str = "rustfs_system_network_internode_msgpack_json_fallback_total";
|
||||
const INTERNODE_MSGPACK_JSON_DECODE_ERROR_TOTAL: &str = "rustfs_system_network_internode_msgpack_json_decode_error_total";
|
||||
const INTERNODE_MSGPACK_SEND_DECISIONS_TOTAL: &str = "rustfs_system_network_internode_msgpack_send_decisions_total";
|
||||
const INTERNODE_SIGNATURE_V1_FALLBACK_TOTAL: &str = "rustfs_system_network_internode_signature_v1_fallback_total";
|
||||
const INTERNODE_BODY_DIGEST_FALLBACK_TOTAL: &str = "rustfs_system_network_internode_body_digest_fallback_total";
|
||||
|
||||
#[cfg(feature = "diagnostic-internode-http2-ping")]
|
||||
tokio::task_local! {
|
||||
static INTERNODE_HTTP2_CLIENT_TASK: ();
|
||||
}
|
||||
|
||||
/// Scope a tonic/Hyper background task as an internode HTTP/2 client task.
|
||||
#[cfg(feature = "diagnostic-internode-http2-ping")]
|
||||
pub async fn scope_internode_http2_client_task<T>(future: impl std::future::Future<Output = T>) -> T {
|
||||
INTERNODE_HTTP2_CLIENT_TASK.scope((), future).await
|
||||
}
|
||||
|
||||
/// Whether the current task belongs to an internode tonic/Hyper client.
|
||||
#[cfg(feature = "diagnostic-internode-http2-ping")]
|
||||
pub fn is_internode_http2_client_task() -> bool {
|
||||
INTERNODE_HTTP2_CLIENT_TASK.try_with(|()| ()).is_ok()
|
||||
}
|
||||
const INTERNODE_REPLAY_SCOPE_FALLBACK_TOTAL: &str = "rustfs_system_network_internode_replay_scope_fallback_total";
|
||||
const INTERNODE_REPLAY_CACHE_OVERFLOW_TOTAL: &str = "rustfs_system_network_internode_replay_cache_overflow_total";
|
||||
const INTERNODE_REPLAY_CACHE_OVERFLOW_BY_OPERATION_TOTAL: &str =
|
||||
@@ -468,6 +493,8 @@ pub struct InternodeOperationMetricDescriptor {
|
||||
const SERVER_OPERATION_BACKEND_LABELS: &[&str] = &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL];
|
||||
const SERVER_OPERATION_BACKEND_SIDE_LABELS: &[&str] = &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL, "side"];
|
||||
const SERVER_OPERATION_BACKEND_OUTCOME_LABELS: &[&str] = &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL, "outcome"];
|
||||
const SERVER_CLASS_REASON_OUTCOME_LABELS: &[&str] = &[SERVER_LABEL, "class", REASON_LABEL, "outcome"];
|
||||
const SERVER_OUTCOME_LABELS: &[&str] = &[SERVER_LABEL, "outcome"];
|
||||
const SERVER_OPERATION_BACKEND_CLASSIFICATION_LABELS: &[&str] =
|
||||
&[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL, CLASSIFICATION_LABEL];
|
||||
const SERVER_OPERATION_BACKEND_HTTP_VERSION_LABELS: &[&str] = &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL, HTTP_VERSION_LABEL];
|
||||
@@ -580,6 +607,26 @@ pub const INTERNODE_OPERATION_METRICS: &[InternodeOperationMetricDescriptor] = &
|
||||
name: INTERNODE_RPC_COMPLETIONS_TOTAL,
|
||||
labels: SERVER_OPERATION_BACKEND_OUTCOME_LABELS,
|
||||
},
|
||||
InternodeOperationMetricDescriptor {
|
||||
name: INTERNODE_CHANNEL_REBUILD_ATTEMPTS_TOTAL,
|
||||
labels: SERVER_CLASS_REASON_OUTCOME_LABELS,
|
||||
},
|
||||
InternodeOperationMetricDescriptor {
|
||||
name: INTERNODE_BULK_BODY_WAIT_DURATION_MS,
|
||||
labels: SERVER_LABELS,
|
||||
},
|
||||
InternodeOperationMetricDescriptor {
|
||||
name: INTERNODE_BULK_BODY_WAITERS,
|
||||
labels: SERVER_LABELS,
|
||||
},
|
||||
InternodeOperationMetricDescriptor {
|
||||
name: INTERNODE_BULK_BODY_ACTIVE,
|
||||
labels: SERVER_LABELS,
|
||||
},
|
||||
InternodeOperationMetricDescriptor {
|
||||
name: INTERNODE_BULK_BODY_ADMISSIONS_TOTAL,
|
||||
labels: SERVER_OUTCOME_LABELS,
|
||||
},
|
||||
];
|
||||
|
||||
static STABLE_SERVER_LABEL: OnceLock<String> = OnceLock::new();
|
||||
@@ -752,6 +799,169 @@ pub async fn measure_client_rpc<T, E>(
|
||||
result
|
||||
}
|
||||
|
||||
/// Record one attempt to publish a replacement after a bounded cached-channel eviction.
|
||||
pub fn record_channel_rebuild_attempt(class: &'static str, reason: &'static str, connected: bool) {
|
||||
if !crate::metrics_enabled() {
|
||||
return;
|
||||
}
|
||||
let class = match class {
|
||||
"control" | "bulk" | "recovery" => class,
|
||||
_ => "unknown",
|
||||
};
|
||||
let reason = match reason {
|
||||
"rpc_failure"
|
||||
| "operation_timeout"
|
||||
| "transport_failure"
|
||||
| "lock_timeout"
|
||||
| "lock_transport"
|
||||
| "peer_probe"
|
||||
| "tls_generation_changed"
|
||||
| "manual" => reason,
|
||||
_ => "unknown",
|
||||
};
|
||||
metrics::counter!(
|
||||
INTERNODE_CHANNEL_REBUILD_ATTEMPTS_TOTAL,
|
||||
SERVER_LABEL => current_server_label(),
|
||||
"class" => class,
|
||||
REASON_LABEL => reason,
|
||||
"outcome" => if connected { "connected" } else { "failed_or_cancelled" }
|
||||
)
|
||||
.increment(1);
|
||||
}
|
||||
|
||||
/// Record one Hyper-confirmed HTTP/2 keepalive PING ACK timeout.
|
||||
#[cfg(feature = "diagnostic-internode-http2-ping")]
|
||||
pub fn record_http2_ping_timeout() {
|
||||
if !crate::metrics_enabled() {
|
||||
return;
|
||||
}
|
||||
metrics::counter!(
|
||||
INTERNODE_HTTP2_PING_TIMEOUTS_TOTAL,
|
||||
SERVER_LABEL => current_server_label()
|
||||
)
|
||||
.increment(1);
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct BulkBodyAdmissionMetricHandles {
|
||||
wait_duration: metrics::Histogram,
|
||||
waiters: metrics::Gauge,
|
||||
active: metrics::Gauge,
|
||||
admitted: metrics::Counter,
|
||||
cancelled: metrics::Counter,
|
||||
timed_out: metrics::Counter,
|
||||
}
|
||||
|
||||
impl BulkBodyAdmissionMetricHandles {
|
||||
fn new() -> Self {
|
||||
let server = current_server_label();
|
||||
Self {
|
||||
wait_duration: metrics::histogram!(INTERNODE_BULK_BODY_WAIT_DURATION_MS, SERVER_LABEL => server),
|
||||
waiters: metrics::gauge!(INTERNODE_BULK_BODY_WAITERS, SERVER_LABEL => server),
|
||||
active: metrics::gauge!(INTERNODE_BULK_BODY_ACTIVE, SERVER_LABEL => server),
|
||||
admitted: metrics::counter!(
|
||||
INTERNODE_BULK_BODY_ADMISSIONS_TOTAL,
|
||||
SERVER_LABEL => server,
|
||||
"outcome" => "admitted"
|
||||
),
|
||||
cancelled: metrics::counter!(
|
||||
INTERNODE_BULK_BODY_ADMISSIONS_TOTAL,
|
||||
SERVER_LABEL => server,
|
||||
"outcome" => "cancelled"
|
||||
),
|
||||
timed_out: metrics::counter!(
|
||||
INTERNODE_BULK_BODY_ADMISSIONS_TOTAL,
|
||||
SERVER_LABEL => server,
|
||||
"outcome" => "timeout"
|
||||
),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(test))]
|
||||
static BULK_BODY_ADMISSION_METRICS: LazyLock<BulkBodyAdmissionMetricHandles> = LazyLock::new(BulkBodyAdmissionMetricHandles::new);
|
||||
|
||||
#[derive(Clone, Copy, Eq, PartialEq)]
|
||||
enum BulkBodyAdmissionState {
|
||||
Waiting,
|
||||
Active,
|
||||
Finished,
|
||||
}
|
||||
|
||||
/// Tracks one wait for complete-unary admission and, after admission, keeps the active gauge
|
||||
/// elevated until the decoded and validated response is released.
|
||||
pub struct InternodeBulkBodyAdmissionGuard {
|
||||
started_at: Option<Instant>,
|
||||
handles: Option<BulkBodyAdmissionMetricHandles>,
|
||||
state: BulkBodyAdmissionState,
|
||||
}
|
||||
|
||||
impl InternodeBulkBodyAdmissionGuard {
|
||||
pub fn waiting() -> Self {
|
||||
#[cfg(not(test))]
|
||||
let handles =
|
||||
(crate::metrics_enabled() && STABLE_SERVER_LABEL.get().is_some()).then(|| BULK_BODY_ADMISSION_METRICS.clone());
|
||||
#[cfg(test)]
|
||||
let handles = crate::metrics_enabled().then(BulkBodyAdmissionMetricHandles::new);
|
||||
if let Some(handles) = &handles {
|
||||
handles.waiters.increment(1.0);
|
||||
}
|
||||
Self {
|
||||
started_at: handles.as_ref().map(|_| Instant::now()),
|
||||
handles,
|
||||
state: BulkBodyAdmissionState::Waiting,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn admitted(&mut self) {
|
||||
if self.state == BulkBodyAdmissionState::Active {
|
||||
return;
|
||||
}
|
||||
if let Some(handles) = &self.handles {
|
||||
handles.waiters.decrement(1.0);
|
||||
handles.active.increment(1.0);
|
||||
handles.admitted.increment(1);
|
||||
if let Some(started_at) = self.started_at.take() {
|
||||
handles.wait_duration.record(started_at.elapsed().as_secs_f64() * 1000.0);
|
||||
}
|
||||
}
|
||||
self.state = BulkBodyAdmissionState::Active;
|
||||
}
|
||||
|
||||
pub fn timed_out(&mut self) {
|
||||
if self.state != BulkBodyAdmissionState::Waiting {
|
||||
return;
|
||||
}
|
||||
if let Some(handles) = &self.handles {
|
||||
handles.waiters.decrement(1.0);
|
||||
handles.timed_out.increment(1);
|
||||
if let Some(started_at) = self.started_at.take() {
|
||||
handles.wait_duration.record(started_at.elapsed().as_secs_f64() * 1000.0);
|
||||
}
|
||||
}
|
||||
self.state = BulkBodyAdmissionState::Finished;
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for InternodeBulkBodyAdmissionGuard {
|
||||
fn drop(&mut self) {
|
||||
let Some(handles) = &self.handles else {
|
||||
return;
|
||||
};
|
||||
match self.state {
|
||||
BulkBodyAdmissionState::Waiting => {
|
||||
handles.waiters.decrement(1.0);
|
||||
handles.cancelled.increment(1);
|
||||
if let Some(started_at) = self.started_at.take() {
|
||||
handles.wait_duration.record(started_at.elapsed().as_secs_f64() * 1000.0);
|
||||
}
|
||||
}
|
||||
BulkBodyAdmissionState::Active => handles.active.decrement(1.0),
|
||||
BulkBodyAdmissionState::Finished => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(test))]
|
||||
struct GrpcReadVersionMetricHandles {
|
||||
sent_bytes: metrics::Counter,
|
||||
@@ -1395,6 +1605,25 @@ impl InternodeMetrics {
|
||||
.increment(1);
|
||||
}
|
||||
|
||||
/// Record one eligible request's fixed-cardinality JSON/msgpack send decision.
|
||||
pub fn record_msgpack_send_decision(&self, decision: &'static str, reason: &'static str) {
|
||||
let decision = match decision {
|
||||
"msgpack_only" | "dual_write" => decision,
|
||||
_ => "dual_write",
|
||||
};
|
||||
let reason = match reason {
|
||||
"manual_fleet" | "peer_capable" | "peer_unknown" | "peer_revoked" | "not_requested" => reason,
|
||||
_ => "peer_unknown",
|
||||
};
|
||||
counter!(
|
||||
INTERNODE_MSGPACK_SEND_DECISIONS_TOTAL,
|
||||
SERVER_LABEL => current_server_label(),
|
||||
"decision" => decision,
|
||||
REASON_LABEL => reason
|
||||
)
|
||||
.increment(1);
|
||||
}
|
||||
|
||||
#[doc(hidden)]
|
||||
pub fn msgpack_json_decode_error_total_for_test(&self) -> u64 {
|
||||
self.msgpack_json_decode_error_total.load(Ordering::Relaxed)
|
||||
@@ -1849,6 +2078,80 @@ mod tests {
|
||||
assert_eq!(active_sides, HashSet::from(["client".to_owned(), "server".to_owned()]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transport_lifecycle_metrics_use_bounded_labels() {
|
||||
let recorder = DebuggingRecorder::new();
|
||||
let snapshotter = recorder.snapshotter();
|
||||
let previously_enabled = crate::metrics_enabled();
|
||||
crate::set_metrics_enabled(true);
|
||||
with_local_recorder(&recorder, || {
|
||||
record_channel_rebuild_attempt("bulk", "tls_generation_changed", true);
|
||||
record_channel_rebuild_attempt("control", "transport_failure", false);
|
||||
record_channel_rebuild_attempt("untrusted-class", "untrusted-reason", false);
|
||||
#[cfg(feature = "diagnostic-internode-http2-ping")]
|
||||
record_http2_ping_timeout();
|
||||
});
|
||||
crate::set_metrics_enabled(previously_enabled);
|
||||
let entries = snapshotter.snapshot().into_vec();
|
||||
let rebuild = entries
|
||||
.iter()
|
||||
.filter(|(key, _, _, _)| key.key().name() == INTERNODE_CHANNEL_REBUILD_ATTEMPTS_TOTAL)
|
||||
.count();
|
||||
assert_eq!(rebuild, 3);
|
||||
assert!(entries.iter().any(|(key, _, _, _)| {
|
||||
key.key().name() == INTERNODE_CHANNEL_REBUILD_ATTEMPTS_TOTAL
|
||||
&& key.key().labels().filter(|label| label.value() == "unknown").count() == 2
|
||||
}));
|
||||
#[cfg(feature = "diagnostic-internode-http2-ping")]
|
||||
assert!(entries.iter().any(|(key, _, _, value)| {
|
||||
key.key().name() == INTERNODE_HTTP2_PING_TIMEOUTS_TOTAL && matches!(value, DebugValue::Counter(1))
|
||||
}));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bulk_body_admission_metrics_balance_wait_cancel_and_active() {
|
||||
let recorder = DebuggingRecorder::new();
|
||||
let snapshotter = recorder.snapshotter();
|
||||
let previously_enabled = crate::metrics_enabled();
|
||||
crate::set_metrics_enabled(true);
|
||||
with_local_recorder(&recorder, || {
|
||||
drop(InternodeBulkBodyAdmissionGuard::waiting());
|
||||
let mut active = InternodeBulkBodyAdmissionGuard::waiting();
|
||||
active.admitted();
|
||||
drop(active);
|
||||
let mut timed_out = InternodeBulkBodyAdmissionGuard::waiting();
|
||||
timed_out.timed_out();
|
||||
drop(timed_out);
|
||||
});
|
||||
crate::set_metrics_enabled(previously_enabled);
|
||||
let entries = snapshotter.snapshot().into_vec();
|
||||
for name in [INTERNODE_BULK_BODY_WAITERS, INTERNODE_BULK_BODY_ACTIVE] {
|
||||
let value = entries
|
||||
.iter()
|
||||
.find(|(key, _, _, _)| key.key().name() == name)
|
||||
.map(|(_, _, _, value)| value)
|
||||
.expect("admission gauge");
|
||||
assert!(matches!(value, DebugValue::Gauge(value) if value.into_inner() == 0.0));
|
||||
}
|
||||
let outcomes = entries
|
||||
.iter()
|
||||
.filter(|(key, _, _, _)| key.key().name() == INTERNODE_BULK_BODY_ADMISSIONS_TOTAL)
|
||||
.map(|(key, _, _, value)| {
|
||||
assert!(matches!(value, DebugValue::Counter(1)));
|
||||
key.key()
|
||||
.labels()
|
||||
.find(|label| label.key() == "outcome")
|
||||
.expect("outcome")
|
||||
.value()
|
||||
.to_owned()
|
||||
})
|
||||
.collect::<HashSet<_>>();
|
||||
assert_eq!(
|
||||
outcomes,
|
||||
HashSet::from(["admitted".to_owned(), "cancelled".to_owned(), "timeout".to_owned()])
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn snapshot_reports_recorded_values() {
|
||||
let metrics = global_internode_metrics();
|
||||
@@ -2019,11 +2322,21 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn operation_metric_descriptors_include_backend_and_operation_labels() {
|
||||
assert_eq!(INTERNODE_OPERATION_METRICS.len(), 25);
|
||||
assert_eq!(INTERNODE_OPERATION_METRICS.len(), 30);
|
||||
assert_eq!(INTERNODE_OPERATION_METRICS[23].name, INTERNODE_RPC_ACTIVE);
|
||||
assert_eq!(INTERNODE_OPERATION_METRICS[23].labels, SERVER_OPERATION_BACKEND_SIDE_LABELS);
|
||||
assert_eq!(INTERNODE_OPERATION_METRICS[24].name, INTERNODE_RPC_COMPLETIONS_TOTAL);
|
||||
assert_eq!(INTERNODE_OPERATION_METRICS[24].labels, SERVER_OPERATION_BACKEND_OUTCOME_LABELS);
|
||||
assert_eq!(INTERNODE_OPERATION_METRICS[25].name, INTERNODE_CHANNEL_REBUILD_ATTEMPTS_TOTAL);
|
||||
assert_eq!(INTERNODE_OPERATION_METRICS[25].labels, SERVER_CLASS_REASON_OUTCOME_LABELS);
|
||||
assert_eq!(INTERNODE_OPERATION_METRICS[26].name, INTERNODE_BULK_BODY_WAIT_DURATION_MS);
|
||||
assert_eq!(INTERNODE_OPERATION_METRICS[26].labels, SERVER_LABELS);
|
||||
assert_eq!(INTERNODE_OPERATION_METRICS[27].name, INTERNODE_BULK_BODY_WAITERS);
|
||||
assert_eq!(INTERNODE_OPERATION_METRICS[27].labels, SERVER_LABELS);
|
||||
assert_eq!(INTERNODE_OPERATION_METRICS[28].name, INTERNODE_BULK_BODY_ACTIVE);
|
||||
assert_eq!(INTERNODE_OPERATION_METRICS[28].labels, SERVER_LABELS);
|
||||
assert_eq!(INTERNODE_OPERATION_METRICS[29].name, INTERNODE_BULK_BODY_ADMISSIONS_TOTAL);
|
||||
assert_eq!(INTERNODE_OPERATION_METRICS[29].labels, SERVER_OUTCOME_LABELS);
|
||||
for metric in &INTERNODE_OPERATION_METRICS[..6] {
|
||||
assert_eq!(metric.labels, &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL]);
|
||||
}
|
||||
@@ -2407,6 +2720,52 @@ mod tests {
|
||||
assert!(observed.contains(&(INTERNODE_MSGPACK_DIRECTION_RESPONSE.to_string(), "RawFileInfo".to_string(), 1)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn msgpack_send_decision_counter_keeps_fixed_decision_and_reason_labels() {
|
||||
let recorder = DebuggingRecorder::new();
|
||||
let snapshotter = recorder.snapshotter();
|
||||
let metrics = InternodeMetrics::default();
|
||||
metrics::with_local_recorder(&recorder, || {
|
||||
for (decision, reason) in [
|
||||
("msgpack_only", "manual_fleet"),
|
||||
("msgpack_only", "peer_capable"),
|
||||
("dual_write", "peer_unknown"),
|
||||
("dual_write", "peer_revoked"),
|
||||
("dual_write", "not_requested"),
|
||||
] {
|
||||
metrics.record_msgpack_send_decision(decision, reason);
|
||||
}
|
||||
metrics.record_msgpack_send_decision("untrusted", "untrusted");
|
||||
});
|
||||
|
||||
let observed: HashMap<(String, String), u64> = snapshotter
|
||||
.snapshot()
|
||||
.into_vec()
|
||||
.into_iter()
|
||||
.filter(|(composite, _, _, _)| composite.key().name() == INTERNODE_MSGPACK_SEND_DECISIONS_TOTAL)
|
||||
.map(|(composite, _, _, value)| {
|
||||
let labels: HashMap<String, String> = composite
|
||||
.key()
|
||||
.labels()
|
||||
.map(|label| (label.key().to_string(), label.value().to_string()))
|
||||
.collect();
|
||||
let count = match value {
|
||||
DebugValue::Counter(count) => count,
|
||||
other => panic!("msgpack send decision must be a counter, got {other:?}"),
|
||||
};
|
||||
(
|
||||
(
|
||||
labels.get("decision").cloned().unwrap_or_default(),
|
||||
labels.get(REASON_LABEL).cloned().unwrap_or_default(),
|
||||
),
|
||||
count,
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
assert_eq!(observed.len(), 5);
|
||||
assert_eq!(observed.get(&("dual_write".to_string(), "peer_unknown".to_string())), Some(&2));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn msgpack_json_decode_counter_tracks_codec_direction_and_message() {
|
||||
let metrics = InternodeMetrics::default();
|
||||
|
||||
@@ -433,8 +433,10 @@ pub const PUT_COMMIT_LOCK_ADMISSION_OUTCOME_LOCK_ERROR: &str = "lock_error";
|
||||
|
||||
pub const PUT_RENAME_FDATASYNC_BATCH_MODE_SERIAL: &str = "serial";
|
||||
pub const PUT_RENAME_FDATASYNC_BATCH_MODE_PARALLEL: &str = "parallel";
|
||||
pub const PUT_RENAME_FDATASYNC_BATCH_MODE_GROUPED: &str = "grouped";
|
||||
pub const PUT_RENAME_FDATASYNC_GROUP_WAIT_ROLE_LEADER: &str = "leader";
|
||||
pub const PUT_RENAME_FDATASYNC_GROUP_WAIT_ROLE_FOLLOWER: &str = "follower";
|
||||
pub const PUT_RENAME_FDATASYNC_GROUP_WAIT_ROLE_FOLLOWER_TIMEOUT: &str = "follower_timeout";
|
||||
pub const PUT_RENAME_FDATASYNC_GROUP_OUTSTANDING_STATE_ENQUEUE_WAITERS: &str = "enqueue_waiters";
|
||||
pub const PUT_RENAME_FDATASYNC_GROUP_OUTSTANDING_STATE_ENQUEUE_FILES: &str = "enqueue_files";
|
||||
pub const PUT_RENAME_FDATASYNC_GROUP_OUTSTANDING_STATE_BATCH_WAITERS: &str = "batch_waiters";
|
||||
@@ -4422,13 +4424,16 @@ mod tests {
|
||||
set_put_stage_metrics_enabled(false);
|
||||
record_put_rename_fdatasync_batch(PUT_RENAME_FDATASYNC_BATCH_MODE_SERIAL, 2);
|
||||
record_put_rename_fdatasync_group_wait(PUT_RENAME_FDATASYNC_GROUP_WAIT_ROLE_LEADER, 1.0);
|
||||
record_put_rename_fdatasync_group_wait(PUT_RENAME_FDATASYNC_GROUP_WAIT_ROLE_FOLLOWER_TIMEOUT, 3.0);
|
||||
record_put_rename_fdatasync_group_outstanding(PUT_RENAME_FDATASYNC_GROUP_OUTSTANDING_STATE_ENQUEUE_WAITERS, 2);
|
||||
record_put_rename_disk_wait_completion(PUT_RENAME_DISK_WAIT_COMPLETION_POSITION_QUORUM_FIRST, 3.0);
|
||||
record_put_rename_quorum_wait_fanout(4, 3, 3, 1, 0);
|
||||
|
||||
set_put_stage_metrics_enabled(true);
|
||||
record_put_rename_fdatasync_batch(PUT_RENAME_FDATASYNC_BATCH_MODE_PARALLEL, 9);
|
||||
record_put_rename_fdatasync_batch(PUT_RENAME_FDATASYNC_BATCH_MODE_GROUPED, 4);
|
||||
record_put_rename_fdatasync_group_wait(PUT_RENAME_FDATASYNC_GROUP_WAIT_ROLE_LEADER, 1.0);
|
||||
record_put_rename_fdatasync_group_wait(PUT_RENAME_FDATASYNC_GROUP_WAIT_ROLE_FOLLOWER_TIMEOUT, 3.0);
|
||||
record_put_rename_fdatasync_group_wait(PUT_RENAME_FDATASYNC_GROUP_WAIT_ROLE_FOLLOWER, 2.0);
|
||||
record_put_rename_fdatasync_group_outstanding(PUT_RENAME_FDATASYNC_GROUP_OUTSTANDING_STATE_ENQUEUE_WAITERS, 2);
|
||||
record_put_rename_fdatasync_group_outstanding(PUT_RENAME_FDATASYNC_GROUP_OUTSTANDING_STATE_ENQUEUE_FILES, 4);
|
||||
@@ -4442,7 +4447,10 @@ mod tests {
|
||||
});
|
||||
|
||||
let rows = snapshotter.snapshot().into_vec();
|
||||
assert_eq!(histogram_samples(&rows, "rustfs_s3_put_object_rename_fdatasync_batch_files"), vec![9.0]);
|
||||
assert_eq!(
|
||||
histogram_samples(&rows, "rustfs_s3_put_object_rename_fdatasync_batch_files"),
|
||||
vec![4.0, 9.0]
|
||||
);
|
||||
let batch_modes = rows
|
||||
.iter()
|
||||
.filter(|(composite, _, _, _)| {
|
||||
@@ -4458,13 +4466,19 @@ mod tests {
|
||||
.collect::<Vec<_>>()
|
||||
})
|
||||
.collect::<HashSet<_>>();
|
||||
assert_eq!(batch_modes, HashSet::from([PUT_RENAME_FDATASYNC_BATCH_MODE_PARALLEL.to_string()]));
|
||||
assert_eq!(
|
||||
batch_modes,
|
||||
HashSet::from([
|
||||
PUT_RENAME_FDATASYNC_BATCH_MODE_GROUPED.to_string(),
|
||||
PUT_RENAME_FDATASYNC_BATCH_MODE_PARALLEL.to_string(),
|
||||
])
|
||||
);
|
||||
|
||||
let quorum_samples = histogram_samples(&rows, "rustfs_s3_put_object_rename_quorum_wait_fanout_disks");
|
||||
assert_eq!(quorum_samples, vec![0.0, 1.0, 3.0, 3.0, 4.0]);
|
||||
assert_eq!(
|
||||
histogram_samples(&rows, "rustfs_s3_put_object_rename_fdatasync_group_wait_ms"),
|
||||
vec![1.0, 2.0]
|
||||
vec![1.0, 2.0, 3.0]
|
||||
);
|
||||
assert_eq!(
|
||||
histogram_samples(&rows, "rustfs_s3_put_object_rename_fdatasync_group_outstanding"),
|
||||
@@ -4494,6 +4508,7 @@ mod tests {
|
||||
HashSet::from([
|
||||
PUT_RENAME_FDATASYNC_GROUP_WAIT_ROLE_LEADER.to_string(),
|
||||
PUT_RENAME_FDATASYNC_GROUP_WAIT_ROLE_FOLLOWER.to_string(),
|
||||
PUT_RENAME_FDATASYNC_GROUP_WAIT_ROLE_FOLLOWER_TIMEOUT.to_string(),
|
||||
])
|
||||
);
|
||||
let group_outstanding_states = rows
|
||||
|
||||
@@ -22,7 +22,7 @@ workspace = true
|
||||
base64-simd = { workspace = true, optional = true }
|
||||
bytes = { workspace = true, optional = true }
|
||||
memmap2 = { workspace = true, optional = true }
|
||||
nixl-sys = { git = "https://github.com/ai-dynamo/nixl", tag = "v1.5.2", optional = true }
|
||||
nixl-sys = { workspace = true, optional = true }
|
||||
parking_lot = { workspace = true, optional = true }
|
||||
thiserror = { workspace = true, optional = true }
|
||||
tokio = { workspace = true, features = ["rt-multi-thread", "sync", "time"], optional = true }
|
||||
|
||||
@@ -29,6 +29,9 @@ documentation = "https://docs.rs/rustfs-obs/latest/rustfs_obs/"
|
||||
default = []
|
||||
# Default-off JSON isolation for sampled local-batch diagnostic root events.
|
||||
diagnostic-local-batch = []
|
||||
# Exact HTTP/2 PING ACK timeout observation. The root feature also enables
|
||||
# Hyper's unstable tracing event in rustfs-protos.
|
||||
diagnostic-internode-http2-ping = ["rustfs-io-metrics/diagnostic-internode-http2-ping"]
|
||||
hotpath = [
|
||||
"hotpath/hotpath",
|
||||
"hotpath/tokio",
|
||||
|
||||
+12
-1
@@ -12,7 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Enforces that the `dial9` feature is paired with `--cfg tokio_unstable`.
|
||||
//! Enforces the cfg contracts of default-off unstable instrumentation features.
|
||||
//!
|
||||
//! `dial9` hooks Tokio's unstable runtime instrumentation. The
|
||||
//! flag lives in `RUSTFLAGS`, which a caller can silently clobber by exporting
|
||||
@@ -25,6 +25,8 @@ fn main() {
|
||||
|
||||
let dial9_enabled = std::env::var_os("CARGO_FEATURE_DIAL9").is_some();
|
||||
let tokio_unstable = std::env::var_os("CARGO_CFG_TOKIO_UNSTABLE").is_some();
|
||||
let ping_observability_enabled = std::env::var_os("CARGO_FEATURE_DIAGNOSTIC_INTERNODE_HTTP2_PING").is_some();
|
||||
let hyper_unstable_tracing = std::env::var_os("CARGO_CFG_HYPER_UNSTABLE_TRACING").is_some();
|
||||
|
||||
if dial9_enabled && !tokio_unstable {
|
||||
panic!(
|
||||
@@ -39,4 +41,13 @@ fn main() {
|
||||
present in the value you export.\n"
|
||||
);
|
||||
}
|
||||
|
||||
if ping_observability_enabled && !hyper_unstable_tracing {
|
||||
panic!(
|
||||
"\n\
|
||||
rustfs-obs: the `diagnostic-internode-http2-ping` feature requires\n\
|
||||
`--cfg hyper_unstable_tracing`. Build with:\n\
|
||||
\n RUSTFLAGS=\"--cfg hyper_unstable_tracing\" cargo build -p rustfs --features diagnostic-internode-http2-ping\n"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -24,6 +24,9 @@ use tracing_subscriber::{
|
||||
filter::{FilterFn, LevelFilter, filter_fn},
|
||||
};
|
||||
|
||||
#[cfg(feature = "diagnostic-internode-http2-ping")]
|
||||
use super::http2_ping::HYPER_HTTP2_CLIENT_TARGET;
|
||||
|
||||
pub const HTTP_SERVER_LOG_TARGET: &str = "rustfs::server::http";
|
||||
|
||||
/// Pyroscope emits raw reqwest errors from its background session manager.
|
||||
@@ -57,7 +60,9 @@ pub(super) fn s3s_log_filter() -> FilterFn {
|
||||
/// `RUST_LOG` environment variable takes precedence over `logger_level`.
|
||||
/// For non-verbose levels (`info`, `warn`, `error`), noisy dependency targets
|
||||
/// are automatically suppressed based on the effective log configuration.
|
||||
/// Transport internals are disabled, while `s3s` remains visible at WARN.
|
||||
/// Transport internals are disabled, while `s3s` remains visible at WARN. The
|
||||
/// diagnostic HTTP/2 PING feature adds back only Hyper's client target at DEBUG
|
||||
/// so its exact keepalive timeout event reaches the metrics layer.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `logger_level` - The desired log level string (e.g., `"info"`, `"debug"`).
|
||||
@@ -294,6 +299,14 @@ pub(super) fn build_env_filter(logger_level: &str, default_level: Option<&str>)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "diagnostic-internode-http2-ping")]
|
||||
{
|
||||
let directive = format!("{HYPER_HTTP2_CLIENT_TARGET}=debug")
|
||||
.parse()
|
||||
.expect("static Hyper HTTP/2 PING tracing directive must parse");
|
||||
filter = filter.add_directive(directive);
|
||||
}
|
||||
|
||||
filter
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,113 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use tracing::{Event, Subscriber, field::Visit};
|
||||
use tracing_subscriber::{Layer, layer::Context};
|
||||
|
||||
pub(super) const HYPER_HTTP2_CLIENT_TARGET: &str = "hyper::proto::h2::client";
|
||||
const HYPER_KEEPALIVE_TIMEOUT_MESSAGE: &str = "connection keep-alive timed out";
|
||||
|
||||
#[derive(Clone, Copy, Debug, Default)]
|
||||
pub(crate) struct Http2PingTimeoutLayer;
|
||||
|
||||
pub(crate) const fn http2_ping_timeout_layer() -> Option<Http2PingTimeoutLayer> {
|
||||
if cfg!(feature = "diagnostic-internode-http2-ping") {
|
||||
Some(Http2PingTimeoutLayer)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct TimeoutMessageVisitor {
|
||||
matched: bool,
|
||||
}
|
||||
|
||||
impl Visit for TimeoutMessageVisitor {
|
||||
fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn std::fmt::Debug) {
|
||||
if field.name() == "message" && format!("{value:?}") == HYPER_KEEPALIVE_TIMEOUT_MESSAGE {
|
||||
self.matched = true;
|
||||
}
|
||||
}
|
||||
|
||||
fn record_str(&mut self, field: &tracing::field::Field, value: &str) {
|
||||
if field.name() == "message" && value == HYPER_KEEPALIVE_TIMEOUT_MESSAGE {
|
||||
self.matched = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<S> Layer<S> for Http2PingTimeoutLayer
|
||||
where
|
||||
S: Subscriber,
|
||||
{
|
||||
fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
|
||||
if event.metadata().target() != HYPER_HTTP2_CLIENT_TARGET
|
||||
|| !rustfs_io_metrics::internode_metrics::is_internode_http2_client_task()
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
let mut visitor = TimeoutMessageVisitor::default();
|
||||
event.record(&mut visitor);
|
||||
if visitor.matched {
|
||||
rustfs_io_metrics::internode_metrics::record_http2_ping_timeout();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(all(test, feature = "diagnostic-internode-http2-ping"))]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use metrics_util::debugging::{DebugValue, DebuggingRecorder};
|
||||
use tracing_subscriber::{Registry, layer::SubscriberExt};
|
||||
|
||||
static METRICS_TEST_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(());
|
||||
|
||||
#[tokio::test]
|
||||
async fn exact_scoped_hyper_timeout_event_records_once_without_close_inference() {
|
||||
let _test_guard = METRICS_TEST_LOCK.lock().await;
|
||||
let recorder = DebuggingRecorder::new();
|
||||
let snapshotter = recorder.snapshotter();
|
||||
let _recorder_guard = metrics::set_default_local_recorder(&recorder);
|
||||
let previously_enabled = rustfs_io_metrics::metrics_enabled();
|
||||
rustfs_io_metrics::set_metrics_enabled(true);
|
||||
|
||||
let subscriber = Registry::default()
|
||||
.with(crate::telemetry::filter::build_env_filter("info", Some("info")))
|
||||
.with(http2_ping_timeout_layer());
|
||||
let dispatch = tracing::Dispatch::new(subscriber);
|
||||
tracing::dispatcher::with_default(&dispatch, || {
|
||||
tracing::debug!(target: HYPER_HTTP2_CLIENT_TARGET, "connection keep-alive timed out");
|
||||
});
|
||||
rustfs_io_metrics::internode_metrics::scope_internode_http2_client_task(async {
|
||||
tracing::dispatcher::with_default(&dispatch, || {
|
||||
tracing::debug!(target: HYPER_HTTP2_CLIENT_TARGET, "connection keep-alive timed out");
|
||||
tracing::debug!(target: HYPER_HTTP2_CLIENT_TARGET, "connection error");
|
||||
tracing::debug!(target: "rustfs::test", "connection keep-alive timed out");
|
||||
});
|
||||
})
|
||||
.await;
|
||||
rustfs_io_metrics::set_metrics_enabled(previously_enabled);
|
||||
|
||||
let timeouts = snapshotter
|
||||
.snapshot()
|
||||
.into_vec()
|
||||
.into_iter()
|
||||
.find(|(key, _, _, _)| key.key().name() == "rustfs_system_network_internode_http2_ping_timeouts_total")
|
||||
.map(|(_, _, _, value)| value)
|
||||
.expect("exact HTTP/2 PING timeout counter");
|
||||
assert!(matches!(timeouts, DebugValue::Counter(1)));
|
||||
}
|
||||
}
|
||||
@@ -421,6 +421,7 @@ fn init_stdout_only(_config: &OtelConfig, logger_level: &str, is_production: boo
|
||||
|
||||
tracing_subscriber::registry()
|
||||
.with(env_filter)
|
||||
.with(super::http2_ping_timeout_layer())
|
||||
.with(pyroscope_log_filter())
|
||||
.with(s3s_log_filter())
|
||||
.with(ErrorLayer::default())
|
||||
@@ -534,6 +535,7 @@ fn init_file_logging_internal(
|
||||
|
||||
tracing_subscriber::registry()
|
||||
.with(env_filter)
|
||||
.with(super::http2_ping_timeout_layer())
|
||||
.with(pyroscope_log_filter())
|
||||
.with(s3s_log_filter())
|
||||
.with(ErrorLayer::default())
|
||||
|
||||
@@ -43,6 +43,8 @@
|
||||
pub mod dial9;
|
||||
mod filter;
|
||||
mod guard;
|
||||
#[cfg(feature = "diagnostic-internode-http2-ping")]
|
||||
mod http2_ping;
|
||||
mod local;
|
||||
mod otel;
|
||||
mod recorder;
|
||||
@@ -53,12 +55,19 @@ use crate::TelemetryError;
|
||||
use crate::config::OtelConfig;
|
||||
pub use filter::HTTP_SERVER_LOG_TARGET;
|
||||
pub use guard::OtelGuard;
|
||||
#[cfg(feature = "diagnostic-internode-http2-ping")]
|
||||
pub(super) use http2_ping::http2_ping_timeout_layer;
|
||||
pub(crate) use recorder::process_global_meter;
|
||||
pub use recorder::{Recorder, retire_metric_series};
|
||||
use rustfs_config::observability::ENV_OBS_LOG_DIRECTORY;
|
||||
use rustfs_config::{DEFAULT_LOG_LEVEL, ENVIRONMENT, observability::DEFAULT_OBS_ENVIRONMENT_PRODUCTION};
|
||||
use rustfs_utils::get_env_opt_str;
|
||||
|
||||
#[cfg(not(feature = "diagnostic-internode-http2-ping"))]
|
||||
pub(super) const fn http2_ping_timeout_layer() -> Option<tracing_subscriber::layer::Identity> {
|
||||
None
|
||||
}
|
||||
|
||||
/// Initialize the telemetry subsystem according to the provided configuration.
|
||||
///
|
||||
/// Evaluates three routing rules in priority order and delegates to the
|
||||
|
||||
@@ -107,6 +107,7 @@ const OBJECT_LOCK_DURATION_HISTOGRAM_BUCKETS: &[f64] = &[
|
||||
const INTERNODE_DURATION_HISTOGRAM_METRICS: &[&str] = &[
|
||||
"rustfs_system_network_internode_operation_duration_ms",
|
||||
"rustfs_system_network_internode_operation_server_duration_ms",
|
||||
"rustfs_system_network_internode_bulk_body_wait_duration_ms",
|
||||
];
|
||||
// Internode instruments record milliseconds, including sub-millisecond control RPCs.
|
||||
const INTERNODE_DURATION_HISTOGRAM_BUCKETS: &[f64] = &[
|
||||
@@ -311,6 +312,7 @@ pub(super) fn init_observability_http(
|
||||
let filter = build_env_filter(logger_level, None);
|
||||
tracing_subscriber::registry()
|
||||
.with(filter)
|
||||
.with(super::http2_ping_timeout_layer())
|
||||
.with(pyroscope_log_filter())
|
||||
.with(s3s_log_filter())
|
||||
.with(ErrorLayer::default())
|
||||
@@ -1181,6 +1183,7 @@ mod tests {
|
||||
(OBJECT_LOCK_DURATION_HISTOGRAM_METRICS[1], [0.002, 12.9, 31.0]),
|
||||
(INTERNODE_DURATION_HISTOGRAM_METRICS[0], [0.04, 3.2, 31000.0]),
|
||||
(INTERNODE_DURATION_HISTOGRAM_METRICS[1], [0.02, 1.2, 1300.0]),
|
||||
(INTERNODE_DURATION_HISTOGRAM_METRICS[2], [0.01, 4.2, 1200.0]),
|
||||
] {
|
||||
let histogram = recorder.register_histogram(&metrics::Key::from_name(name), &metadata);
|
||||
for seconds in observations {
|
||||
@@ -1200,6 +1203,7 @@ mod tests {
|
||||
"rustfs_object_lock_diag_hold_duration_seconds" => [(0.002, 1), (15.0, 1), (60.0, 1)],
|
||||
"rustfs_system_network_internode_operation_duration_ms" => [(0.05, 1), (4.0, 1), (60000.0, 1)],
|
||||
"rustfs_system_network_internode_operation_server_duration_ms" => [(0.025, 1), (1.5, 1), (2500.0, 1)],
|
||||
"rustfs_system_network_internode_bulk_body_wait_duration_ms" => [(0.01, 1), (5.0, 1), (2500.0, 1)],
|
||||
_ => continue,
|
||||
};
|
||||
let AggregatedMetrics::F64(MetricData::Histogram(histogram)) = metric.data() else {
|
||||
@@ -1215,7 +1219,7 @@ mod tests {
|
||||
assert_eq!(occupied, expected, "seconds remain distinct at millisecond and long-hold scales");
|
||||
checked += 1;
|
||||
}
|
||||
assert_eq!(checked, 4, "lock and internode duration histograms were exported");
|
||||
assert_eq!(checked, 5, "lock and internode duration histograms were exported");
|
||||
provider.shutdown().expect("stop lock histogram test provider");
|
||||
}
|
||||
|
||||
|
||||
@@ -30,6 +30,7 @@ pub mod variables;
|
||||
pub use action::ActionSet;
|
||||
pub use doc::PolicyDoc;
|
||||
pub use effect::Effect;
|
||||
pub use function::key_name::SVC_TARGET_USER;
|
||||
pub use function::{Functions, is_server_derived_condition_key};
|
||||
pub use id::ID;
|
||||
pub use policy::*;
|
||||
|
||||
@@ -495,6 +495,8 @@ pub enum AdminAction {
|
||||
SiteReplicationOperationAction,
|
||||
#[strum(serialize = "admin:CreateServiceAccount")]
|
||||
CreateServiceAccountAdminAction,
|
||||
#[strum(serialize = "admin:CreateServiceAccountForUser")]
|
||||
CreateServiceAccountForUserAdminAction,
|
||||
#[strum(serialize = "admin:UpdateServiceAccount")]
|
||||
UpdateServiceAccountAdminAction,
|
||||
#[strum(serialize = "admin:RemoveServiceAccount")]
|
||||
@@ -705,6 +707,7 @@ impl AdminAction {
|
||||
| AdminAction::SiteReplicationInfoAction
|
||||
| AdminAction::SiteReplicationOperationAction
|
||||
| AdminAction::CreateServiceAccountAdminAction
|
||||
| AdminAction::CreateServiceAccountForUserAdminAction
|
||||
| AdminAction::UpdateServiceAccountAdminAction
|
||||
| AdminAction::RemoveServiceAccountAdminAction
|
||||
| AdminAction::ListServiceAccountsAdminAction
|
||||
|
||||
@@ -48,6 +48,9 @@ impl TryFrom<&str> for KeyName {
|
||||
}
|
||||
}
|
||||
|
||||
/// Canonical condition-map key for the resolved service-account parent.
|
||||
pub const SVC_TARGET_USER: &str = "TargetUser";
|
||||
|
||||
impl KeyName {
|
||||
pub const COMMON_KEYS: &'static [KeyName] = &[
|
||||
// s3
|
||||
@@ -136,7 +139,11 @@ impl KeyName {
|
||||
/// Names (prefix stripped) of the well-known condition keys that request input
|
||||
/// must never populate. See [`KeyName::is_server_derived`].
|
||||
pub fn server_derived_key_names() -> impl Iterator<Item = &'static str> {
|
||||
Self::COMMON_KEYS.iter().filter(|k| k.is_server_derived()).map(|k| k.name())
|
||||
Self::COMMON_KEYS
|
||||
.iter()
|
||||
.filter(|k| k.is_server_derived())
|
||||
.map(|k| k.name())
|
||||
.chain([SVC_TARGET_USER])
|
||||
}
|
||||
|
||||
/// Returns the policy variable form of the key, e.g. `${jwt:sub}`.
|
||||
@@ -328,6 +335,8 @@ pub enum JwtKeyName {
|
||||
pub enum SvcKeyName {
|
||||
#[strum(serialize = "svc:DurationSeconds")]
|
||||
SVCDurationSeconds,
|
||||
#[strum(serialize = "svc:TargetUser")]
|
||||
TargetUser,
|
||||
}
|
||||
|
||||
#[derive(Clone, EnumString, Debug, IntoStaticStr, Eq, PartialEq, Serialize, Deserialize)]
|
||||
|
||||
@@ -644,6 +644,58 @@ mod test {
|
||||
use crate::error::Result;
|
||||
use crate::policy::action::{AdminAction, KmsAction, S3Action};
|
||||
|
||||
#[tokio::test]
|
||||
async fn delegated_service_account_obeys_admin_wildcards_and_target_conditions() -> Result<()> {
|
||||
let conditions = HashMap::from([("TargetUser".into(), vec!["backup".into()])]);
|
||||
let args = Args {
|
||||
account: "operator",
|
||||
groups: &None,
|
||||
action: Action::AdminAction(AdminAction::CreateServiceAccountForUserAdminAction),
|
||||
bucket: "",
|
||||
object: "",
|
||||
conditions: &conditions,
|
||||
claims: &HashMap::new(),
|
||||
is_owner: false,
|
||||
deny_only: false,
|
||||
};
|
||||
for (statement, allowed) in [
|
||||
(serde_json::json!({"Effect":"Allow","Action":"admin:*"}), true),
|
||||
(serde_json::json!({"Effect":"Allow","Action":"admin:CreateServiceAccount"}), false),
|
||||
(serde_json::json!({"Effect":"Allow","NotAction":"admin:DeleteUser","Resource":"*"}), true),
|
||||
(
|
||||
serde_json::json!({"Effect":"Allow","NotAction":"admin:CreateServiceAccountForUser","Resource":"*"}),
|
||||
false,
|
||||
),
|
||||
(
|
||||
serde_json::json!({"Effect":"Allow","Action":"admin:CreateServiceAccountForUser","Condition":{"StringEquals":{"svc:TargetUser":"backup"}}}),
|
||||
true,
|
||||
),
|
||||
(
|
||||
serde_json::json!({"Effect":"Allow","Action":"admin:CreateServiceAccountForUser","Condition":{"StringEquals":{"svc:TargetUser":"archive"}}}),
|
||||
false,
|
||||
),
|
||||
] {
|
||||
let policy = Policy::parse_config(
|
||||
&serde_json::to_vec(&serde_json::json!({"Version":DEFAULT_VERSION,"Statement":[statement]}))
|
||||
.expect("encode policy"),
|
||||
)?;
|
||||
assert_eq!(policy.is_allowed(&args).await, allowed, "{statement}");
|
||||
}
|
||||
let admin = default::DEFAULT_POLICIES
|
||||
.iter()
|
||||
.find(|(name, _)| *name == "consoleAdmin")
|
||||
.expect("built-in admin policy");
|
||||
assert!(admin.1.is_allowed(&args).await);
|
||||
for denied_action in ["admin:*", "admin:CreateServiceAccountForUser"] {
|
||||
let deny = Policy::parse_config(&serde_json::to_vec(&serde_json::json!({
|
||||
"Version": DEFAULT_VERSION,
|
||||
"Statement": [{"Effect": "Deny", "Action": denied_action}]
|
||||
}))?)?;
|
||||
assert!(!Policy::merge_policies(vec![admin.1.clone(), deny]).is_allowed(&args).await);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_parse_policy() -> Result<()> {
|
||||
let data = r#"
|
||||
|
||||
@@ -34,6 +34,9 @@ path = "src/main.rs"
|
||||
|
||||
[features]
|
||||
default = []
|
||||
# Exact HTTP/2 keepalive timeout observation through Hyper's unstable tracing
|
||||
# event. Requires `RUSTFLAGS="--cfg hyper_unstable_tracing"`.
|
||||
diagnostic-internode-http2-ping = ["dep:hyper", "rustfs-io-metrics/diagnostic-internode-http2-ping"]
|
||||
hotpath = [
|
||||
"hotpath/hotpath",
|
||||
"hotpath/tokio",
|
||||
@@ -75,6 +78,8 @@ prost = { workspace = true }
|
||||
tonic = { workspace = true, features = ["transport", "tls-native-roots", "tls-aws-lc", "gzip", "deflate"] }
|
||||
tonic-prost = { workspace = true }
|
||||
tonic-prost-build = { workspace = true }
|
||||
hyper = { workspace = true, optional = true, features = ["tracing"] }
|
||||
hyper-util = { workspace = true, features = ["client-legacy", "tokio"] }
|
||||
tokio = { workspace = true, features = ["sync", "fs", "rt-multi-thread"] }
|
||||
tracing = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
@@ -90,3 +95,4 @@ serde_json = { workspace = true }
|
||||
temp-env = { workspace = true, features = ["async_closure"] }
|
||||
tempfile = { workspace = true }
|
||||
tokio = { workspace = true, features = ["macros"] }
|
||||
tracing-subscriber = { workspace = true }
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
fn main() {
|
||||
println!("cargo::rerun-if-changed=build.rs");
|
||||
println!("cargo::rerun-if-env-changed=RUSTFLAGS");
|
||||
|
||||
let ping_observability_enabled = std::env::var_os("CARGO_FEATURE_DIAGNOSTIC_INTERNODE_HTTP2_PING").is_some();
|
||||
let hyper_unstable_tracing = std::env::var_os("CARGO_CFG_HYPER_UNSTABLE_TRACING").is_some();
|
||||
|
||||
if ping_observability_enabled && !hyper_unstable_tracing {
|
||||
panic!(
|
||||
"\n\
|
||||
rustfs-protos: the `diagnostic-internode-http2-ping` feature requires\n\
|
||||
`--cfg hyper_unstable_tracing` so Hyper emits the exact keepalive\n\
|
||||
timeout event. Build with:\n\
|
||||
\n RUSTFLAGS=\"--cfg hyper_unstable_tracing\" cargo build -p rustfs --features diagnostic-internode-http2-ping\n"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -73,7 +73,7 @@ pub const REQUEST_COMPAT_SEND_SITES: &[RequestCompatSendSite] = &[
|
||||
json_field: "batch_read_version_req",
|
||||
bin_field: "batch_read_version_req_bin",
|
||||
},
|
||||
json_encoder: "let batch_read_version_req = compat_json(&req)?;",
|
||||
json_encoder: "let batch_read_version_req = compat_json(batch_rpc_req.as_ref(), &policy)?;",
|
||||
policy: RequestJsonPolicy::MsgpackOnlyEligible,
|
||||
},
|
||||
RequestCompatSendSite {
|
||||
@@ -118,7 +118,7 @@ pub const REQUEST_COMPAT_SEND_SITES: &[RequestCompatSendSite] = &[
|
||||
json_field: "read_multiple_req",
|
||||
bin_field: "read_multiple_req_bin",
|
||||
},
|
||||
json_encoder: "let read_multiple_req = compat_json(&req)?;",
|
||||
json_encoder: "let read_multiple_req = compat_json(&req, &policy)?;",
|
||||
policy: RequestJsonPolicy::MsgpackOnlyEligible,
|
||||
},
|
||||
RequestCompatSendSite {
|
||||
@@ -127,7 +127,7 @@ pub const REQUEST_COMPAT_SEND_SITES: &[RequestCompatSendSite] = &[
|
||||
json_field: "opts",
|
||||
bin_field: "opts_bin",
|
||||
},
|
||||
json_encoder: "let encoded_opts = compat_json(opts).and_then(|opts_str| encode_msgpack(opts).map(|opts_bin| (opts_str, opts_bin)));",
|
||||
json_encoder: "let opts_str = compat_json(opts, &policy)?;",
|
||||
policy: RequestJsonPolicy::MsgpackOnlyEligible,
|
||||
},
|
||||
RequestCompatSendSite {
|
||||
@@ -136,7 +136,7 @@ pub const REQUEST_COMPAT_SEND_SITES: &[RequestCompatSendSite] = &[
|
||||
json_field: "file_info",
|
||||
bin_field: "file_info_bin",
|
||||
},
|
||||
json_encoder: "let file_info = compat_json(&fi)?;",
|
||||
json_encoder: "let file_info = compat_json(&fi, &policy)?;",
|
||||
policy: RequestJsonPolicy::MsgpackOnlyEligible,
|
||||
},
|
||||
RequestCompatSendSite {
|
||||
@@ -145,7 +145,7 @@ pub const REQUEST_COMPAT_SEND_SITES: &[RequestCompatSendSite] = &[
|
||||
json_field: "file_info",
|
||||
bin_field: "file_info_bin",
|
||||
},
|
||||
json_encoder: "let file_info = compat_json(&fi)?;",
|
||||
json_encoder: "let file_info = compat_json(&fi, &policy)?;",
|
||||
policy: RequestJsonPolicy::MsgpackOnlyEligible,
|
||||
},
|
||||
RequestCompatSendSite {
|
||||
@@ -154,7 +154,7 @@ pub const REQUEST_COMPAT_SEND_SITES: &[RequestCompatSendSite] = &[
|
||||
json_field: "opts",
|
||||
bin_field: "opts_bin",
|
||||
},
|
||||
json_encoder: "let opts_str = compat_json(&opts)?;",
|
||||
json_encoder: "let opts_str = compat_json(opts, &policy)?;",
|
||||
policy: RequestJsonPolicy::MsgpackOnlyEligible,
|
||||
},
|
||||
RequestCompatSendSite {
|
||||
@@ -163,7 +163,7 @@ pub const REQUEST_COMPAT_SEND_SITES: &[RequestCompatSendSite] = &[
|
||||
json_field: "file_info",
|
||||
bin_field: "file_info_bin",
|
||||
},
|
||||
json_encoder: "let file_info = compat_json(&fi)?;",
|
||||
json_encoder: "let file_info = compat_json(&fi, &policy)?;",
|
||||
policy: RequestJsonPolicy::MsgpackOnlyEligible,
|
||||
},
|
||||
];
|
||||
|
||||
+827
-45
File diff suppressed because it is too large
Load Diff
@@ -34,6 +34,10 @@ pub(crate) fn record_grpc_dial_result(duration: Duration, success: bool) {
|
||||
global_internode_metrics().record_dial_result(duration, success);
|
||||
}
|
||||
|
||||
pub(crate) fn record_channel_rebuild_attempt(class: &'static str, reason: &'static str, connected: bool) {
|
||||
rustfs_io_metrics::internode_metrics::record_channel_rebuild_attempt(class, reason, connected);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn grpc_dial_errors_total() -> u64 {
|
||||
global_internode_metrics().snapshot().dial_errors_total
|
||||
|
||||
@@ -83,6 +83,13 @@ pub enum S3Operation {
|
||||
SelectObjectContent,
|
||||
UploadPart,
|
||||
UploadPartCopy,
|
||||
GetBucketAccelerateConfiguration,
|
||||
GetBucketRequestPayment,
|
||||
GetBucketWebsite,
|
||||
PutBucketAccelerateConfiguration,
|
||||
PutBucketRequestPayment,
|
||||
PutBucketWebsite,
|
||||
DeleteBucketWebsite,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
@@ -126,7 +133,7 @@ impl EventMapping {
|
||||
}
|
||||
|
||||
impl S3Operation {
|
||||
pub const ALL: [Self; 66] = [
|
||||
pub const ALL: [Self; 73] = [
|
||||
Self::AbortMultipartUpload,
|
||||
Self::CompleteMultipartUpload,
|
||||
Self::CopyObject,
|
||||
@@ -193,6 +200,13 @@ impl S3Operation {
|
||||
Self::SelectObjectContent,
|
||||
Self::UploadPart,
|
||||
Self::UploadPartCopy,
|
||||
Self::GetBucketAccelerateConfiguration,
|
||||
Self::GetBucketRequestPayment,
|
||||
Self::GetBucketWebsite,
|
||||
Self::PutBucketAccelerateConfiguration,
|
||||
Self::PutBucketRequestPayment,
|
||||
Self::PutBucketWebsite,
|
||||
Self::DeleteBucketWebsite,
|
||||
];
|
||||
|
||||
#[inline]
|
||||
@@ -214,10 +228,12 @@ impl S3Operation {
|
||||
Self::DeleteBucketPolicy => "s3:DeleteBucketPolicy",
|
||||
Self::DeleteBucketReplication => "s3:DeleteBucketReplication",
|
||||
Self::DeleteBucketTagging => "s3:DeleteBucketTagging",
|
||||
Self::DeleteBucketWebsite => "s3:DeleteBucketWebsite",
|
||||
Self::DeleteObject => "s3:DeleteObject",
|
||||
Self::DeleteObjectTagging => "s3:DeleteObjectTagging",
|
||||
Self::DeleteObjects => "s3:DeleteObjects",
|
||||
Self::DeletePublicAccessBlock => "s3:DeletePublicAccessBlock",
|
||||
Self::GetBucketAccelerateConfiguration => "s3:GetBucketAccelerateConfiguration",
|
||||
Self::GetBucketAcl => "s3:GetBucketAcl",
|
||||
Self::GetBucketCors => "s3:GetBucketCors",
|
||||
Self::GetBucketEncryption => "s3:GetBucketEncryption",
|
||||
@@ -228,8 +244,10 @@ impl S3Operation {
|
||||
Self::GetBucketPolicy => "s3:GetBucketPolicy",
|
||||
Self::GetBucketPolicyStatus => "s3:GetBucketPolicyStatus",
|
||||
Self::GetBucketReplication => "s3:GetBucketReplication",
|
||||
Self::GetBucketRequestPayment => "s3:GetBucketRequestPayment",
|
||||
Self::GetBucketTagging => "s3:GetBucketTagging",
|
||||
Self::GetBucketVersioning => "s3:GetBucketVersioning",
|
||||
Self::GetBucketWebsite => "s3:GetBucketWebsite",
|
||||
Self::GetObject => "s3:GetObject",
|
||||
Self::GetObjectAcl => "s3:GetObjectAcl",
|
||||
Self::GetObjectAttributes => "s3:GetObjectAttributes",
|
||||
@@ -247,6 +265,7 @@ impl S3Operation {
|
||||
Self::ListObjects => "s3:ListObjects",
|
||||
Self::ListObjectsV2 => "s3:ListObjectsV2",
|
||||
Self::ListParts => "s3:ListParts",
|
||||
Self::PutBucketAccelerateConfiguration => "s3:PutBucketAccelerateConfiguration",
|
||||
Self::PutBucketAcl => "s3:PutBucketAcl",
|
||||
Self::PutBucketCors => "s3:PutBucketCors",
|
||||
Self::PutBucketEncryption => "s3:PutBucketEncryption",
|
||||
@@ -255,8 +274,10 @@ impl S3Operation {
|
||||
Self::PutBucketNotificationConfiguration => "s3:PutBucketNotificationConfiguration",
|
||||
Self::PutBucketPolicy => "s3:PutBucketPolicy",
|
||||
Self::PutBucketReplication => "s3:PutBucketReplication",
|
||||
Self::PutBucketRequestPayment => "s3:PutBucketRequestPayment",
|
||||
Self::PutBucketTagging => "s3:PutBucketTagging",
|
||||
Self::PutBucketVersioning => "s3:PutBucketVersioning",
|
||||
Self::PutBucketWebsite => "s3:PutBucketWebsite",
|
||||
Self::PutObject => "s3:PutObject",
|
||||
Self::PutObjectAcl => "s3:PutObjectAcl",
|
||||
Self::PutObjectLegalHold => "s3:PutObjectLegalHold",
|
||||
@@ -301,7 +322,9 @@ impl S3Operation {
|
||||
| Self::DeleteBucketPolicy
|
||||
| Self::DeleteBucketReplication
|
||||
| Self::DeleteBucketTagging
|
||||
| Self::DeleteBucketWebsite
|
||||
| Self::DeletePublicAccessBlock
|
||||
| Self::GetBucketAccelerateConfiguration
|
||||
| Self::GetBucketAcl
|
||||
| Self::GetBucketCors
|
||||
| Self::GetBucketEncryption
|
||||
@@ -312,8 +335,10 @@ impl S3Operation {
|
||||
| Self::GetBucketPolicy
|
||||
| Self::GetBucketPolicyStatus
|
||||
| Self::GetBucketReplication
|
||||
| Self::GetBucketRequestPayment
|
||||
| Self::GetBucketTagging
|
||||
| Self::GetBucketVersioning
|
||||
| Self::GetBucketWebsite
|
||||
| Self::GetObjectAcl
|
||||
| Self::GetObjectLockConfiguration
|
||||
| Self::GetObjectTagging
|
||||
@@ -326,6 +351,7 @@ impl S3Operation {
|
||||
| Self::ListObjects
|
||||
| Self::ListObjectsV2
|
||||
| Self::ListParts
|
||||
| Self::PutBucketAccelerateConfiguration
|
||||
| Self::PutBucketAcl
|
||||
| Self::PutBucketCors
|
||||
| Self::PutBucketEncryption
|
||||
@@ -334,8 +360,10 @@ impl S3Operation {
|
||||
| Self::PutBucketNotificationConfiguration
|
||||
| Self::PutBucketPolicy
|
||||
| Self::PutBucketReplication
|
||||
| Self::PutBucketRequestPayment
|
||||
| Self::PutBucketTagging
|
||||
| Self::PutBucketVersioning
|
||||
| Self::PutBucketWebsite
|
||||
| Self::PutObjectLockConfiguration
|
||||
| Self::PutPublicAccessBlock
|
||||
| Self::UploadPart
|
||||
@@ -516,12 +544,29 @@ mod tests {
|
||||
assert_eq!(mask & EventName::ObjectRemovedDelete.mask(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn metric_indexes_and_labels_identify_each_operation() {
|
||||
let mut labels = std::collections::HashSet::new();
|
||||
for (index, op) in S3Operation::ALL.into_iter().enumerate() {
|
||||
assert_eq!(op.metric_index(), index, "metric index for {op:?}");
|
||||
assert!(labels.insert(op.as_str()), "repeated metric label for {op:?}");
|
||||
}
|
||||
assert_eq!(S3Operation::UploadPartCopy.metric_index(), 65);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_operations_without_event_mapping_remain_unmapped() {
|
||||
let unmapped = [
|
||||
S3Operation::DeleteBucketWebsite,
|
||||
S3Operation::GetBucketAccelerateConfiguration,
|
||||
S3Operation::GetBucketAcl,
|
||||
S3Operation::GetBucketRequestPayment,
|
||||
S3Operation::GetBucketWebsite,
|
||||
S3Operation::ListObjectsV2,
|
||||
S3Operation::PutBucketAccelerateConfiguration,
|
||||
S3Operation::PutBucketNotificationConfiguration,
|
||||
S3Operation::PutBucketRequestPayment,
|
||||
S3Operation::PutBucketWebsite,
|
||||
S3Operation::UploadPart,
|
||||
];
|
||||
for op in unmapped {
|
||||
|
||||
@@ -740,6 +740,7 @@ async fn scanner_recovery_intent_startup_rejects_corrupt_pending_record() {
|
||||
save_config(store.clone(), &path, b"{corrupt".to_vec())
|
||||
.await
|
||||
.expect("corrupt intent payload should persist");
|
||||
crate::scanner_io::tests::wait_for_namespace_commit_tails(store.as_ref()).await;
|
||||
|
||||
let restarted = restart_scanner_cycle_store_from(&store).await;
|
||||
let error = scanner_usage_recovery_intents_for_startup(&CancellationToken::new(), restarted)
|
||||
|
||||
@@ -109,7 +109,7 @@ pub use bucket::{BucketInfo, BucketOperations, BucketOptions, DeleteBucketOption
|
||||
pub use capability::{CapabilitySnapshotError, CapabilityState, CapabilityStatus};
|
||||
pub use error::{StorageErrorCode, StorageResult};
|
||||
pub use multipart::{CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartUploadResult, PartInfo};
|
||||
pub use multipart::{MultipartCompletionAuthorization, MultipartUploadInspection};
|
||||
pub use multipart::{MultipartAbortAuthorization, MultipartCompletionAuthorization, MultipartOwner, MultipartUploadInspection};
|
||||
pub use object::AuthoritativeListOperations;
|
||||
pub use object::DeleteAccounting;
|
||||
pub use object::ObjectLockDeleteOptions;
|
||||
|
||||
@@ -15,6 +15,24 @@
|
||||
use std::collections::HashMap;
|
||||
use time::OffsetDateTime;
|
||||
|
||||
/// Ownership observed in the selected upload metadata.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct MultipartOwner {
|
||||
/// The bucket recorded by the upload.
|
||||
pub bucket: String,
|
||||
/// The recorded object, decoded after checking the encoded storage namespace.
|
||||
pub object: String,
|
||||
}
|
||||
|
||||
/// Authorizes Abort against the upload selected under its existing write lock.
|
||||
/// An implementation must reject missing ownership and must not acquire storage locks or mutate storage.
|
||||
pub trait MultipartAbortAuthorization: Send + Sync {
|
||||
type Error: std::error::Error + Send + Sync + 'static;
|
||||
|
||||
/// Returns a refusal before storage prepares or deletes any staged data.
|
||||
fn authorize(&self, owner: Option<&MultipartOwner>) -> Result<(), Self::Error>;
|
||||
}
|
||||
|
||||
/// Request authorization against the source selected under completion locks.
|
||||
/// Implementations must not mutate storage or acquire object/upload locks again.
|
||||
/// Called before comparing client checksums/parts or acknowledging a replay.
|
||||
@@ -73,8 +91,18 @@ pub struct ListMultipartsInfo {
|
||||
pub common_prefixes: Vec<String>,
|
||||
}
|
||||
|
||||
/// The owner found in the upload metadata used to read a ListParts page.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ListPartsOwner {
|
||||
pub bucket: String,
|
||||
/// The recorded object after decoding the storage namespace spelling.
|
||||
pub object: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct ListPartsInfo {
|
||||
/// Present when the recorded bucket and encoded object agree with the storage namespace.
|
||||
pub recorded_owner: Option<ListPartsOwner>,
|
||||
pub bucket: String,
|
||||
pub object: String,
|
||||
pub upload_id: String,
|
||||
|
||||
@@ -43,6 +43,9 @@ allow-git = [
|
||||
# Completion ownership patch fixed to rustfs/uring#20.
|
||||
# owner: overtrue review: 2026-10
|
||||
"https://github.com/rustfs/uring.git",
|
||||
# NIXL bindings fork pinned for the optional RustFS RDMA transport.
|
||||
# owner: houseme review: 2027-02
|
||||
"https://github.com/houseme/nixl",
|
||||
]
|
||||
|
||||
[bans]
|
||||
|
||||
@@ -60,6 +60,7 @@ Run **before** at the baseline column and **after** at the enabled column, every
|
||||
| P1 isolation | `RUSTFS_INTERNODE_CHANNEL_ISOLATION` | `false` (default) | `true` |
|
||||
| P1 bulk pool | `RUSTFS_INTERNODE_BULK_CHANNELS` | unset | `2` |
|
||||
| P2 msgpack-only | `RUSTFS_INTERNODE_RPC_MSGPACK_ONLY` + `RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED` | both `false` (default) | per the convergence runbook |
|
||||
| P2 per-peer msgpack | `RUSTFS_INTERNODE_RPC_MSGPACK_ONLY` + `RUSTFS_INTERNODE_RPC_MSGPACK_PEER_CAPABILITY` | both `false` (default) | both `true`, fleet confirmation `false`, stable direct/affine peer connections only |
|
||||
| P3 prewarm | `RUSTFS_INTERNODE_PREWARM` | `false` (default) | `true` |
|
||||
| P3 offline bypass | `RUSTFS_INTERNODE_OFFLINE_BYPASS` | `false` (default) | `true` |
|
||||
| P3 reprobe / threshold | `RUSTFS_INTERNODE_OFFLINE_REPROBE_SECS` / `RUSTFS_INTERNODE_OFFLINE_FAILURE_THRESHOLD` | defaults | defaults (`DEFAULT_INTERNODE_OFFLINE_REPROBE_SECS`, `DEFAULT_INTERNODE_OFFLINE_FAILURE_THRESHOLD`) |
|
||||
@@ -81,6 +82,11 @@ Explicit channel dial metrics start after endpoint settings validate and cover T
|
||||
| `rustfs_system_network_internode_operation_payload_bytes` | payload size distribution (P0/P1 sizing) |
|
||||
| `rustfs_system_network_internode_operation_large_payloads_total` | large unary RPCs sharing a channel (P1 target) |
|
||||
| `rustfs_system_network_internode_dial_avg_time_nanos`, `rustfs_system_network_internode_dial_errors_total` | connect cost and failures (P3) |
|
||||
| `rustfs_system_network_internode_channel_rebuild_attempts_total{class,reason,outcome}` | explicit RustFS cache rebuild attempts after a bounded eviction reason |
|
||||
| `rustfs_system_network_internode_http2_ping_timeouts_total` | exact Hyper keepalive PING ACK timeouts in diagnostic builds; generic closes and RPC errors do not contribute |
|
||||
| `rustfs_system_network_internode_bulk_body_{waiters,active}` | work waiting for or holding complete bulk-unary admission |
|
||||
| `rustfs_system_network_internode_bulk_body_wait_duration_ms` | time spent before a complete bulk-unary attempt enters tonic |
|
||||
| `rustfs_system_network_internode_bulk_body_admissions_total{outcome}` | admitted attempts and waits cancelled by an outer deadline |
|
||||
| `rustfs_system_network_internode_msgpack_json_decode_total{direction,message,codec}`, `..._msgpack_json_fallback_total`, `..._msgpack_json_decode_error_total` | P2 gate inputs |
|
||||
| `rustfs_cluster_servers_offline_total` | offline detection correctness (P3 bypass) |
|
||||
| lock p99 (lock metrics, needs `RUSTFS_OBJECT_LOCK_DIAG_ENABLE=true`) | P1 head-of-line-blocking win |
|
||||
@@ -99,11 +105,28 @@ P1 phase files clear bulk-specific overrides so an isolation experiment inherits
|
||||
| `RUSTFS_INTERNODE_BULK_HTTP2_CONN_WINDOW_SIZE` | Inherit `RUSTFS_INTERNODE_RPC_HTTP2_CONN_WINDOW_SIZE` | Bulk response receive credit per connection, bytes |
|
||||
| `RUSTFS_INTERNODE_BULK_CONCURRENCY_LIMIT` | `64` | Active tonic transport response futures per bulk channel; minimum 1 |
|
||||
| `RUSTFS_INTERNODE_BULK_BUFFER_SIZE` | `1024` | Pending tonic service buffer capacity per bulk channel; minimum 1 |
|
||||
| `RUSTFS_INTERNODE_BULK_BODY_CONCURRENCY_LIMIT` | `0` | Complete bulk-unary attempts per peer, held through response decoding and validation; `0` disables the additional gate |
|
||||
| `RUSTFS_INTERNODE_BULK_BODY_WAIT_TIMEOUT_MS` | `1000` | Local wait budget before returning `DiskOngoingReq`; minimum 1 ms |
|
||||
|
||||
Zero receive-window values select the linked library default. In the locked Hyper 1.12 client this is 2 MiB per stream and 5 MiB per connection, while RustFS explicitly defaults to 1 MiB and 2 MiB. Record which policy the experiment actually selects. Values above the HTTP/2 31-bit limit fail channel acquisition with an invalid-input error. Change connection receive credit first, then stream credit only when measured bandwidth-delay product and resource budgets justify it. For example, compare the default 2 MiB connection window with `RUSTFS_INTERNODE_BULK_HTTP2_CONN_WINDOW_SIZE=8388608`, holding other settings fixed. Request-upload flow control is configured by the receiving server. Keep server-wide window changes in a separate experiment.
|
||||
|
||||
Bulk selection advances independently for each peer. Cursor bookkeeping caches up to 512 peer addresses. Each entry keeps four immutable slot keys so common bulk pools borrow their cache keys on hits. Larger pools construct the remaining slot keys on demand with the same cache identity. Replacing an entry restarts its selection sequence while established channels keep their existing cache keys. HTTPS cache hits share the TLS-generation read lock, with generation validation retained before handoff. Connection misses still coalesce under the existing per-key dial lock.
|
||||
|
||||
The complete-body gate is per peer and applies to every RPC routed through the bulk client, including recovery inventories. Acquire it before the existing remote-operation deadline so queued work does not occupy tonic's transport queue or mark a healthy peer faulty. Its own bounded wait returns `DiskOngoingReq`. The permit remains owned by the caller through protobuf body decoding and application response validation. It does not reduce memory already owned by a queued WriteAll caller. Start with an explicit value such as `32`, check waiters, active work, p99 and memory, and keep the value and wait timeout identical across all nodes in one cell.
|
||||
|
||||
The body limit is read when remote-disk handles are built and therefore requires a process restart. The per-peer registry retains weak entries and uses one conservative shared overflow gate only if more than 512 peer identities remain live. Values above Tokio's semaphore capacity fail startup instead of being truncated.
|
||||
|
||||
Explicit rebuild reasons are `rpc_failure`, `operation_timeout`, `transport_failure`, `lock_timeout`, `lock_transport`, `peer_probe`, `tls_generation_changed`, and `manual`; unexpected inputs collapse to `unknown`. Outcomes are `connected` and `failed_or_cancelled`; `connected` means a replacement passed TLS-generation validation and was published or observed in the cache. A transport connection rejected during publication counts as `failed_or_cancelled` and retries under the same pending reason. Concurrent cache hits atomically consume the successful reason once. One eviction can schedule control and bulk rebuilds independently, and every failed attempt remains visible until a replacement is published. These metrics do not claim that tonic's private reconnect loop was observed.
|
||||
|
||||
The locked tonic/Hyper public API does not expose HTTP/2 PING ACK timeout outcomes. A default-off diagnostic build can enable Hyper's exact internal timeout event and the RustFS counter without adding a second timer or changing reconnect behavior:
|
||||
|
||||
```bash
|
||||
RUSTFLAGS="--cfg hyper_unstable_tracing" \
|
||||
cargo build -p rustfs --features diagnostic-internode-http2-ping
|
||||
```
|
||||
|
||||
The build scripts reject the feature when the required cfg is missing. The subscriber enables only Hyper's HTTP/2 client debug target and counts the exact `connection keep-alive timed out` event. Other low-frequency debug events from that target can also reach configured diagnostic log/export sinks, but generic connection errors, TCP closure, GOAWAY and RPC timeouts are not classified as PING failures. The feature is diagnostic-only because Hyper documents its tracing event contract as unstable; ordinary builds do not enable the dependency feature or pay its tracing cost. Tonic-internal reconnects without a RustFS cache eviction remain separate from explicit rebuild attempts. A stable production-default metric still requires an upstream typed observer.
|
||||
|
||||
Custom-CA channels use tonic's endpoint URI handling for certificate names and SNI, including IPv6 literals. The regression suite exercises real control and bulk TLS connections to IPv6 loopback and verifies that a trusted certificate for a different host is rejected.
|
||||
|
||||
When the outbound snapshot has no custom CA, channels explicitly load the platform trust roots. A configured mTLS client identity applies to both platform-root and custom-CA channels. An explicit custom CA selects its own trust store. TLS configuration loads and parses certificates on the blocking pool. The acquisition deadline covers waiting for that configuration. A timed-out blocking task can finish later and drops its endpoint, while the acquisition future owns connection establishment and channel publication. The native-root regression uses a process-local `SSL_CERT_FILE` fixture and exercises both channel classes, required client authentication, and rejection of a server outside the explicit private trust store.
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# Internode msgpack/JSON convergence runbook
|
||||
|
||||
**Use this when:** operating or changing the staged retirement of the JSON compatibility fields on internode gRPC metadata RPCs, deciding whether a fleet may enable `RUSTFS_INTERNODE_RPC_MSGPACK_ONLY`, or adding a new `*_bin` proto field.
|
||||
**Source of truth:** `crates/config/src/constants/internode.rs` (`ENV_INTERNODE_RPC_MSGPACK_ONLY`, `ENV_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED` and their compile-time default-off asserts), `crates/protos/src/node.proto` (`*_bin` fields), `decode_msgpack_or_json` in `rustfs/src/storage/rpc/node_service/disk.rs` (server side) and `crates/ecstore/src/cluster/rpc/remote_disk.rs` (client side), `crates/io-metrics/src/internode_metrics.rs` (counters).
|
||||
**Source of truth:** `crates/config/src/constants/internode.rs` (`ENV_INTERNODE_RPC_MSGPACK_ONLY`, `ENV_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED`, `ENV_INTERNODE_RPC_MSGPACK_PEER_CAPABILITY` and their compile-time default-off asserts), `crates/protos/src/node.proto` (`*_bin` fields), `decode_msgpack_or_json` in `rustfs/src/storage/rpc/node_service/disk.rs` (server side) and `crates/ecstore/src/cluster/rpc/remote_disk.rs` (client side), `crates/io-metrics/src/internode_metrics.rs` (counters).
|
||||
|
||||
This is a cross-version change and proceeds by observation-gated stages, never in one step.
|
||||
|
||||
@@ -16,6 +16,7 @@ Internode RPCs dual-encode each metadata value as a msgpack binary field (`*_bin
|
||||
| `rustfs_system_network_internode_msgpack_json_decode_total` | `direction`, `message`, `codec` | a msgpack or JSON decode succeeded |
|
||||
| `rustfs_system_network_internode_msgpack_json_fallback_total` | `direction`, `message` | a decode fell back to JSON because `_bin` was empty |
|
||||
| `rustfs_system_network_internode_msgpack_json_decode_error_total` | `direction`, `message`, `codec` | either codec failed to decode (`codec` = the failed codec) |
|
||||
| `rustfs_system_network_internode_msgpack_send_decisions_total` | `decision`, `reason` | an eligible RPC selected `msgpack_only` or `dual_write`; reasons are fixed to `manual_fleet`, `peer_capable`, `peer_unknown`, `peer_revoked`, or `not_requested` |
|
||||
|
||||
`direction="request"` is a server decoding a peer's request (`node_service/disk.rs`); `direction="response"` is a client decoding a peer's response (`cluster/rpc/remote_disk.rs`), including the list-level `ReadMultiple` / `BatchReadVersion` fallbacks. `message` is the value name (`FileInfo`, `RawFileInfo`, `ReadMultipleResp`, ...).
|
||||
|
||||
@@ -92,15 +93,55 @@ Run the fleet with both flags at their default (`false`) for at least one full o
|
||||
|
||||
## Stage 1 — Stop writing JSON (env-gated)
|
||||
|
||||
Flag semantics are defined on `ENV_INTERNODE_RPC_MSGPACK_ONLY` and `ENV_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED` in `crates/config/src/constants/internode.rs`; both default off and only their conjunction stops the JSON write for convergence-ready fields. Decoders keep the JSON fallback in every state.
|
||||
Flag semantics are defined in `crates/config/src/constants/internode.rs`; all three flags default off. `RUSTFS_INTERNODE_RPC_MSGPACK_ONLY=true` requests msgpack-only, and JSON stops only when either fleet confirmation is true or the peer-capability gate is true and the exact current peer boot epoch plus physical connection generation has an authenticated capability proof. Unknown, unsupported, revoked, stale, or reconnecting peers continue dual-write. Decoders keep the JSON fallback in every state.
|
||||
|
||||
1. Rehearse `request-only`: `RUSTFS_INTERNODE_RPC_MSGPACK_ONLY=true`, `..._FLEET_CONFIRMED=false`. Behaviour is unchanged (still dual-write) and safe with old peers; any fallback or decode-error increment still blocks.
|
||||
2. Canary: set both flags `true` on one node, restart that node only, and watch the fallback and decode-error counters for a soak period with real internode traffic.
|
||||
3. Fleet: if the counters stay zero, enable both flags fleet-wide with a rolling restart.
|
||||
4. Rollback: set either flag to `false` (or unset it) and restart; no wire format changed, so rollback is immediate.
|
||||
|
||||
An alternative per-peer canary keeps `RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED=false`, sets `RUSTFS_INTERNODE_RPC_MSGPACK_ONLY=true` and `RUSTFS_INTERNODE_RPC_MSGPACK_PEER_CAPABILITY=true`, and confirms `reason="peer_capable"` only for proved peers while every `peer_unknown` or `peer_revoked` decision remains `dual_write`. Enable this mode only when one client HTTP/2 connection has stable affinity to one RustFS peer process; an L7 proxy that routes streams from one connection to different backend versions invalidates the connection-generation proof. A connection-generation change may rebuild one request as dual-write only when the retained local guard proves rejection before the first payload frame; remote status text, a dispatched request, or an ordinary transport error never authorizes that compatibility retry. Disable either the base request flag or peer-capability flag to roll back immediately.
|
||||
|
||||
The benchmark driver pins these operator states as dry-run phases (`before`, `request-only`, `canary`, `after`, `rollback`); see [internode-grpc-benchmark-runbook.md](internode-grpc-benchmark-runbook.md).
|
||||
|
||||
### Per-peer mixed-version evidence gate
|
||||
|
||||
The per-peer gate is not qualified by a successful command transcript alone. Record three ordered four-node measurement phases: `mixed-old-peer` (three candidate nodes and one previous-version node), `all-new` (four candidate nodes), and `rollback-old-peer` (replace one candidate with the previous-version image and verify readback). Every candidate node uses `MSGPACK_ONLY=true`, `MSGPACK_PEER_CAPABILITY=true`, and `MSGPACK_ONLY_FLEET_CONFIRMED=false`; previous-version nodes use all three false. Each phase must run real PUT/GET/metadata traffic and retain the exact candidate/baseline revisions, topology, timestamps, workload result, environment, and raw Prometheus HTTP API JSON.
|
||||
|
||||
Assign one unique `msgpack-<32-hex>` cluster ID for this run, set `OTEL_RESOURCE_ATTRIBUTES=rustfs.cluster.id=<id>` on every node, and retain the collector’s resource-to-telemetry conversion so Prometheus exposes `rustfs_cluster_id`. Capture these PromQL vectors over each non-overlapping phase window and store the complete HTTP API response, not extracted numbers:
|
||||
|
||||
```promql
|
||||
sum by (decision, reason) (increase(rustfs_system_network_internode_msgpack_send_decisions_total{rustfs_cluster_id="$CLUSTER_ID"}[$WINDOW]))
|
||||
sum(increase(rustfs_system_network_internode_msgpack_json_fallback_total{rustfs_cluster_id="$CLUSTER_ID"}[$WINDOW])) or vector(0)
|
||||
sum(increase(rustfs_system_network_internode_msgpack_json_decode_error_total{rustfs_cluster_id="$CLUSTER_ID"}[$WINDOW])) or vector(0)
|
||||
sum(increase(rustfs_system_network_internode_operation_errors_total{rustfs_cluster_id="$CLUSTER_ID"}[$WINDOW])) or vector(0)
|
||||
```
|
||||
|
||||
Build a schema-1 manifest with `scope="internode-msgpack-peer-canary"`, `source_revision`, `baseline_revision`, `stable_connection_affinity=true`, `max_metric_lag_secs`, `cluster_id`, an `images` path to sanitized image inspections, and the three phase records. Each phase records integer `start_unix`/`end_unix`, exact topology counts plus a `node1..node4` revision map, the new/old node environment maps, the exact four PromQL strings with a window equal to the phase duration, an `images` path to the verified per-node image ID/revision map, a relative workload JSON path, and relative paths for `decisions`, `fallback`, `decode_errors`, and `rpc_errors`. The workload artifact records `status`, a non-zero integer `requests`, and `rollback_readback` (true only for `rollback-old-peer`). Validate without rewriting raw evidence:
|
||||
|
||||
```bash
|
||||
python3 scripts/check_internode_msgpack_peer_canary.py \
|
||||
--manifest /path/to/evidence/manifest.json \
|
||||
--out /path/to/evidence/verification.json
|
||||
```
|
||||
|
||||
The verifier rejects reused/escaping/overwritten artifacts, duplicate or non-exact decision labels, labeled scalar queries, mismatched PromQL windows, wrong per-node revisions, missing peer-capable traffic, missing mixed/rollback dual-write traffic, manual-fleet or not-requested leakage, failed workload/readback, non-zero fallback/decode/RPC errors, stale timestamps, invalid topology, and candidate/baseline identity reuse. The workload artifact binds its raw benchmark `summary.csv` by relative path and SHA-256; rollback also records equal source/download content hashes. Verifier output includes SHA-256 receipts for every raw query, workload, and summary artifact. Passing this gate is compatibility evidence only; production soak and ABBA performance drift gates remain separate.
|
||||
|
||||
Use the orchestration runner to create the isolated Compose project, execute the three phases, preserve a known object across node2 rollback, capture the four raw Prometheus responses, build the manifest, and invoke the verifier. Without `--execute` it prints a redacted plan and makes no external changes. The runner verifies distinct immutable image IDs and full `vcs-ref` or OCI revision labels against the requested revisions, pins Compose to those IDs, and checks each running container’s image and telemetry identity in every phase. Images without matching full revision labels are rejected. It retains sanitized identity receipts without container credentials. Execution refuses an existing Compose project/volume set and removes its isolated containers and volumes on exit unless `--keep-up` is explicit.
|
||||
|
||||
```bash
|
||||
scripts/run_internode_msgpack_peer_canary.sh \
|
||||
--candidate-image rustfs/rustfs:candidate \
|
||||
--baseline-image rustfs/rustfs:previous \
|
||||
--candidate-revision <40-hex-candidate-commit> \
|
||||
--baseline-revision <40-hex-previous-commit> \
|
||||
--prometheus-query-url http://127.0.0.1:9090/api/v1/query \
|
||||
--out-dir /path/to/evidence \
|
||||
--execute
|
||||
```
|
||||
|
||||
The runner requires Docker Compose, AWS CLI, curl, Warp through `run_object_batch_bench.sh`, Python, the four-node local cluster compose file, prebuilt candidate/baseline images, and an already reachable Prometheus/OTLP pipeline. It never builds, pulls, or starts observability images implicitly. Execution rejects reused Compose resources or a non-empty evidence directory and performs a Prometheus vector preflight before starting the cluster. The workload summary must contain only successful zero-error rows with parseable request rates; the rollback phase must read byte-identical object data written before the version replacement.
|
||||
|
||||
## Stage 2 — Remove the proto JSON fields
|
||||
|
||||
Only after Stage 1 has been stable fleet-wide for a full window with the counters still zero.
|
||||
@@ -113,7 +154,7 @@ Only after Stage 1 has been stable fleet-wide for a full window with the counter
|
||||
| Stage | Wire format broken? | Rollback |
|
||||
| --- | --- | --- |
|
||||
| 0 Observe | no | n/a (metrics only) |
|
||||
| 1 msgpack-only send | no | unset either flag and restart |
|
||||
| 1 msgpack-only send | no | unset the base request flag, or unset the selected fleet/peer gate, and restart |
|
||||
| 2 remove fields | yes | redeploy the prior release; field numbers stay `reserved` |
|
||||
|
||||
## Related
|
||||
|
||||
@@ -31,6 +31,12 @@ A previously published gauge can remain visible after a statistic becomes unavai
|
||||
|
||||
When live requested bytes are unavailable, capture a supported heap profile or allocation/free trace. Preserve attribution to allocation stacks and lifetime; committed bytes and RSS alone cannot establish a leak. Other raw allocator statistics, including page accounting, depend on the allocator build and are not interchangeable with live application bytes.
|
||||
|
||||
### Diagnostic allocator and runtime metrics
|
||||
|
||||
Build with `--features diagnostic-allocator-live` to count live requested bytes for Rust allocations routed through the binary's MiMalloc `GlobalAlloc` wrapper. Read `rustfs_memory_allocator_rust_global_requested_live_bytes` together with `rustfs_memory_allocator_rust_global_requested_live_bytes_available`. Availability is `1` for this feature on non-Windows builds when the `hotpath` and `hotpath-alloc` combination is inactive. Direct native allocations are outside this counter. The wrapper records successful allocations, zeroed allocations, reallocations, and frees using allocation-free atomic updates. Per-allocation accounting adds overhead, so collect diagnostic and throughput measurements in separate runs.
|
||||
|
||||
The memory sampler also exports `rustfs_runtime_worker_threads`. With `RUSTFLAGS="--cfg tokio_unstable"`, it exports serving-runtime blocking-thread total, idle, and active gauges under `rustfs_runtime_blocking_threads_*`, with `rustfs_runtime_blocking_pool_metrics_available = 1`. An ordinary build reports availability `0`. These metrics describe the serving runtime; isolated disk fsync and cleanup runtimes have their own execution ownership. Enable the allocator feature separately when both views are needed. Tokio's `tokio_unstable` configuration uses the stable compiler and opts into Tokio's unstable telemetry API.
|
||||
|
||||
### Linux transparent huge pages
|
||||
|
||||
Linux RustFS binaries disable transparent huge pages for mimalloc by default. This reduces resident memory when reclaimable allocations occupy only part of a huge page. Check `THP_enabled` in `/proc/<pid>/status` and `AnonHugePages` in `/proc/<pid>/smaps_rollup` to verify the running process. Both values should be zero with the default. The Linux binary also defaults to immediately purging free arena ranges. Set `MIMALLOC_PURGE_DELAY` before startup to override the delay in milliseconds.
|
||||
|
||||
@@ -108,6 +108,28 @@ Claim values are used verbatim as policy names (after `claim_prefix`, if any). N
|
||||
|
||||
For first-contact testing only, `RUSTFS_IDENTITY_OPENID_ROLE_POLICY=consoleAdmin` grants every login full access; remove it before production.
|
||||
|
||||
### Creating access keys for a local user
|
||||
|
||||
In the Console, open **Users → select the local user → Access Keys → Create**. An unrestricted root Console session or a local or OIDC administrator with the built-in `consoleAdmin` policy can create a key for an enabled local user. The key uses the target user's permissions and any narrower session policy supplied at creation.
|
||||
|
||||
For a custom administrator, grant both actions. The following policy permits delegated creation only for `backup-user`:
|
||||
|
||||
```json
|
||||
{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{"Effect": "Allow", "Action": "admin:CreateServiceAccount"},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": "admin:CreateServiceAccountForUser",
|
||||
"Condition": {"StringEquals": {"svc:TargetUser": "backup-user"}}
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
Self-service uses the existing `admin:CreateServiceAccount` permission. The built-in `consoleAdmin` policy and custom policies granting `admin:*` cover both actions. This also applies to existing wildcard grants after an upgrade. To exclude delegation from a wildcard grant, add an explicit deny for `admin:CreateServiceAccountForUser`. Deploy the new server version to all nodes before installing policies that name the new action or condition.
|
||||
|
||||
## Validation
|
||||
|
||||
1. Discovery:
|
||||
|
||||
@@ -193,3 +193,19 @@ The 1024 PUT admission limit includes uploading requests and detached owners;
|
||||
it is an object-count bound, not a memory or byte budget. The older deferred queue
|
||||
remains optional and disabled by default; its overflow path cannot bypass the
|
||||
new coordinator or disk limits. No unbounded foreground fallback is introduced.
|
||||
|
||||
## Experimental file-sync follower timeout
|
||||
|
||||
`RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_ENABLE` enables file-sync grouping and defaults to `false`. `RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS` controls the batch collection window, defaults to `0`, and is capped at `1000` microseconds. `RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_FOLLOWER_TIMEOUT_MICROS` controls how long a follower waits before trying to leave a queued batch. It defaults to `0` (disabled) and is capped at `1000` microseconds. Configure these process-lifetime settings before startup.
|
||||
|
||||
After its deadline, a follower can atomically take ownership of its queued files and resource unit. It then executes file sync through the existing per-disk and global limits, followed by source-directory fsync in the normal caller. The original batch loses those paths before the follower can publish its data directory. A worker that has already claimed the batch retains exclusive ownership, and the follower waits for that worker's result. The timeout therefore bounds queued batching delay; disk admission and native sync can take longer. Cancellation retains the existing native-work resource lifetime and worker cleanup guarantees.
|
||||
|
||||
The worker guard owns batch claim, in-flight accounting and idle publication. Claiming an empty queue publishes the idle state under the same group lock used by enqueue. A request arriving during cleanup can establish successor-worker ownership, and stale idle cleanup checks that ownership before removing the group.
|
||||
|
||||
Pending followers use indexed slab slots and a monotonic waiter identity. Detaching a queued follower performs constant-time lookup under the coordinator locks. The identity check protects a claimed request when its old slot is reused by a later batch. Queue capacity remains bounded by the existing file and waiter limits.
|
||||
|
||||
A source that disappears before native open fails its own waiter, while healthy requests in that batch still complete file and directory sync. Other file-sync errors retain the existing fail-fast batch behavior, preventing repeated I/O against a failing disk. Admission or native-dispatch failure also fails the batch. File sets and their existing resource units move together into one blocking dispatch, and result senders remain with the async worker so cancellation keeps its established notification contract.
|
||||
|
||||
With detailed PUT metrics enabled, a successful queue ownership transfer adds a `follower_timeout` role sample to `rustfs_s3_put_object_rename_fdatasync_group_wait_ms`. Evaluate the experiment against the same source, durability setting, topology, and workload. Include sync failures, follower publication before batch completion, cancellation, and peak resource usage in acceptance.
|
||||
|
||||
Grouped worker batches use `mode="grouped"` on `rustfs_s3_put_object_rename_fdatasync_batch_files`. The `parallel` mode is reserved for the independent per-file fanout path, so dashboards can distinguish serialized user-space grouping from bounded parallel dispatch.
|
||||
|
||||
Generated
+6
-6
@@ -2,11 +2,11 @@
|
||||
"nodes": {
|
||||
"nixpkgs": {
|
||||
"locked": {
|
||||
"lastModified": 1793977579,
|
||||
"narHash": "sha256-WrSODdZFVIs/Hh2NzC0bY+ROByi0bATg8WfePX8dPj8=",
|
||||
"lastModified": 1794495034,
|
||||
"narHash": "sha256-E0hYoioqUtM2XmPPvYYpJgdADF5AW9oOoPSEiM9dTHQ=",
|
||||
"owner": "NixOS",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "4fe25e12eb9f4c744966dd3de5a05246cf098601",
|
||||
"rev": "2074cc672349f14240fb7ea90d509e0b643e9dd4",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
@@ -29,11 +29,11 @@
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1794040562,
|
||||
"narHash": "sha256-7eEiezSku3lbZPUOXW8we1a7+xd5axkCQdyI3rzOKjo=",
|
||||
"lastModified": 1794646145,
|
||||
"narHash": "sha256-rpqDyUM/d5fACYB8W+wKxxO57ICWjjalzb2X25oFFkY=",
|
||||
"owner": "oxalica",
|
||||
"repo": "rust-overlay",
|
||||
"rev": "d3fe74269f2a6652437e21e78eb7e83be81f6982",
|
||||
"rev": "5b87ee0513c29127a01ad06e9cb362ea5ce91e78",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
|
||||
@@ -101,6 +101,12 @@ dial9 = ["rustfs-obs/dial9"]
|
||||
diagnostic-allocator-live = []
|
||||
# Default-off sampled local batch wall/CPU and request correlation diagnostics.
|
||||
diagnostic-local-batch = ["rustfs-ecstore/diagnostic-local-batch", "rustfs-obs/diagnostic-local-batch"]
|
||||
# Exact Hyper HTTP/2 PING ACK timeout accounting. Diagnostic-only because the
|
||||
# upstream tracing event is unstable and requires an explicit build cfg.
|
||||
diagnostic-internode-http2-ping = [
|
||||
"rustfs-obs/diagnostic-internode-http2-ping",
|
||||
"rustfs-protos/diagnostic-internode-http2-ping",
|
||||
]
|
||||
# Diagnostic-only S3-operation attribution for internode ReadVersion and list-walker metrics.
|
||||
diagnostic-read-version-caller = ["rustfs-ecstore/diagnostic-read-version-caller", "rustfs-io-metrics/diagnostic-read-version-caller", "rustfs-scanner/diagnostic-read-version-caller"]
|
||||
# Diagnostic-only fixed-prefix attribution for actual Scanner metadata visits.
|
||||
|
||||
@@ -15,6 +15,7 @@
|
||||
use std::io::Write;
|
||||
|
||||
fn main() -> shadow_rs::SdResult<()> {
|
||||
println!("cargo:rustc-check-cfg=cfg(tokio_unstable)");
|
||||
println!("cargo:rerun-if-env-changed=RUSTFS_BUILD_VERSION");
|
||||
if let Ok(version) = std::env::var("RUSTFS_BUILD_VERSION")
|
||||
&& !version.is_empty()
|
||||
|
||||
@@ -57,7 +57,8 @@ use tracing::{Span, debug, error, info, instrument, warn};
|
||||
use tracing_opentelemetry::OpenTelemetrySpanExt;
|
||||
|
||||
#[derive(RustEmbed)]
|
||||
#[folder = "$CARGO_MANIFEST_DIR/static"]
|
||||
#[cfg_attr(test, folder = "$CARGO_MANIFEST_DIR/tests/fixtures/native-console")]
|
||||
#[cfg_attr(not(test), folder = "$CARGO_MANIFEST_DIR/static")]
|
||||
struct StaticFiles;
|
||||
|
||||
/// Static file handler
|
||||
|
||||
@@ -41,7 +41,7 @@ use rustfs_madmin::{
|
||||
ServiceAccountInfo, TemporaryAccountInfoResp, UpdateServiceAccountReq,
|
||||
};
|
||||
use rustfs_policy::policy::action::{Action, AdminAction};
|
||||
use rustfs_policy::policy::{Args, DEFAULT_VERSION, Policy};
|
||||
use rustfs_policy::policy::{Args, DEFAULT_VERSION, Policy, SVC_TARGET_USER};
|
||||
use rustfs_s3_types::{Body, S3ErrorCode::InvalidRequest};
|
||||
use rustfs_s3_types::{S3Error, S3ErrorCode, S3Result, s3_error};
|
||||
use serde::Deserialize;
|
||||
@@ -139,19 +139,8 @@ fn is_service_account_owner_of(caller: &StoredCredentials, target_parent_user: &
|
||||
caller_parent == target_parent_user
|
||||
}
|
||||
|
||||
/// GHSA-5354: whether a caller resolving to `owner` may create a service account
|
||||
/// parented to `target_user`, given the caller's own key (`req_user`) and its
|
||||
/// effective parent (`req_parent_user`, which equals `req_user` for a top-level
|
||||
/// user or the parent for a derived credential).
|
||||
///
|
||||
/// The `CreateServiceAccountAdminAction` permission gates *whether* a caller may
|
||||
/// create service accounts, not *for whom*. A non-owner is therefore confined to
|
||||
/// its own non-root scope; only an owner may target another user — including the root
|
||||
/// credential, whose existence check is deliberately lenient. Without this a
|
||||
/// caller holding only that action could pass `target_user = <root access key>`
|
||||
/// and mint a root-parented service account that authenticates as owner via
|
||||
/// `prepare_service_account_auth`, a full-takeover privilege escalation. Mirrors
|
||||
/// `imported_service_account_parent_allowed` on the ImportIam path (GHSA-566f).
|
||||
/// Scope available with the base create permission. Creating for another local
|
||||
/// user requires separate authorization in IAM, even when the caller is owner.
|
||||
fn add_service_account_parent_within_scope(
|
||||
root_access_key: &str,
|
||||
target_user: &str,
|
||||
@@ -332,6 +321,73 @@ pub fn register_service_account_route(r: &mut S3Router<AdminOperation>) -> std::
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn map_service_account_create_error(error: rustfs_iam::error::Error) -> S3Error {
|
||||
match error {
|
||||
rustfs_iam::error::Error::NoSuchUser(_) => s3_error!(InvalidRequest, "target user not exist"),
|
||||
rustfs_iam::error::Error::InvalidArgument
|
||||
| rustfs_iam::error::Error::InvalidAccessKeyLength
|
||||
| rustfs_iam::error::Error::InvalidSecretKeyLength
|
||||
| rustfs_iam::error::Error::AccessKeyAlreadyExists => iam_error_to_s3_error(error),
|
||||
rustfs_iam::error::Error::IAMActionNotAllowed => s3_error!(AccessDenied, "access denied"),
|
||||
_ => s3_error!(InternalError, "create service account failed"),
|
||||
}
|
||||
}
|
||||
|
||||
// The Admin operation owns the local write and site notification together.
|
||||
// A disconnected requester can stop waiting without splitting those effects.
|
||||
async fn create_and_replicate_service_account<R: Future<Output = S3Result<()>> + Send>(
|
||||
create: impl Future<Output = S3Result<(StoredCredentials, OffsetDateTime, HashMap<String, serde_json::Value>)>> + Send + 'static,
|
||||
replication_policy: Option<String>,
|
||||
replication_expiration: Option<OffsetDateTime>,
|
||||
replicate: impl FnOnce(SRIAMItem) -> R + Send + 'static,
|
||||
) -> S3Result<StoredCredentials> {
|
||||
super::supervise_admin_mutation("service account creation", async move {
|
||||
let (new_cred, updated_at, replication_claims) = create.await?;
|
||||
let (replication_session_policy, oidc_service_account_envelope) =
|
||||
encode_service_account_replication_policy(&replication_claims, replication_policy.as_deref())?;
|
||||
|
||||
if let Err(err) = replicate(SRIAMItem {
|
||||
r#type: "service-account".to_string(),
|
||||
svc_acc_change: Some(SRSvcAccChange {
|
||||
create: Some(SRSvcAccCreate {
|
||||
parent: new_cred.parent_user.clone(),
|
||||
access_key: new_cred.access_key.clone(),
|
||||
secret_key: new_cred.secret_key.clone(),
|
||||
groups: new_cred.groups.clone().unwrap_or_default(),
|
||||
claims: replication_claims,
|
||||
session_policy: replication_session_policy,
|
||||
status: String::new(),
|
||||
name: new_cred.name.clone().unwrap_or_default(),
|
||||
description: new_cred.description.clone().unwrap_or_default(),
|
||||
expiration: replication_expiration,
|
||||
api_version: Some(SITE_REPL_API_VERSION.to_string()),
|
||||
}),
|
||||
oidc_service_account_envelope,
|
||||
api_version: Some(SITE_REPL_API_VERSION.to_string()),
|
||||
..Default::default()
|
||||
}),
|
||||
updated_at: Some(updated_at),
|
||||
api_version: Some(SITE_REPL_API_VERSION.to_string()),
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
{
|
||||
warn!(
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_SERVICE_ACCOUNT,
|
||||
event = EVENT_ADMIN_SERVICE_ACCOUNT_STATE,
|
||||
access_key = %new_cred.access_key,
|
||||
action = "create",
|
||||
result = "site_replication_hook_failed",
|
||||
error = ?err,
|
||||
"admin service account state"
|
||||
);
|
||||
}
|
||||
Ok(new_cred)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
pub struct AddServiceAccount {}
|
||||
#[async_trait::async_trait]
|
||||
impl Operation for AddServiceAccount {
|
||||
@@ -376,20 +432,13 @@ impl Operation for AddServiceAccount {
|
||||
return Err(s3_error!(InvalidArgument, "can't create user with system access key"));
|
||||
}
|
||||
|
||||
let mut target_user = if let Some(u) = create_req.target_user {
|
||||
u
|
||||
} else {
|
||||
cred.access_key.clone()
|
||||
};
|
||||
|
||||
let req_user = cred.access_key.clone();
|
||||
let mut req_parent_user = cred.access_key.clone();
|
||||
let req_groups = cred.groups.clone();
|
||||
let mut req_is_derived_cred = false;
|
||||
|
||||
if cred.is_service_account() || cred.is_temp() {
|
||||
req_parent_user = cred.parent_user.clone();
|
||||
req_is_derived_cred = true;
|
||||
let requested_user = create_req.target_user.as_deref().unwrap_or(&cred.access_key);
|
||||
let derived = cred.is_service_account() || cred.is_temp();
|
||||
let caller_parent = if derived { &cred.parent_user } else { &cred.access_key };
|
||||
let self_scoped = requested_user == cred.access_key || requested_user == caller_parent;
|
||||
let target_user = if self_scoped { caller_parent.as_str() } else { requested_user };
|
||||
if target_user.is_empty() {
|
||||
return Err(s3_error!(AccessDenied, "service account requires a parent"));
|
||||
}
|
||||
|
||||
let Ok(iam_store) = crate::admin::runtime_sources::current_ready_iam_handle() else {
|
||||
@@ -400,48 +449,45 @@ impl Operation for AddServiceAccount {
|
||||
let mut conditions = get_condition_values(req.headers(), &cred, None, None, req.policy_source_addr());
|
||||
set_service_account_duration_condition(&mut conditions, create_req.expiration, OffsetDateTime::now_utc());
|
||||
|
||||
// This family deliberately calls `is_allowed` directly instead of going
|
||||
// through `validate_admin_request`, and must keep doing so
|
||||
// (backlog#1886). The shared helper returns as soon as *any* candidate
|
||||
// action is allowed — an OR. The checks here are an AND: each one must
|
||||
// pass, and a later stage additionally needs `owner` for the GHSA-5354
|
||||
// parent-scope guard and drives `deny_only` dynamically. Replacing these
|
||||
// with the shared gate would widen authorization.
|
||||
if !iam_store
|
||||
.is_allowed(&Args {
|
||||
account: &cred.access_key,
|
||||
groups: &cred.groups,
|
||||
action: Action::AdminAction(AdminAction::CreateServiceAccountAdminAction),
|
||||
bucket: "",
|
||||
conditions: &conditions,
|
||||
is_owner: owner,
|
||||
object: "",
|
||||
claims: cred.claims_or_empty(),
|
||||
deny_only: false, // Always require explicit Allow permission
|
||||
})
|
||||
.await
|
||||
{
|
||||
return Err(s3_error!(AccessDenied, "access denied"));
|
||||
}
|
||||
|
||||
if target_user != cred.access_key {
|
||||
let has_user = iam_store.get_user(&target_user).await;
|
||||
if has_user.is_none() && target_user != sys_cred.access_key {
|
||||
// The policy and the write must see the same resolved parent, including
|
||||
// when a derived credential names itself. OPA receives this map too.
|
||||
conditions.retain(|key, _| !key.eq_ignore_ascii_case(SVC_TARGET_USER));
|
||||
conditions.insert(SVC_TARGET_USER.to_owned(), vec![target_user.to_owned()]);
|
||||
if self_scoped {
|
||||
if !iam_store
|
||||
.is_allowed(&Args {
|
||||
account: &cred.access_key,
|
||||
groups: &cred.groups,
|
||||
action: Action::AdminAction(AdminAction::CreateServiceAccountAdminAction),
|
||||
bucket: "",
|
||||
conditions: &conditions,
|
||||
is_owner: owner,
|
||||
object: "",
|
||||
claims: cred.claims_or_empty(),
|
||||
deny_only: false,
|
||||
})
|
||||
.await
|
||||
{
|
||||
return Err(s3_error!(AccessDenied, "access denied"));
|
||||
}
|
||||
if !add_service_account_parent_within_scope(
|
||||
&sys_cred.access_key,
|
||||
requested_user,
|
||||
&cred.access_key,
|
||||
caller_parent,
|
||||
owner,
|
||||
) {
|
||||
return Err(s3_error!(AccessDenied, "service account parent is outside requester scope"));
|
||||
}
|
||||
if requested_user != cred.access_key
|
||||
&& requested_user != sys_cred.access_key
|
||||
&& iam_store.get_user(requested_user).await.is_none()
|
||||
{
|
||||
return Err(s3_error!(InvalidRequest, "target user not exist"));
|
||||
}
|
||||
}
|
||||
|
||||
let is_svc_acc = target_user == req_user || target_user == req_parent_user;
|
||||
// GHSA-5354: confine a non-owner caller to its own scope so it cannot mint
|
||||
// a root-parented (owner) service account. Evaluated on the original
|
||||
// `target_user` the caller submitted, before the derived-credential rewrite
|
||||
// to `req_parent_user` below, so the guard is bound to the parent actually
|
||||
// requested and to the effective parent after that rewrite.
|
||||
if !add_service_account_parent_within_scope(&sys_cred.access_key, &target_user, &req_user, &req_parent_user, owner) {
|
||||
return Err(s3_error!(AccessDenied, "service account parent is outside requester scope"));
|
||||
}
|
||||
|
||||
let mut target_groups = None;
|
||||
let target_groups = if self_scoped { cred.groups.clone() } else { None };
|
||||
let opts = NewServiceAccountOpts {
|
||||
access_key: create_req.access_key,
|
||||
secret_key: create_req.secret_key,
|
||||
@@ -452,84 +498,39 @@ impl Operation for AddServiceAccount {
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
if is_svc_acc {
|
||||
if req_is_derived_cred {
|
||||
if req_parent_user.is_empty() {
|
||||
return Err(s3_error!(AccessDenied, "only derived cred can create service account"));
|
||||
}
|
||||
target_user = req_parent_user;
|
||||
}
|
||||
|
||||
target_groups = req_groups;
|
||||
}
|
||||
|
||||
let replication_groups = target_groups.clone().unwrap_or_default();
|
||||
let replication_name = opts.name.clone().unwrap_or_default();
|
||||
let replication_description = opts.description.clone().unwrap_or_default();
|
||||
let target_user = target_user.to_owned();
|
||||
let response_secret = cred.secret_key.clone();
|
||||
let replication_expiration = opts.expiration;
|
||||
|
||||
let create_result = iam_store
|
||||
.new_service_account_from_caller(&target_user, target_groups, opts, &cred)
|
||||
.await;
|
||||
let (new_cred, updated_at, replication_claims) = create_result.map_err(|e| {
|
||||
debug!(
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_SERVICE_ACCOUNT,
|
||||
event = EVENT_ADMIN_SERVICE_ACCOUNT_STATE,
|
||||
target_user = %target_user,
|
||||
result = "create_failed",
|
||||
error = ?e,
|
||||
"admin service account state"
|
||||
);
|
||||
match e {
|
||||
rustfs_iam::error::Error::InvalidArgument
|
||||
| rustfs_iam::error::Error::InvalidAccessKeyLength
|
||||
| rustfs_iam::error::Error::InvalidSecretKeyLength
|
||||
| rustfs_iam::error::Error::AccessKeyAlreadyExists => iam_error_to_s3_error(e),
|
||||
rustfs_iam::error::Error::IAMActionNotAllowed => s3_error!(AccessDenied, "access denied"),
|
||||
err => s3_error!(InternalError, "create service account failed, e: {:?}", err),
|
||||
}
|
||||
})?;
|
||||
let (replication_session_policy, oidc_service_account_envelope) =
|
||||
encode_service_account_replication_policy(&replication_claims, replication_policy.as_deref())?;
|
||||
|
||||
if let Err(err) = site_replication_iam_change_hook(SRIAMItem {
|
||||
r#type: "service-account".to_string(),
|
||||
svc_acc_change: Some(SRSvcAccChange {
|
||||
create: Some(SRSvcAccCreate {
|
||||
parent: target_user.clone(),
|
||||
access_key: new_cred.access_key.clone(),
|
||||
secret_key: new_cred.secret_key.clone(),
|
||||
groups: replication_groups,
|
||||
claims: replication_claims,
|
||||
session_policy: replication_session_policy,
|
||||
status: String::new(),
|
||||
name: replication_name,
|
||||
description: replication_description,
|
||||
expiration: replication_expiration,
|
||||
api_version: Some(SITE_REPL_API_VERSION.to_string()),
|
||||
}),
|
||||
oidc_service_account_envelope,
|
||||
api_version: Some(SITE_REPL_API_VERSION.to_string()),
|
||||
..Default::default()
|
||||
}),
|
||||
updated_at: Some(updated_at),
|
||||
api_version: Some(SITE_REPL_API_VERSION.to_string()),
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
{
|
||||
warn!(
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_SERVICE_ACCOUNT,
|
||||
event = EVENT_ADMIN_SERVICE_ACCOUNT_STATE,
|
||||
access_key = %new_cred.access_key,
|
||||
action = "create",
|
||||
result = "site_replication_hook_failed",
|
||||
error = ?err,
|
||||
"admin service account state"
|
||||
);
|
||||
}
|
||||
let create = async move {
|
||||
let create_result = if self_scoped {
|
||||
iam_store
|
||||
.new_service_account_from_caller(&target_user, target_groups, opts, &cred)
|
||||
.await
|
||||
} else {
|
||||
iam_store
|
||||
.new_service_account_for_user(&target_user, opts, &cred, &conditions, owner)
|
||||
.await
|
||||
};
|
||||
create_result.map_err(|e| {
|
||||
debug!(
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_SERVICE_ACCOUNT,
|
||||
event = EVENT_ADMIN_SERVICE_ACCOUNT_STATE,
|
||||
target_user = %target_user,
|
||||
result = "create_failed",
|
||||
error = ?e,
|
||||
"admin service account state"
|
||||
);
|
||||
map_service_account_create_error(e)
|
||||
})
|
||||
};
|
||||
let new_cred = create_and_replicate_service_account(
|
||||
create,
|
||||
replication_policy,
|
||||
replication_expiration,
|
||||
site_replication_iam_change_hook,
|
||||
)
|
||||
.await?;
|
||||
|
||||
let resp = AddServiceAccountResp {
|
||||
credentials: Credentials {
|
||||
@@ -541,7 +542,7 @@ impl Operation for AddServiceAccount {
|
||||
};
|
||||
|
||||
let body = serde_json::to_vec(&resp).map_err(|e| s3_error!(InternalError, "marshal body failed, e: {:?}", e))?;
|
||||
let (body, content_type) = encode_compatible_admin_payload(req.uri().path(), &cred.secret_key, body)?;
|
||||
let (body, content_type) = encode_compatible_admin_payload(req.uri().path(), &response_secret, body)?;
|
||||
|
||||
let mut header = HeaderMap::new();
|
||||
header.insert(CONTENT_TYPE, content_type.parse().expect("valid header value"));
|
||||
@@ -1851,22 +1852,58 @@ mod tests {
|
||||
assert_eq!(err.message(), Some("service account 'missing' does not exist"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delegated_service_account_replication_survives_request_cancellation() {
|
||||
use tokio::sync::oneshot;
|
||||
use tokio::time::{Duration, timeout};
|
||||
|
||||
let (started_tx, started_rx) = oneshot::channel();
|
||||
let (release_tx, release_rx) = oneshot::channel();
|
||||
let (event_tx, event_rx) = oneshot::channel();
|
||||
let updated_at = OffsetDateTime::now_utc();
|
||||
let create = async move {
|
||||
started_tx.send(()).expect("signal creation");
|
||||
release_rx.await.expect("release write");
|
||||
let credentials = StoredCredentials {
|
||||
access_key: "backup-key".to_owned(),
|
||||
parent_user: "backup-user".to_owned(),
|
||||
..Default::default()
|
||||
};
|
||||
Ok((credentials, updated_at, HashMap::new()))
|
||||
};
|
||||
let waiter = tokio::spawn(create_and_replicate_service_account(create, None, None, |item| async move {
|
||||
event_tx.send(item).expect("capture site replication event");
|
||||
Ok(())
|
||||
}));
|
||||
let deadline = Duration::from_secs(5);
|
||||
timeout(deadline, started_rx)
|
||||
.await
|
||||
.expect("creation started")
|
||||
.expect("start signal");
|
||||
waiter.abort();
|
||||
assert!(waiter.await.expect_err("request cancelled").is_cancelled());
|
||||
release_tx.send(()).expect("write still admitted");
|
||||
let item = timeout(deadline, event_rx)
|
||||
.await
|
||||
.expect("replication completed")
|
||||
.expect("replication event");
|
||||
assert_eq!(item.updated_at, Some(updated_at));
|
||||
let created = item.svc_acc_change.and_then(|change| change.create).expect("create event");
|
||||
assert_eq!(created.access_key, "backup-key");
|
||||
assert_eq!(created.parent, "backup-user");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn add_service_account_maps_iam_create_errors_to_s3_errors() {
|
||||
let src = include_str!("service_account.rs");
|
||||
let create_start = src
|
||||
.find("impl Operation for AddServiceAccount")
|
||||
.expect("AddServiceAccount operation should exist");
|
||||
let create_block = &src[create_start..];
|
||||
let create_end = create_block
|
||||
.find("if let Err(err) = site_replication_iam_change_hook")
|
||||
.expect("site replication hook marker should exist");
|
||||
let create_block = &create_block[..create_end];
|
||||
use rustfs_iam::error::Error;
|
||||
|
||||
assert!(
|
||||
create_block.contains("AccessKeyAlreadyExists") && create_block.contains("iam_error_to_s3_error(e)"),
|
||||
"duplicate access keys must map to client S3 errors"
|
||||
);
|
||||
for (error, expected) in [
|
||||
(Error::AccessKeyAlreadyExists, S3ErrorCode::InvalidArgument),
|
||||
(Error::NoSuchUser("missing".into()), S3ErrorCode::InvalidRequest),
|
||||
(Error::Io(std::io::Error::other("storage unavailable")), S3ErrorCode::InternalError),
|
||||
] {
|
||||
assert_eq!(map_service_account_create_error(error).code(), &expected);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -922,6 +922,14 @@ pub(crate) fn claims_route(method: &Method, uri: &Uri, headers: &HeaderMap) -> b
|
||||
|| is_control_plane_route(method, uri, headers)
|
||||
}
|
||||
|
||||
/// Console paths whose requests do not belong to an earlier extension route.
|
||||
#[cfg(feature = "gateway-stack")]
|
||||
pub(crate) fn claims_native_console_route(method: &Method, uri: &Uri) -> bool {
|
||||
is_console_path(uri.path())
|
||||
&& parse_object_lambda_request(method, uri).is_none()
|
||||
&& s3_extension::route_key(method, uri).is_none()
|
||||
}
|
||||
|
||||
fn is_public_health_path(path: &str) -> bool {
|
||||
path == HEALTH_PREFIX || path == HEALTH_READY_PATH
|
||||
}
|
||||
|
||||
@@ -33,9 +33,9 @@ use super::storage_api::bucket_usecase::bucket::{
|
||||
run_stale_multipart_upload_cleanup_once, validate_lifecycle_config, validate_transition_tier,
|
||||
},
|
||||
metadata::{
|
||||
BUCKET_CORS_CONFIG, BUCKET_LIFECYCLE_CONFIG, BUCKET_NOTIFICATION_CONFIG, BUCKET_POLICY_CONFIG,
|
||||
BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG, BUCKET_REPLICATION_CONFIG, BUCKET_SSECONFIG, BUCKET_TARGETS_FILE,
|
||||
BUCKET_VERSIONING_CONFIG,
|
||||
BUCKET_ACCELERATE_CONFIG, BUCKET_CORS_CONFIG, BUCKET_LIFECYCLE_CONFIG, BUCKET_NOTIFICATION_CONFIG, BUCKET_POLICY_CONFIG,
|
||||
BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG, BUCKET_REPLICATION_CONFIG, BUCKET_REQUEST_PAYMENT_CONFIG, BUCKET_SSECONFIG,
|
||||
BUCKET_TARGETS_FILE, BUCKET_VERSIONING_CONFIG, BUCKET_WEBSITE_CONFIG,
|
||||
},
|
||||
metadata_sys,
|
||||
policy_sys::PolicySys,
|
||||
@@ -61,10 +61,7 @@ use super::storage_api::bucket_usecase::s3_api::bucket::{
|
||||
ListObjectVersionsParams, ListObjectsV2Params, encode_list_objects_v2_continuation_token, parse_list_object_versions_params,
|
||||
parse_list_objects_v2_params, rustfs_owner,
|
||||
};
|
||||
use super::storage_api::bucket_usecase::{
|
||||
get_validated_store, process_lambda_configurations, process_queue_configurations, process_topic_configurations,
|
||||
request_context, validate_list_object_unordered_with_delimiter,
|
||||
};
|
||||
use super::storage_api::bucket_usecase::{get_validated_store, request_context, validate_list_object_unordered_with_delimiter};
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
use super::storage_api::diagnostic;
|
||||
use crate::app::RequestEnvelope;
|
||||
@@ -95,7 +92,6 @@ use rustfs_gateway_types::persistence::{
|
||||
PersistedVersioningConfiguration,
|
||||
};
|
||||
use rustfs_gateway_types::persistence::{PersistedLifecycleConfiguration, PersistedReplicationConfiguration};
|
||||
use rustfs_io_metrics::record_s3_op;
|
||||
use rustfs_madmin::{SITE_REPL_API_VERSION, SRBucketMeta};
|
||||
use rustfs_policy::policy::{
|
||||
action::{Action, S3Action},
|
||||
@@ -128,6 +124,8 @@ mod core;
|
||||
#[cfg(any(test, feature = "gateway-stack"))]
|
||||
pub(crate) use self::core::requested_location_constraint;
|
||||
mod cors;
|
||||
#[cfg(test)]
|
||||
mod delete_config_tests;
|
||||
mod encryption;
|
||||
#[cfg(test)]
|
||||
mod guards;
|
||||
@@ -138,6 +136,7 @@ pub(crate) mod list_objects;
|
||||
mod listing;
|
||||
mod logging;
|
||||
mod notification;
|
||||
pub(crate) use notification::NotificationDestination;
|
||||
mod object_lock;
|
||||
mod policy;
|
||||
mod public_access_block;
|
||||
@@ -275,6 +274,23 @@ pub(crate) async fn notify_table_bucket_metadata_reload(
|
||||
notify_bucket_metadata_reload_with_context(bucket, operation, None, false, app_context).await;
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
type BucketMetadataReloadObserver = Box<dyn Fn(bool, &'static str, bool) + Send + Sync>;
|
||||
|
||||
#[cfg(test)]
|
||||
tokio::task_local! {
|
||||
static BUCKET_METADATA_RELOAD_PROBE: (String, BucketMetadataReloadObserver);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn record_bucket_metadata_reload(bucket: &str, completed: bool, operation: &'static str, scanner_maintenance_change: bool) {
|
||||
let _ = BUCKET_METADATA_RELOAD_PROBE.try_with(|(observed_bucket, record)| {
|
||||
if observed_bucket == bucket {
|
||||
record(completed, operation, scanner_maintenance_change);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async fn notify_bucket_metadata_reload_with_context(
|
||||
bucket: String,
|
||||
operation: &'static str,
|
||||
@@ -282,6 +298,11 @@ async fn notify_bucket_metadata_reload_with_context(
|
||||
scanner_maintenance_change: bool,
|
||||
app_context: Option<Arc<AppContext>>,
|
||||
) {
|
||||
#[cfg(test)]
|
||||
record_bucket_metadata_reload(&bucket, false, operation, scanner_maintenance_change);
|
||||
#[cfg(test)]
|
||||
let observed_bucket = bucket.clone();
|
||||
|
||||
record_local_scanner_maintenance_reload(&bucket, scanner_maintenance_change);
|
||||
// Keep reload detached across request cancellation, but wait before a healthy peer can serve the previous config.
|
||||
let (completed_tx, completed_rx) = tokio::sync::oneshot::channel();
|
||||
@@ -303,7 +324,12 @@ async fn notify_bucket_metadata_reload_with_context(
|
||||
}
|
||||
let _ = completed_tx.send(());
|
||||
});
|
||||
let _ = completed_rx.await;
|
||||
let completed = completed_rx.await;
|
||||
#[cfg(test)]
|
||||
if completed.is_ok() {
|
||||
record_bucket_metadata_reload(&observed_bucket, true, operation, scanner_maintenance_change);
|
||||
}
|
||||
let _ = completed;
|
||||
}
|
||||
|
||||
fn record_local_scanner_maintenance_reload(bucket: &str, scanner_maintenance_change: bool) {
|
||||
|
||||
@@ -12,4 +12,44 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Bucket accelerate configuration use-case operations (reserved for the next batch).
|
||||
//! Bucket accelerate configuration shared by the S3 edges.
|
||||
|
||||
use super::BUCKET_ACCELERATE_CONFIG;
|
||||
use super::{
|
||||
ApiError, DefaultBucketUsecase, StorageError, bucket_config_mutation_incarnation, metadata_sys, serialize,
|
||||
update_bucket_config_for_incarnation,
|
||||
};
|
||||
use crate::app::RequestEnvelope;
|
||||
use rustfs_gateway_types::persistence::PersistedAccelerateConfiguration;
|
||||
use rustfs_s3_types::S3Result;
|
||||
|
||||
impl DefaultBucketUsecase {
|
||||
pub(crate) async fn execute_get_bucket_accelerate_configuration(
|
||||
&self,
|
||||
bucket: &str,
|
||||
) -> S3Result<PersistedAccelerateConfiguration> {
|
||||
self.ensure_config_bucket(bucket).await?;
|
||||
match metadata_sys::get_accelerate_config(bucket).await {
|
||||
Ok((config, _)) => Ok(config),
|
||||
Err(StorageError::ConfigNotFound) => Ok(PersistedAccelerateConfiguration::default()),
|
||||
Err(err) => Err(ApiError::from(err).into()),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn execute_put_bucket_accelerate_configuration(
|
||||
&self,
|
||||
envelope: RequestEnvelope,
|
||||
bucket: &str,
|
||||
configuration: impl FnOnce() -> S3Result<PersistedAccelerateConfiguration> + Send,
|
||||
) -> S3Result<()> {
|
||||
let expected_incarnation_id = bucket_config_mutation_incarnation(&envelope, bucket)?;
|
||||
self.ensure_config_bucket(bucket).await?;
|
||||
|
||||
let config = configuration()?;
|
||||
let data = serialize(&config).map_err(|err| rustfs_s3_types::s3_error!(MalformedXML, "{err}"))?;
|
||||
update_bucket_config_for_incarnation(bucket, BUCKET_ACCELERATE_CONFIG, data, expected_incarnation_id)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,6 +17,25 @@
|
||||
use super::*;
|
||||
|
||||
impl DefaultBucketUsecase {
|
||||
pub(crate) async fn execute_put_bucket_acl(
|
||||
&self,
|
||||
envelope: RequestEnvelope,
|
||||
bucket: &str,
|
||||
has_xml_policy: bool,
|
||||
) -> rustfs_s3_types::S3Result<()> {
|
||||
RequestEnvelope::audit_only_operation(envelope, S3Operation::PutBucketAcl, StatusCode::OK, |_| async move {
|
||||
self.ensure_config_bucket(bucket).await?;
|
||||
if has_xml_policy {
|
||||
return Err(rustfs_s3_types::s3_error!(
|
||||
NotImplemented,
|
||||
"ACL XML grants are not supported; use canned ACL headers or omit ACL"
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
/// RustFS has one owner and no ACL storage: every bucket answers that owner with full control.
|
||||
pub async fn execute_get_bucket_acl(
|
||||
&self,
|
||||
|
||||
@@ -55,6 +55,18 @@ pub(crate) fn requested_location_constraint(input: &gateway_dto::CreateBucketInp
|
||||
}
|
||||
|
||||
impl DefaultBucketUsecase {
|
||||
/// Checks that the bucket exists before reading or writing a stored configuration.
|
||||
pub(super) async fn ensure_config_bucket(&self, bucket: &str) -> rustfs_s3_types::S3Result<()> {
|
||||
let Some(store) = self.object_store() else {
|
||||
return Err(rustfs_s3_types::s3_error!(InternalError, "Not init"));
|
||||
};
|
||||
store
|
||||
.get_bucket_info(bucket, &BucketOptions::default())
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn create_bucket_exists_result(&self) -> rustfs_s3_types::S3Result<gateway_dto::CreateBucketOutput> {
|
||||
let region = self
|
||||
.global_region()
|
||||
|
||||
@@ -15,8 +15,6 @@
|
||||
//! Bucket CORS configuration use-case operations.
|
||||
|
||||
use super::*;
|
||||
use crate::app::storage_api::s3::dto::GetBucketCorsInput;
|
||||
use crate::app::storage_api::s3::{S3Error, S3ErrorCode, S3Request, S3Result};
|
||||
use rustfs_gateway_types::cors_tagging::parse_cors;
|
||||
use rustfs_gateway_types::persistence::serialize_cors_dto;
|
||||
|
||||
@@ -64,26 +62,30 @@ impl DefaultBucketUsecase {
|
||||
.await
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self, req))]
|
||||
pub async fn execute_get_bucket_cors(&self, req: S3Request<GetBucketCorsInput>) -> S3Result<PersistedCorsConfiguration> {
|
||||
let GetBucketCorsInput { bucket, .. } = req.input;
|
||||
|
||||
/// Returns the stored bucket CORS configuration after checking the bucket exists.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Returns `InternalError` if the store is uninitialized, `NoSuchCORSConfiguration`
|
||||
/// if the configuration is absent, or the storage error from either lookup.
|
||||
#[instrument(level = "debug", skip(self, bucket))]
|
||||
pub async fn execute_get_bucket_cors(&self, bucket: &str) -> rustfs_s3_types::S3Result<PersistedCorsConfiguration> {
|
||||
let Some(store) = self.object_store() else {
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
|
||||
return Err(rustfs_s3_types::s3_error!(InternalError, "Not init"));
|
||||
};
|
||||
|
||||
store
|
||||
.get_bucket_info(&bucket, &BucketOptions::default())
|
||||
.get_bucket_info(bucket, &BucketOptions::default())
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
let cors_configuration = match metadata_sys::get_cors_config(&bucket).await {
|
||||
let cors_configuration = match metadata_sys::get_cors_config(bucket).await {
|
||||
Ok((config, _)) => config,
|
||||
Err(err) => {
|
||||
if err == StorageError::ConfigNotFound {
|
||||
return Err(S3Error::with_message(
|
||||
S3ErrorCode::NoSuchCORSConfiguration,
|
||||
"The CORS configuration does not exist".to_string(),
|
||||
return Err(rustfs_s3_types::s3_error!(
|
||||
NoSuchCORSConfiguration,
|
||||
"The CORS configuration does not exist"
|
||||
));
|
||||
}
|
||||
warn!(
|
||||
@@ -154,7 +156,7 @@ impl DefaultBucketUsecase {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::app::bucket_usecase::test_support::{build_request, gateway_bucket, owned_envelope};
|
||||
use crate::app::bucket_usecase::test_support::{gateway_bucket, owned_envelope};
|
||||
use http::Method;
|
||||
|
||||
#[tokio::test]
|
||||
@@ -174,16 +176,10 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_get_bucket_cors_returns_internal_error_when_store_uninitialized() {
|
||||
let input = GetBucketCorsInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let req = build_request(input, Method::GET);
|
||||
let usecase = DefaultBucketUsecase::without_context();
|
||||
|
||||
let err = usecase.execute_get_bucket_cors(req).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
let err = usecase.execute_get_bucket_cors("test-bucket").await.unwrap_err();
|
||||
assert_eq!(err.code(), &rustfs_s3_types::S3ErrorCode::InternalError);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -0,0 +1,350 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Bucket configuration mutation tests over the shared use case and durable metadata.
|
||||
|
||||
use super::*;
|
||||
use crate::app::bucket_usecase::test_support::build_request;
|
||||
use crate::app::gating_test_env::{run_large_stack_test, shared_gating_ambient};
|
||||
use crate::app::runtime_sources::ServerContextSlot;
|
||||
use crate::app::storage_api::bucket_usecase::helper::MutationCompletionProbe;
|
||||
use crate::app::storage_api::owned_parity::authorize_bucket_config_mutation;
|
||||
use crate::site_replication::hooks::BUCKET_META_HOOK_CALLS;
|
||||
use http::Method;
|
||||
use rustfs_credentials::Credentials;
|
||||
use rustfs_s3_types::{S3ErrorCode, S3Result};
|
||||
use serial_test::serial;
|
||||
use std::{cell::RefCell, sync::Mutex};
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
enum Config {
|
||||
Encryption,
|
||||
Policy,
|
||||
Tagging,
|
||||
}
|
||||
|
||||
impl Config {
|
||||
async fn seed(self, bucket: &str) {
|
||||
match self {
|
||||
Self::Encryption => {
|
||||
metadata_sys::update(
|
||||
bucket,
|
||||
BUCKET_SSECONFIG,
|
||||
b"<ServerSideEncryptionConfiguration><Rule><ApplyServerSideEncryptionByDefault>\
|
||||
<SSEAlgorithm>AES256</SSEAlgorithm></ApplyServerSideEncryptionByDefault></Rule></ServerSideEncryptionConfiguration>"
|
||||
.to_vec(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
Self::Policy => {
|
||||
metadata_sys::update(bucket, BUCKET_POLICY_CONFIG, br#"{"Version":"2012-10-17","Statement":[]}"#.to_vec()).await
|
||||
}
|
||||
Self::Tagging => {
|
||||
let tags = rustfs_gateway_types::cors_tagging::parse_tagging(
|
||||
b"<Tagging><TagSet><Tag><Key>team</Key><Value>storage</Value></Tag></TagSet></Tagging>",
|
||||
)
|
||||
.expect("valid tagging fixture");
|
||||
metadata_sys::update_s3_tagging_config(bucket, Some(tags), None).await
|
||||
}
|
||||
}
|
||||
.expect("seed bucket configuration");
|
||||
}
|
||||
|
||||
async fn stored(self, bucket: &str) -> Vec<u8> {
|
||||
let metadata = metadata_sys::get_global_bucket_metadata_sys()
|
||||
.expect("initialized metadata system")
|
||||
.read()
|
||||
.await
|
||||
.get_config_from_disk(bucket)
|
||||
.await
|
||||
.expect("read bucket metadata from disk");
|
||||
match self {
|
||||
Self::Encryption => metadata.encryption_config_xml,
|
||||
Self::Policy => metadata.policy_config_json,
|
||||
Self::Tagging => metadata.tagging_config_xml,
|
||||
}
|
||||
}
|
||||
|
||||
async fn put(self, usecase: &DefaultBucketUsecase, envelope: RequestEnvelope, bucket: &str) -> S3Result<()> {
|
||||
match self {
|
||||
Self::Encryption => {
|
||||
let config = rustfs_gateway_types::persistence::parse_bucket_encryption(ENCRYPTION_WRITE).unwrap();
|
||||
usecase.execute_put_bucket_encryption(envelope, bucket, Ok(config)).await
|
||||
}
|
||||
Self::Policy => usecase.execute_put_bucket_policy(envelope, bucket, POLICY_WRITE).await,
|
||||
Self::Tagging => {
|
||||
let tags = rustfs_gateway_types::cors_tagging::parse_tagging(TAGGING_WRITE).unwrap();
|
||||
usecase.execute_put_bucket_tagging(envelope, bucket, tags).await
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn execute(self, usecase: &DefaultBucketUsecase, envelope: RequestEnvelope, bucket: &str) -> S3Result<()> {
|
||||
match self {
|
||||
Self::Encryption => usecase.execute_delete_bucket_encryption(envelope, bucket).await,
|
||||
Self::Policy => usecase.execute_delete_bucket_policy(envelope, bucket).await,
|
||||
Self::Tagging => usecase.execute_delete_bucket_tagging(envelope, bucket).await,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const ENCRYPTION_WRITE: &[u8] = b"<ServerSideEncryptionConfiguration><Rule><ApplyServerSideEncryptionByDefault><SSEAlgorithm>aws:kms</SSEAlgorithm><KMSMasterKeyID>fixture-key</KMSMasterKeyID></ApplyServerSideEncryptionByDefault></Rule></ServerSideEncryptionConfiguration>";
|
||||
const POLICY_WRITE: &str = "{ \"Version\": \"2012-10-17\", \"Statement\": [] } ";
|
||||
const TAGGING_WRITE: &[u8] = b"<Tagging><TagSet><Tag><Key>team</Key><Value>gateway</Value></Tag></TagSet></Tagging>";
|
||||
|
||||
async fn check_mutation(config: Config, stale: bool, write: bool) {
|
||||
let (subresource, action, operation, reload, replicated, audit_status) = match (config, write) {
|
||||
(Config::Encryption, false) => (
|
||||
"encryption",
|
||||
S3Action::PutBucketEncryptionAction,
|
||||
S3Operation::DeleteBucketEncryption,
|
||||
"delete bucket encryption",
|
||||
"sse-config",
|
||||
204,
|
||||
),
|
||||
(Config::Policy, false) => (
|
||||
"policy",
|
||||
S3Action::DeleteBucketPolicyAction,
|
||||
S3Operation::DeleteBucketPolicy,
|
||||
"delete bucket policy",
|
||||
"policy",
|
||||
200,
|
||||
),
|
||||
(Config::Tagging, false) => (
|
||||
"tagging",
|
||||
S3Action::PutBucketTaggingAction,
|
||||
S3Operation::DeleteBucketTagging,
|
||||
"delete bucket tagging",
|
||||
"tags",
|
||||
200,
|
||||
),
|
||||
(Config::Encryption, true) => (
|
||||
"encryption",
|
||||
S3Action::PutBucketEncryptionAction,
|
||||
S3Operation::PutBucketEncryption,
|
||||
"put bucket encryption",
|
||||
"sse-config",
|
||||
200,
|
||||
),
|
||||
(Config::Policy, true) => (
|
||||
"policy",
|
||||
S3Action::PutBucketPolicyAction,
|
||||
S3Operation::PutBucketPolicy,
|
||||
"put bucket policy",
|
||||
"policy",
|
||||
200,
|
||||
),
|
||||
(Config::Tagging, true) => (
|
||||
"tagging",
|
||||
S3Action::PutBucketTaggingAction,
|
||||
S3Operation::PutBucketTagging,
|
||||
"put bucket tagging",
|
||||
"tags",
|
||||
200,
|
||||
),
|
||||
};
|
||||
let context = shared_gating_ambient().await;
|
||||
let store = context.object_store();
|
||||
let bucket = format!("config-mutation-{}", uuid::Uuid::new_v4().simple());
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
let server = ServerContextSlot::new();
|
||||
assert!(server.install(Arc::clone(&context)));
|
||||
let mut request = build_request((), if write { Method::PUT } else { Method::DELETE });
|
||||
request.uri = format!("/{bucket}?{subresource}").parse().expect("request URI");
|
||||
request.extensions.insert(Arc::clone(&server));
|
||||
request.extensions.insert(ReqInfo {
|
||||
cred: Some(Credentials {
|
||||
access_key: "config-delete-owner".to_owned(),
|
||||
..Default::default()
|
||||
}),
|
||||
is_owner: true,
|
||||
..Default::default()
|
||||
});
|
||||
request.extensions.insert(request_context::RequestContext {
|
||||
request_id: "bucket-config-delete".to_owned(),
|
||||
x_amz_request_id: "bucket-config-delete".to_owned(),
|
||||
trace_id: None,
|
||||
span_id: None,
|
||||
start_time: std::time::Instant::now(),
|
||||
});
|
||||
let mut envelope = RequestEnvelope::from_s3s(&request);
|
||||
authorize_bucket_config_mutation(&server, &mut envelope, &bucket, Action::S3Action(action))
|
||||
.await
|
||||
.expect("capture authorized bucket incarnation");
|
||||
if stale {
|
||||
store
|
||||
.delete_bucket(&bucket, &DeleteBucketOptions::default())
|
||||
.await
|
||||
.expect("delete authorized bucket");
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("recreate bucket");
|
||||
}
|
||||
config.seed(&bucket).await;
|
||||
let seeded = config.stored(&bucket).await;
|
||||
assert!(!seeded.is_empty(), "durable configuration fixture");
|
||||
|
||||
let before = rustfs_scanner::scanner_dirty_usage_bucket_generation(&bucket);
|
||||
let maintenance_before = rustfs_scanner::scanner_maintenance_generation();
|
||||
let audits = MutationCompletionProbe::install(&bucket);
|
||||
let reloads = Arc::new(Mutex::new(Vec::new()));
|
||||
let observe = Arc::clone(&reloads);
|
||||
let usecase = DefaultBucketUsecase::with_context(Some(context));
|
||||
let (result, hooks) = BUCKET_META_HOOK_CALLS
|
||||
.scope(RefCell::default(), async {
|
||||
let result = BUCKET_METADATA_RELOAD_PROBE
|
||||
.scope(
|
||||
(
|
||||
bucket.clone(),
|
||||
Box::new(move |completed, label, maintenance| {
|
||||
let replication_calls = BUCKET_META_HOOK_CALLS.with(|calls| calls.borrow().len());
|
||||
observe
|
||||
.lock()
|
||||
.expect("reload entries")
|
||||
.push((completed, label, maintenance, replication_calls));
|
||||
}),
|
||||
),
|
||||
async {
|
||||
if write {
|
||||
config.put(&usecase, envelope, &bucket).await
|
||||
} else {
|
||||
config.execute(&usecase, envelope, &bucket).await
|
||||
}
|
||||
},
|
||||
)
|
||||
.await;
|
||||
(result, BUCKET_META_HOOK_CALLS.with(RefCell::take))
|
||||
})
|
||||
.await;
|
||||
|
||||
let stored = config.stored(&bucket).await;
|
||||
let after = rustfs_scanner::scanner_dirty_usage_bucket_generation(&bucket);
|
||||
let reloads = reloads.lock().expect("reload entries");
|
||||
let entries = audits.audits.lock().expect("audit entries");
|
||||
assert!(audits.events.lock().expect("event entries").is_empty());
|
||||
let audit = entries
|
||||
.iter()
|
||||
.map(|entry| (entry.api.name.as_deref(), entry.api.status_code))
|
||||
.collect::<Vec<_>>();
|
||||
if stale {
|
||||
assert_eq!(
|
||||
(
|
||||
result.err().map(|err| err.code().clone()),
|
||||
stored,
|
||||
reloads.len(),
|
||||
hooks.len(),
|
||||
after,
|
||||
audit
|
||||
),
|
||||
(
|
||||
Some(S3ErrorCode::NoSuchBucket),
|
||||
seeded,
|
||||
0,
|
||||
0,
|
||||
before,
|
||||
vec![(Some(operation.as_str()), Some(404))]
|
||||
),
|
||||
"{operation:?} must refuse the stale authorization and preserve replacement bytes"
|
||||
);
|
||||
} else {
|
||||
result.expect("authorized mutation");
|
||||
if write {
|
||||
match config {
|
||||
Config::Encryption => {
|
||||
let actual = rustfs_gateway_types::persistence::parse_bucket_encryption(&stored).unwrap();
|
||||
let expected = rustfs_gateway_types::persistence::parse_bucket_encryption(ENCRYPTION_WRITE).unwrap();
|
||||
assert_eq!(actual, expected);
|
||||
}
|
||||
Config::Policy => assert_eq!(stored, POLICY_WRITE.as_bytes()),
|
||||
Config::Tagging => {
|
||||
let actual = rustfs_gateway_types::cors_tagging::parse_tagging(&stored).unwrap();
|
||||
let expected = rustfs_gateway_types::cors_tagging::parse_tagging(TAGGING_WRITE).unwrap();
|
||||
assert_eq!(actual, expected);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
assert!(stored.is_empty(), "{operation:?} must remove the durable configuration");
|
||||
}
|
||||
assert_eq!(*reloads, [(false, reload, false, 0), (true, reload, false, 0)]);
|
||||
let [hook] = hooks.as_slice() else { panic!("{operation:?}: expected one replication call") };
|
||||
assert_eq!((&hook.bucket, hook.item_type.as_str()), (&bucket, replicated));
|
||||
assert_eq!(hook.scanner_maintenance_generation, maintenance_before);
|
||||
assert_eq!(rustfs_scanner::scanner_maintenance_generation(), maintenance_before);
|
||||
if matches!(config, Config::Tagging) {
|
||||
assert!(after > hook.scanner_generation);
|
||||
} else {
|
||||
assert_eq!(after, hook.scanner_generation);
|
||||
}
|
||||
assert_eq!(audit, [(Some(operation.as_str()), Some(audit_status))]);
|
||||
assert_eq!(entries[0].request_id.as_deref(), Some("bucket-config-delete"));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn encryption_delete_fences_recreated_bucket_and_orders_callbacks() {
|
||||
run_large_stack_test("delete-encryption", || async {
|
||||
check_mutation(Config::Encryption, false, false).await;
|
||||
check_mutation(Config::Encryption, true, false).await;
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn policy_delete_fences_recreated_bucket_and_orders_callbacks() {
|
||||
run_large_stack_test("delete-policy", || async {
|
||||
check_mutation(Config::Policy, false, false).await;
|
||||
check_mutation(Config::Policy, true, false).await;
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn tagging_delete_fences_recreated_bucket_and_orders_callbacks() {
|
||||
run_large_stack_test("delete-tagging", || async {
|
||||
check_mutation(Config::Tagging, false, false).await;
|
||||
check_mutation(Config::Tagging, true, false).await;
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn encryption_put_fences_recreated_bucket_and_orders_callbacks() {
|
||||
run_large_stack_test("put-encryption", || async {
|
||||
check_mutation(Config::Encryption, false, true).await;
|
||||
check_mutation(Config::Encryption, true, true).await;
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn policy_put_fences_recreated_bucket_and_orders_callbacks() {
|
||||
run_large_stack_test("put-policy", || async {
|
||||
check_mutation(Config::Policy, false, true).await;
|
||||
check_mutation(Config::Policy, true, true).await;
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn tagging_put_fences_recreated_bucket_and_orders_callbacks() {
|
||||
run_large_stack_test("put-tagging", || async {
|
||||
check_mutation(Config::Tagging, false, true).await;
|
||||
check_mutation(Config::Tagging, true, true).await;
|
||||
});
|
||||
}
|
||||
@@ -15,17 +15,11 @@
|
||||
//! Bucket default-encryption use-case operations.
|
||||
|
||||
use super::*;
|
||||
use crate::app::storage_api::s3::dto::{
|
||||
DeleteBucketEncryptionInput, DeleteBucketEncryptionOutput, GetBucketEncryptionInput, PutBucketEncryptionInput,
|
||||
PutBucketEncryptionOutput, ServerSideEncryption,
|
||||
};
|
||||
use crate::app::storage_api::s3::{S3Error, S3ErrorCode, S3Request, S3Response, S3Result, s3_error};
|
||||
use rustfs_s3_types::{S3Error, S3ErrorCode, S3Result, s3_error};
|
||||
|
||||
/// Refuse a default-encryption configuration the write path could not honour
|
||||
/// as written. `bucket_default_write_sse` falls back to AES256 for any
|
||||
/// algorithm it does not know, so storing one would make GetBucketEncryption
|
||||
/// advertise a scheme no object is encrypted under. s3s parses `SSEAlgorithm`
|
||||
/// as an open string, so the schema check has to happen here.
|
||||
/// as written. `bucket_default_write_sse` falls back to AES256 for an unknown algorithm,
|
||||
/// which would make GetBucketEncryption advertise a scheme the write path cannot use.
|
||||
fn validate_bucket_encryption_configuration(config: &PersistedBucketEncryptionConfiguration) -> S3Result<()> {
|
||||
if config.rules.is_empty() {
|
||||
return Err(S3Error::with_message(
|
||||
@@ -42,14 +36,14 @@ fn validate_bucket_encryption_configuration(config: &PersistedBucketEncryptionCo
|
||||
};
|
||||
let names_kms_key = by_default.kms_master_key_id.as_deref().is_some_and(|id| !id.is_empty());
|
||||
match by_default.sse_algorithm.as_str() {
|
||||
ServerSideEncryption::AWS_KMS => {}
|
||||
ServerSideEncryption::AES256 if names_kms_key => {
|
||||
"aws:kms" => {}
|
||||
"AES256" if names_kms_key => {
|
||||
return Err(S3Error::with_message(
|
||||
S3ErrorCode::InvalidArgument,
|
||||
"KMSMasterKeyID can only be specified when SSEAlgorithm is aws:kms".to_string(),
|
||||
));
|
||||
}
|
||||
ServerSideEncryption::AES256 => {}
|
||||
"AES256" => {}
|
||||
other => {
|
||||
return Err(S3Error::with_message(
|
||||
S3ErrorCode::MalformedXML,
|
||||
@@ -62,57 +56,72 @@ fn validate_bucket_encryption_configuration(config: &PersistedBucketEncryptionCo
|
||||
}
|
||||
|
||||
impl DefaultBucketUsecase {
|
||||
pub async fn execute_delete_bucket_encryption(
|
||||
pub(crate) async fn execute_delete_bucket_encryption(
|
||||
&self,
|
||||
req: S3Request<DeleteBucketEncryptionInput>,
|
||||
) -> S3Result<S3Response<DeleteBucketEncryptionOutput>> {
|
||||
let expected_incarnation_id = bucket_config_mutation_incarnation(&req, &req.input.bucket)?;
|
||||
let request_context = req.extensions.get::<request_context::RequestContext>().cloned();
|
||||
let DeleteBucketEncryptionInput { bucket, .. } = req.input;
|
||||
envelope: RequestEnvelope,
|
||||
bucket: &str,
|
||||
) -> rustfs_s3_types::S3Result<()> {
|
||||
RequestEnvelope::audit_only_operation(
|
||||
envelope,
|
||||
S3Operation::DeleteBucketEncryption,
|
||||
StatusCode::NO_CONTENT,
|
||||
|envelope| async move {
|
||||
let expected_incarnation_id = bucket_config_mutation_incarnation(&envelope, bucket)?;
|
||||
let request_context = envelope.request_context().cloned();
|
||||
|
||||
let Some(store) = self.object_store() else {
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
|
||||
};
|
||||
let Some(store) = self.object_store() else {
|
||||
return Err(rustfs_s3_types::s3_error!(InternalError, "Not init"));
|
||||
};
|
||||
|
||||
store
|
||||
.get_bucket_info(&bucket, &BucketOptions::default())
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
store
|
||||
.get_bucket_info(bucket, &BucketOptions::default())
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
delete_bucket_config_for_incarnation(&bucket, BUCKET_SSECONFIG, expected_incarnation_id)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
delete_bucket_config_for_incarnation(bucket, BUCKET_SSECONFIG, expected_incarnation_id)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
notify_bucket_metadata_reload(bucket.clone(), "delete bucket encryption", request_context, false).await;
|
||||
notify_bucket_metadata_reload(bucket.to_owned(), "delete bucket encryption", request_context, false).await;
|
||||
|
||||
let item = sr_bucket_meta_item(bucket.clone(), "sse-config");
|
||||
if let Err(err) = site_replication_bucket_meta_hook(item).await {
|
||||
warn!(bucket = %bucket, error = ?err, "site replication bucket encryption delete hook failed");
|
||||
}
|
||||
let item = sr_bucket_meta_item(bucket.to_owned(), "sse-config");
|
||||
if let Err(err) = site_replication_bucket_meta_hook(item).await {
|
||||
warn!(bucket = %bucket, error = ?err, "site replication bucket encryption delete hook failed");
|
||||
}
|
||||
|
||||
Ok(S3Response::with_status(DeleteBucketEncryptionOutput::default(), StatusCode::NO_CONTENT))
|
||||
Ok(())
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Returns the stored bucket encryption configuration after checking the bucket exists.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Returns `InternalError` if the store is uninitialized, `ServerSideEncryptionConfigurationNotFoundError`
|
||||
/// if the configuration is absent, or the storage error from either lookup.
|
||||
pub async fn execute_get_bucket_encryption(
|
||||
&self,
|
||||
req: S3Request<GetBucketEncryptionInput>,
|
||||
) -> S3Result<Option<PersistedBucketEncryptionConfiguration>> {
|
||||
let GetBucketEncryptionInput { bucket, .. } = req.input;
|
||||
|
||||
bucket: &str,
|
||||
) -> rustfs_s3_types::S3Result<PersistedBucketEncryptionConfiguration> {
|
||||
let Some(store) = self.object_store() else {
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
|
||||
return Err(rustfs_s3_types::s3_error!(InternalError, "Not init"));
|
||||
};
|
||||
|
||||
store
|
||||
.get_bucket_info(&bucket, &BucketOptions::default())
|
||||
.get_bucket_info(bucket, &BucketOptions::default())
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
let server_side_encryption_configuration = match metadata_sys::get_sse_config(&bucket).await {
|
||||
Ok((cfg, _)) => Some(cfg),
|
||||
let server_side_encryption_configuration = match metadata_sys::get_sse_config(bucket).await {
|
||||
Ok((cfg, _)) => cfg,
|
||||
Err(err) => {
|
||||
if err == StorageError::ConfigNotFound {
|
||||
return Err(s3_error!(ServerSideEncryptionConfigurationNotFoundError));
|
||||
return Err(rustfs_s3_types::s3_error!(
|
||||
ServerSideEncryptionConfigurationNotFoundError,
|
||||
"The server-side encryption configuration was not found."
|
||||
));
|
||||
}
|
||||
warn!(
|
||||
component = LOG_COMPONENT_APP,
|
||||
@@ -129,64 +138,65 @@ impl DefaultBucketUsecase {
|
||||
Ok(server_side_encryption_configuration)
|
||||
}
|
||||
|
||||
pub async fn execute_put_bucket_encryption(
|
||||
pub(crate) async fn execute_put_bucket_encryption(
|
||||
&self,
|
||||
req: S3Request<PutBucketEncryptionInput>,
|
||||
prepared: S3Result<PersistedBucketEncryptionConfiguration>,
|
||||
) -> S3Result<S3Response<PutBucketEncryptionOutput>> {
|
||||
let expected_incarnation_id = bucket_config_mutation_incarnation(&req, &req.input.bucket)?;
|
||||
let request_context = req.extensions.get::<request_context::RequestContext>().cloned();
|
||||
let PutBucketEncryptionInput { bucket, .. } = req.input;
|
||||
envelope: RequestEnvelope,
|
||||
bucket: &str,
|
||||
prepared: rustfs_s3_types::S3Result<PersistedBucketEncryptionConfiguration>,
|
||||
) -> rustfs_s3_types::S3Result<()> {
|
||||
RequestEnvelope::audit_only_operation(envelope, S3Operation::PutBucketEncryption, StatusCode::OK, |envelope| async move {
|
||||
let expected_incarnation_id = bucket_config_mutation_incarnation(&envelope, bucket)?;
|
||||
let request_context = envelope.request_context().cloned();
|
||||
|
||||
let mut server_side_encryption_configuration = prepared?;
|
||||
validate_bucket_encryption_configuration(&server_side_encryption_configuration)?;
|
||||
let mut server_side_encryption_configuration = prepared?;
|
||||
validate_bucket_encryption_configuration(&server_side_encryption_configuration)?;
|
||||
|
||||
// When SSE-KMS is set without a specific key ID, populate the default
|
||||
// KMS key so that GetBucketEncryption responses include it. Clients like
|
||||
// mc rely on the presence of KMSMasterKeyID to distinguish SSE-KMS from
|
||||
// SSE-S3 in their display logic.
|
||||
if let Some(rule) = server_side_encryption_configuration.rules.first_mut()
|
||||
&& let Some(ref mut by_default) = rule.apply_server_side_encryption_by_default
|
||||
&& by_default.sse_algorithm.as_str() == ServerSideEncryption::AWS_KMS
|
||||
&& by_default.kms_master_key_id.as_deref().is_none_or(str::is_empty)
|
||||
{
|
||||
let service = current_encryption_service()
|
||||
// When SSE-KMS is set without a specific key ID, populate the default
|
||||
// KMS key so that GetBucketEncryption responses include it. Clients like
|
||||
// mc rely on the presence of KMSMasterKeyID to distinguish SSE-KMS from
|
||||
// SSE-S3 in their display logic.
|
||||
if let Some(rule) = server_side_encryption_configuration.rules.first_mut()
|
||||
&& let Some(ref mut by_default) = rule.apply_server_side_encryption_by_default
|
||||
&& by_default.sse_algorithm.as_str() == "aws:kms"
|
||||
&& by_default.kms_master_key_id.as_deref().is_none_or(str::is_empty)
|
||||
{
|
||||
let service = current_encryption_service()
|
||||
.await
|
||||
.ok_or_else(|| s3_error!(InternalError, "KMS service not initialized"))?;
|
||||
let default_key = service
|
||||
.get_default_key_id()
|
||||
.filter(|key| !key.is_empty())
|
||||
.ok_or_else(|| s3_error!(InternalError, "KMS default key not configured"))?;
|
||||
by_default.kms_master_key_id = Some(default_key.clone());
|
||||
}
|
||||
|
||||
let Some(store) = self.object_store() else {
|
||||
return Err(s3_error!(InternalError, "Not init"));
|
||||
};
|
||||
|
||||
store
|
||||
.get_bucket_info(bucket, &BucketOptions::default())
|
||||
.await
|
||||
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "KMS service not initialized".to_string()))?;
|
||||
let default_key = service
|
||||
.get_default_key_id()
|
||||
.filter(|key| !key.is_empty())
|
||||
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "KMS default key not configured".to_string()))?;
|
||||
by_default.kms_master_key_id = Some(default_key.clone());
|
||||
}
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
info!("sse_config {:?}", &server_side_encryption_configuration);
|
||||
let data = serialize_config(&server_side_encryption_configuration)?;
|
||||
update_bucket_config_for_incarnation(bucket, BUCKET_SSECONFIG, data, expected_incarnation_id)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
let Some(store) = self.object_store() else {
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
|
||||
};
|
||||
notify_bucket_metadata_reload(bucket.to_owned(), "put bucket encryption", request_context, false).await;
|
||||
|
||||
store
|
||||
.get_bucket_info(&bucket, &BucketOptions::default())
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
let data = serialize_config(&server_side_encryption_configuration)?;
|
||||
update_bucket_config_for_incarnation(&bucket, BUCKET_SSECONFIG, data, expected_incarnation_id)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
notify_bucket_metadata_reload(bucket.clone(), "put bucket encryption", request_context, false).await;
|
||||
|
||||
let mut item = sr_bucket_meta_item(bucket.clone(), "sse-config");
|
||||
item.sse_config = Some(
|
||||
serialize_config(&server_side_encryption_configuration)
|
||||
.and_then(|bytes| String::from_utf8(bytes).map_err(to_internal_error))?,
|
||||
);
|
||||
if let Err(err) = site_replication_bucket_meta_hook(item).await {
|
||||
warn!(bucket = %bucket, error = ?err, "site replication bucket encryption hook failed");
|
||||
}
|
||||
Ok(S3Response::new(PutBucketEncryptionOutput::default()))
|
||||
let mut item = sr_bucket_meta_item(bucket.to_owned(), "sse-config");
|
||||
item.sse_config = Some(
|
||||
serialize_config(&server_side_encryption_configuration)
|
||||
.and_then(|bytes| String::from_utf8(bytes).map_err(to_internal_error))?,
|
||||
);
|
||||
if let Err(err) = site_replication_bucket_meta_hook(item).await {
|
||||
warn!(bucket = %bucket, error = ?err, "site replication bucket encryption hook failed");
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
@@ -194,7 +204,7 @@ impl DefaultBucketUsecase {
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::app::bucket_usecase::test_support::build_request;
|
||||
use crate::app::storage_api::s3::dto;
|
||||
use crate::app::storage_api::s3::dto::{self, PutBucketEncryptionInput, ServerSideEncryption};
|
||||
use http::Method;
|
||||
use rustfs_gateway_types::persistence::{
|
||||
PersistedBucketEncryptionConfiguration as ServerSideEncryptionConfiguration,
|
||||
@@ -257,16 +267,10 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_get_bucket_encryption_returns_internal_error_when_store_uninitialized() {
|
||||
let input = GetBucketEncryptionInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let req = build_request(input, Method::GET);
|
||||
let usecase = DefaultBucketUsecase::without_context();
|
||||
|
||||
let err = usecase.execute_get_bucket_encryption(req).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
let err = usecase.execute_get_bucket_encryption("test-bucket").await.unwrap_err();
|
||||
assert_eq!(err.code(), &rustfs_s3_types::S3ErrorCode::InternalError);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -292,7 +296,20 @@ mod tests {
|
||||
let prepared = crate::storage::persisted_from_s3s(&req.input.server_side_encryption_configuration);
|
||||
let usecase = DefaultBucketUsecase::without_context();
|
||||
|
||||
let err = usecase.execute_put_bucket_encryption(req, prepared).await.unwrap_err();
|
||||
let err = usecase
|
||||
.execute_put_bucket_encryption(RequestEnvelope::from_s3s(&req), &req.input.bucket, prepared.map_err(Into::into))
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
|
||||
let err = usecase
|
||||
.execute_put_bucket_encryption(
|
||||
RequestEnvelope::from_s3s(&req),
|
||||
&req.input.bucket,
|
||||
Ok(PersistedBucketEncryptionConfiguration { rules: vec![] }),
|
||||
)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::MalformedXML);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,8 +15,7 @@
|
||||
//! Bucket lifecycle configuration use-case operations.
|
||||
|
||||
use super::*;
|
||||
use crate::app::storage_api::s3::dto::{DeleteBucketLifecycleInput, DeleteBucketLifecycleOutput};
|
||||
use crate::app::storage_api::s3::{S3Error, S3ErrorCode, S3Request, S3Response, S3Result, s3_error};
|
||||
use crate::app::storage_api::s3::{S3Error, S3ErrorCode, s3_error};
|
||||
use rustfs_gateway_types::persistence::{PersistedLifecycleRule, parse_lifecycle_dto};
|
||||
|
||||
const ERR_LIFECYCLE_RULE_STATUS: &str = "Rule status must be either Enabled or Disabled";
|
||||
@@ -128,36 +127,44 @@ fn lifecycle_has_abort_multipart_rules(config: &PersistedLifecycleConfiguration)
|
||||
}
|
||||
|
||||
impl DefaultBucketUsecase {
|
||||
#[instrument(level = "debug", skip(self, req))]
|
||||
pub async fn execute_delete_bucket_lifecycle(
|
||||
#[instrument(level = "debug", skip(self, envelope, bucket))]
|
||||
pub(crate) async fn execute_delete_bucket_lifecycle(
|
||||
&self,
|
||||
req: S3Request<DeleteBucketLifecycleInput>,
|
||||
) -> S3Result<S3Response<DeleteBucketLifecycleOutput>> {
|
||||
let expected_incarnation_id = bucket_config_mutation_incarnation(&req, &req.input.bucket)?;
|
||||
let request_context = req.extensions.get::<request_context::RequestContext>().cloned();
|
||||
let DeleteBucketLifecycleInput { bucket, .. } = req.input;
|
||||
envelope: RequestEnvelope,
|
||||
bucket: &str,
|
||||
) -> rustfs_s3_types::S3Result<gateway_dto::DeleteBucketLifecycleOutput> {
|
||||
RequestEnvelope::audit_only_operation(
|
||||
envelope,
|
||||
S3Operation::DeleteBucketLifecycle,
|
||||
StatusCode::OK,
|
||||
|envelope| async move {
|
||||
let expected_incarnation_id = bucket_config_mutation_incarnation(&envelope, bucket)?;
|
||||
let request_context = envelope.request_context().cloned();
|
||||
|
||||
let Some(store) = self.object_store() else {
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
|
||||
};
|
||||
let Some(store) = self.object_store() else {
|
||||
return Err(rustfs_s3_types::s3_error!(InternalError, "Not init"));
|
||||
};
|
||||
|
||||
store
|
||||
.get_bucket_info(&bucket, &BucketOptions::default())
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
store
|
||||
.get_bucket_info(bucket, &BucketOptions::default())
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
delete_bucket_config_for_incarnation(&bucket, BUCKET_LIFECYCLE_CONFIG, expected_incarnation_id)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
delete_bucket_config_for_incarnation(bucket, BUCKET_LIFECYCLE_CONFIG, expected_incarnation_id)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
notify_bucket_metadata_reload(bucket.clone(), "delete bucket lifecycle", request_context, true).await;
|
||||
notify_bucket_metadata_reload(bucket.to_owned(), "delete bucket lifecycle", request_context, true).await;
|
||||
|
||||
let item = sr_bucket_meta_item(bucket.clone(), "lc-config");
|
||||
if let Err(err) = site_replication_bucket_meta_hook(item).await {
|
||||
warn!(bucket = %bucket, error = ?err, "site replication bucket lifecycle delete hook failed");
|
||||
}
|
||||
let item = sr_bucket_meta_item(bucket.to_owned(), "lc-config");
|
||||
if let Err(err) = site_replication_bucket_meta_hook(item).await {
|
||||
warn!(bucket = %bucket, error = ?err, "site replication bucket lifecycle delete hook failed");
|
||||
}
|
||||
|
||||
Ok(S3Response::new(DeleteBucketLifecycleOutput::default()))
|
||||
Ok(gateway_dto::DeleteBucketLifecycleOutput::default())
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// The stored lifecycle configuration as the gateway's answer holds it.
|
||||
@@ -353,12 +360,201 @@ impl DefaultBucketUsecase {
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::app::bucket_usecase::test_support::build_request;
|
||||
use crate::app::gating_test_env::{run_large_stack_test, shared_gating_ambient};
|
||||
use crate::app::runtime_sources::ServerContextSlot;
|
||||
use crate::app::storage_api::bucket_usecase::helper::MutationCompletionProbe;
|
||||
use crate::app::storage_api::owned_parity::{ReqInfo, authorize_bucket_config_mutation};
|
||||
use crate::app::storage_api::s3::dto;
|
||||
use crate::app::storage_api::s3::dto::{
|
||||
BucketLifecycleConfiguration, ExpirationStatus, LifecycleExpiration, LifecycleRule, NoncurrentVersionTransition,
|
||||
PutBucketLifecycleConfigurationInput, Transition, TransitionStorageClass,
|
||||
};
|
||||
use crate::site_replication::hooks::{BUCKET_META_HOOK_CALLS, BucketMetaHookCall};
|
||||
use http::Method;
|
||||
use rustfs_credentials::Credentials;
|
||||
use rustfs_policy::policy::action::{Action, S3Action};
|
||||
use serial_test::serial;
|
||||
use std::cell::RefCell;
|
||||
use std::sync::Arc;
|
||||
|
||||
const DELETE_LIFECYCLE_XML: &[u8] = b"<LifecycleConfiguration><Rule><ID>kept</ID><Status>Disabled</Status>\
|
||||
<Filter/><Expiration><Days>1</Days></Expiration></Rule></LifecycleConfiguration>";
|
||||
const DELETE_REQUEST_ID: &str = "lifecycle-delete-context";
|
||||
|
||||
fn delete_envelope(context: &Arc<AppContext>, bucket: &str) -> (Arc<ServerContextSlot>, RequestEnvelope) {
|
||||
let server = ServerContextSlot::new();
|
||||
assert!(server.install(Arc::clone(context)));
|
||||
let mut request = build_request((), Method::DELETE);
|
||||
request.uri = format!("/{bucket}?lifecycle").parse().expect("bucket request URI");
|
||||
request.extensions.insert(Arc::clone(&server));
|
||||
request.extensions.insert(ReqInfo {
|
||||
cred: Some(Credentials {
|
||||
access_key: "lifecycle-delete-owner".to_owned(),
|
||||
..Default::default()
|
||||
}),
|
||||
is_owner: true,
|
||||
..Default::default()
|
||||
});
|
||||
request.extensions.insert(request_context::RequestContext {
|
||||
request_id: DELETE_REQUEST_ID.to_owned(),
|
||||
x_amz_request_id: DELETE_REQUEST_ID.to_owned(),
|
||||
trace_id: None,
|
||||
span_id: None,
|
||||
start_time: std::time::Instant::now(),
|
||||
});
|
||||
(server, RequestEnvelope::from_s3s(&request))
|
||||
}
|
||||
|
||||
async fn authorized_lifecycle_envelope(context: &Arc<AppContext>, bucket: &str) -> RequestEnvelope {
|
||||
let (server, mut envelope) = delete_envelope(context, bucket);
|
||||
authorize_bucket_config_mutation(&server, &mut envelope, bucket, Action::S3Action(S3Action::PutBucketLifecycleAction))
|
||||
.await
|
||||
.expect("bucket owner authorization installs the incarnation snapshot");
|
||||
envelope
|
||||
}
|
||||
|
||||
async fn seed_lifecycle(bucket: &str) {
|
||||
metadata_sys::update(bucket, BUCKET_LIFECYCLE_CONFIG, DELETE_LIFECYCLE_XML.to_vec())
|
||||
.await
|
||||
.expect("store lifecycle configuration");
|
||||
}
|
||||
|
||||
async fn persisted_lifecycle(bucket: &str) -> Vec<u8> {
|
||||
metadata_sys::get_global_bucket_metadata_sys()
|
||||
.expect("initialized metadata system")
|
||||
.read()
|
||||
.await
|
||||
.get_config_from_disk(bucket)
|
||||
.await
|
||||
.expect("persisted bucket metadata")
|
||||
.lifecycle_config_xml
|
||||
}
|
||||
|
||||
async fn observed_delete(
|
||||
usecase: &DefaultBucketUsecase,
|
||||
envelope: RequestEnvelope,
|
||||
bucket: &str,
|
||||
) -> (
|
||||
rustfs_s3_types::S3Result<gateway_dto::DeleteBucketLifecycleOutput>,
|
||||
Vec<BucketMetaHookCall>,
|
||||
) {
|
||||
BUCKET_META_HOOK_CALLS
|
||||
.scope(RefCell::default(), async move {
|
||||
let result = usecase.execute_delete_bucket_lifecycle(envelope, bucket).await;
|
||||
(result, BUCKET_META_HOOK_CALLS.with(RefCell::take))
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn delete_lifecycle_rejects_a_stale_bucket_incarnation() {
|
||||
run_large_stack_test("delete-lifecycle-stale", || async {
|
||||
let ambient = shared_gating_ambient().await;
|
||||
let store = ambient.object_store();
|
||||
let bucket = format!("lifecycle-delete-stale-{}", uuid::Uuid::new_v4().simple());
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create original bucket");
|
||||
let envelope = authorized_lifecycle_envelope(&ambient, &bucket).await;
|
||||
|
||||
store
|
||||
.delete_bucket(&bucket, &DeleteBucketOptions::default())
|
||||
.await
|
||||
.expect("delete original bucket");
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("recreate bucket");
|
||||
seed_lifecycle(&bucket).await;
|
||||
|
||||
let before = rustfs_scanner::scanner_dirty_usage_bucket_generation(&bucket);
|
||||
let usecase = DefaultBucketUsecase::with_context(Some(ambient));
|
||||
let (result, calls) = observed_delete(&usecase, envelope, &bucket).await;
|
||||
let error = result.expect_err("a stale bucket incarnation cannot delete replacement configuration");
|
||||
assert_eq!(error.code(), &rustfs_s3_types::S3ErrorCode::NoSuchBucket);
|
||||
assert_eq!(persisted_lifecycle(&bucket).await, DELETE_LIFECYCLE_XML);
|
||||
assert!(calls.is_empty());
|
||||
assert_eq!(rustfs_scanner::scanner_dirty_usage_bucket_generation(&bucket), before);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn delete_lifecycle_rejects_misbound_or_missing_bucket_snapshots() {
|
||||
run_large_stack_test("delete-lifecycle-unbound", || async {
|
||||
let ambient = shared_gating_ambient().await;
|
||||
let store = ambient.object_store();
|
||||
let authorized = format!("lifecycle-delete-authorized-{}", uuid::Uuid::new_v4().simple());
|
||||
let target = format!("lifecycle-delete-target-{}", uuid::Uuid::new_v4().simple());
|
||||
for bucket in [&authorized, &target] {
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
}
|
||||
seed_lifecycle(&target).await;
|
||||
let misbound = authorized_lifecycle_envelope(&ambient, &authorized).await;
|
||||
let (_, missing) = delete_envelope(&ambient, &target);
|
||||
let usecase = DefaultBucketUsecase::with_context(Some(ambient));
|
||||
|
||||
for (envelope, message) in [
|
||||
(misbound, "bucket config mutation snapshot does not match request bucket"),
|
||||
(missing, "bucket config mutation snapshot is missing"),
|
||||
] {
|
||||
let before = rustfs_scanner::scanner_dirty_usage_bucket_generation(&target);
|
||||
let (result, calls) = observed_delete(&usecase, envelope, &target).await;
|
||||
let error = result.expect_err("an unbound snapshot cannot delete lifecycle configuration");
|
||||
assert_eq!(error.code(), &rustfs_s3_types::S3ErrorCode::InternalError);
|
||||
assert_eq!(error.message(), Some(message));
|
||||
assert_eq!(persisted_lifecycle(&target).await, DELETE_LIFECYCLE_XML);
|
||||
assert!(calls.is_empty());
|
||||
assert_eq!(rustfs_scanner::scanner_dirty_usage_bucket_generation(&target), before);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn delete_lifecycle_reloads_scanner_and_audits_success_once() {
|
||||
run_large_stack_test("delete-lifecycle-completion", || async {
|
||||
let ambient = shared_gating_ambient().await;
|
||||
let store = ambient.object_store();
|
||||
let bucket = format!("lifecycle-delete-audit-{}", uuid::Uuid::new_v4().simple());
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
seed_lifecycle(&bucket).await;
|
||||
let envelope = authorized_lifecycle_envelope(&ambient, &bucket).await;
|
||||
let before = rustfs_scanner::scanner_dirty_usage_bucket_generation(&bucket);
|
||||
let maintenance_before = rustfs_scanner::scanner_maintenance_generation();
|
||||
let audit = MutationCompletionProbe::install(&bucket);
|
||||
|
||||
let usecase = DefaultBucketUsecase::with_context(Some(ambient));
|
||||
let (result, calls) = observed_delete(&usecase, envelope, &bucket).await;
|
||||
result.expect("delete lifecycle configuration");
|
||||
|
||||
assert!(persisted_lifecycle(&bucket).await.is_empty());
|
||||
let [call] = calls.as_slice() else { panic!("expected one site replication hook call") };
|
||||
assert_eq!(call.bucket, bucket);
|
||||
assert_eq!(call.item_type, "lc-config");
|
||||
// Persistence and the local lifecycle reload each advance the maintenance generation.
|
||||
assert_eq!(
|
||||
call.scanner_maintenance_generation,
|
||||
maintenance_before.saturating_add(2),
|
||||
"lifecycle reload advances maintenance before the site hook"
|
||||
);
|
||||
assert_eq!(rustfs_scanner::scanner_maintenance_generation(), call.scanner_maintenance_generation);
|
||||
assert!(call.scanner_generation > before, "scanner reload precedes site replication");
|
||||
assert_eq!(rustfs_scanner::scanner_dirty_usage_bucket_generation(&bucket), call.scanner_generation);
|
||||
let audits = audit.audits.lock().expect("audit entries");
|
||||
assert_eq!(audits.len(), 1, "one audit entry");
|
||||
assert_eq!(audits[0].request_id.as_deref(), Some(DELETE_REQUEST_ID));
|
||||
assert_eq!(audits[0].api.status_code, Some(200), "audit status uses 200");
|
||||
});
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_get_bucket_lifecycle_configuration_returns_internal_error_when_store_uninitialized() {
|
||||
|
||||
@@ -12,4 +12,141 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Bucket logging configuration use-case operations (reserved for the next batch).
|
||||
//! Bucket logging configuration shared by the S3 edges.
|
||||
|
||||
use super::{
|
||||
ApiError, DefaultBucketUsecase, S3Operation, StatusCode, StorageError, bucket_config_mutation_incarnation, metadata_sys,
|
||||
update_bucket_config_for_incarnation,
|
||||
};
|
||||
use crate::app::RequestEnvelope;
|
||||
use crate::app::storage_api::bucket_usecase::bucket::metadata::BUCKET_LOGGING_CONFIG;
|
||||
use rustfs_gateway_types::persistence::PersistedBucketLoggingStatus;
|
||||
use rustfs_s3_types::S3Result;
|
||||
|
||||
impl DefaultBucketUsecase {
|
||||
pub(crate) async fn execute_put_bucket_logging(
|
||||
&self,
|
||||
envelope: RequestEnvelope,
|
||||
bucket: &str,
|
||||
serialize_configuration: impl FnOnce() -> S3Result<Vec<u8>> + Send,
|
||||
) -> S3Result<()> {
|
||||
RequestEnvelope::audit_only_operation(envelope, S3Operation::PutBucketLogging, StatusCode::OK, |envelope| async move {
|
||||
let expected_incarnation_id = bucket_config_mutation_incarnation(&envelope, bucket)?;
|
||||
self.ensure_config_bucket(bucket).await?;
|
||||
|
||||
let data = serialize_configuration()?;
|
||||
update_bucket_config_for_incarnation(bucket, BUCKET_LOGGING_CONFIG, data, expected_incarnation_id)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn execute_get_bucket_logging(&self, bucket: &str) -> S3Result<PersistedBucketLoggingStatus> {
|
||||
self.ensure_config_bucket(bucket).await?;
|
||||
match metadata_sys::get_logging_config(bucket).await {
|
||||
Ok((config, _)) => Ok(config),
|
||||
Err(StorageError::ConfigNotFound) => Ok(PersistedBucketLoggingStatus::default()),
|
||||
Err(err) => Err(ApiError::from(err).into()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::app::bucket_usecase::test_support::build_request;
|
||||
use crate::app::bucket_usecase::{BucketOperations, DeleteBucketOptions, MakeBucketOptions};
|
||||
use crate::app::gating_test_env::{run_large_stack_test, shared_gating_ambient};
|
||||
use crate::app::runtime_sources::ServerContextSlot;
|
||||
use crate::app::storage_api::owned_parity::{ReqInfo, authorize_bucket_config_mutation};
|
||||
use http::Method;
|
||||
use rustfs_credentials::Credentials;
|
||||
use rustfs_policy::policy::action::{Action, S3Action};
|
||||
use rustfs_s3_types::S3ErrorCode;
|
||||
use serial_test::serial;
|
||||
use std::sync::Arc;
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn put_bucket_logging_checks_bucket_before_serializing() {
|
||||
run_large_stack_test("put-bucket-logging-lookup", || async {
|
||||
let context = shared_gating_ambient().await;
|
||||
let bucket = format!("logging-missing-{}", uuid::Uuid::new_v4().simple());
|
||||
let usecase = DefaultBucketUsecase::with_context(Some(context));
|
||||
let request = build_request((), Method::PUT);
|
||||
let envelope = RequestEnvelope::from_s3s(&request);
|
||||
let error = usecase
|
||||
.execute_put_bucket_logging(envelope, &bucket, || {
|
||||
Err(rustfs_s3_types::s3_error!(MalformedXML, "invalid logging configuration"))
|
||||
})
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(error.code(), &S3ErrorCode::NoSuchBucket);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn put_bucket_logging_preserves_replacement_bytes_after_stale_authorization() {
|
||||
run_large_stack_test("put-bucket-logging-incarnation", || async {
|
||||
const ORIGINAL: &[u8] = b"<BucketLoggingStatus></BucketLoggingStatus>";
|
||||
const UPDATE: &[u8] = b"<BucketLoggingStatus><LoggingEnabled><TargetBucket>other-bucket</TargetBucket><TargetPrefix>logs/</TargetPrefix></LoggingEnabled></BucketLoggingStatus>";
|
||||
let context = shared_gating_ambient().await;
|
||||
let store = context.object_store();
|
||||
let bucket = format!("logging-stale-{}", uuid::Uuid::new_v4().simple());
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create original bucket");
|
||||
let server = ServerContextSlot::new();
|
||||
assert!(server.install(Arc::clone(&context)));
|
||||
let mut request = build_request((), Method::PUT);
|
||||
request.uri = format!("/{bucket}?logging").parse().expect("logging request URI");
|
||||
request.extensions.insert(Arc::clone(&server));
|
||||
request.extensions.insert(ReqInfo {
|
||||
cred: Some(Credentials {
|
||||
access_key: "logging-owner".to_owned(),
|
||||
..Default::default()
|
||||
}),
|
||||
is_owner: true,
|
||||
..Default::default()
|
||||
});
|
||||
let mut envelope = RequestEnvelope::from_s3s(&request);
|
||||
authorize_bucket_config_mutation(&server, &mut envelope, &bucket, Action::S3Action(S3Action::PutBucketLoggingAction))
|
||||
.await
|
||||
.expect("snapshot the original bucket incarnation");
|
||||
|
||||
store
|
||||
.delete_bucket(&bucket, &DeleteBucketOptions::default())
|
||||
.await
|
||||
.expect("delete original bucket");
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create replacement bucket");
|
||||
metadata_sys::update(&bucket, BUCKET_LOGGING_CONFIG, ORIGINAL.to_vec())
|
||||
.await
|
||||
.expect("seed replacement logging config");
|
||||
|
||||
let usecase = DefaultBucketUsecase::with_context(Some(context));
|
||||
let result = usecase
|
||||
.execute_put_bucket_logging(envelope, &bucket, || Ok(UPDATE.to_vec()))
|
||||
.await;
|
||||
let stored = metadata_sys::get_global_bucket_metadata_sys()
|
||||
.expect("initialized metadata system")
|
||||
.read()
|
||||
.await
|
||||
.get_config_from_disk(&bucket)
|
||||
.await
|
||||
.expect("replacement metadata")
|
||||
.logging_config_xml;
|
||||
assert_eq!(
|
||||
(result.err().map(|err| err.code().clone()), stored.as_slice()),
|
||||
(Some(S3ErrorCode::NoSuchBucket), ORIGINAL),
|
||||
"the stale authorization must leave the replacement configuration intact"
|
||||
);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,13 +15,19 @@
|
||||
//! Bucket notification configuration use-case operations.
|
||||
|
||||
use super::*;
|
||||
use crate::app::storage_api::s3::dto::{
|
||||
NotificationConfiguration, NotificationConfigurationFilter, PutBucketNotificationConfigurationInput,
|
||||
PutBucketNotificationConfigurationOutput,
|
||||
};
|
||||
use crate::app::storage_api::s3::region::Region;
|
||||
use crate::app::storage_api::s3::{S3Error, S3ErrorCode, S3Request, S3Response, S3Result, s3_error};
|
||||
use rustfs_gateway_types::persistence::parse_notification_dto;
|
||||
#[cfg(test)]
|
||||
use rustfs_s3_types::S3ErrorCode;
|
||||
use rustfs_s3_types::{S3Result, s3_error};
|
||||
|
||||
/// Runtime rules input retained alongside the prepared persisted configuration.
|
||||
pub(crate) struct NotificationDestination {
|
||||
pub(crate) kind: &'static str,
|
||||
pub(crate) id: Option<String>,
|
||||
pub(crate) events: Vec<String>,
|
||||
pub(crate) target_arn: String,
|
||||
pub(crate) filters: Vec<(Option<String>, Option<String>)>,
|
||||
}
|
||||
|
||||
fn is_valid_notification_filter_value(value: &str) -> bool {
|
||||
if value.len() > 1024 || value.contains('\\') {
|
||||
@@ -41,30 +47,24 @@ fn invalid_filter_name_message(cfg_scope: &str, name: &str) -> String {
|
||||
)
|
||||
}
|
||||
|
||||
fn validate_notification_filter_rules(
|
||||
filter: Option<&NotificationConfigurationFilter>,
|
||||
cfg_kind: &str,
|
||||
cfg_id: Option<&str>,
|
||||
) -> S3Result<()> {
|
||||
let Some(filter) = filter else {
|
||||
fn validate_notification_filter_rules(destination: &NotificationDestination) -> S3Result<()> {
|
||||
if destination.filters.is_empty() {
|
||||
return Ok(());
|
||||
};
|
||||
let Some(s3key_filter) = filter.key.as_ref() else {
|
||||
return Ok(());
|
||||
};
|
||||
let Some(rules) = s3key_filter.filter_rules.as_ref() else {
|
||||
return Ok(());
|
||||
};
|
||||
}
|
||||
|
||||
let mut has_prefix = false;
|
||||
let mut has_suffix = false;
|
||||
let cfg_scope = cfg_id.map_or_else(|| cfg_kind.to_string(), |id| format!("{cfg_kind} id={id}"));
|
||||
let kind = destination.kind;
|
||||
let cfg_scope = destination
|
||||
.id
|
||||
.as_ref()
|
||||
.map_or_else(|| kind.to_owned(), |id| format!("{kind} id={id}"));
|
||||
|
||||
for rule in rules {
|
||||
let Some(name) = rule.name.as_ref() else {
|
||||
for (name, value) in &destination.filters {
|
||||
let Some(name) = name.as_deref() else {
|
||||
return Err(s3_error!(InvalidArgument, "invalid notification filter rule: missing Name ({cfg_scope})"));
|
||||
};
|
||||
let Some(value) = rule.value.as_ref() else {
|
||||
let Some(value) = value.as_deref() else {
|
||||
return Err(s3_error!(
|
||||
InvalidArgument,
|
||||
"invalid notification filter rule: missing Value ({cfg_scope})"
|
||||
@@ -75,43 +75,24 @@ fn validate_notification_filter_rules(
|
||||
return Err(s3_error!(InvalidArgument, "{}", invalid_filter_value_message(&cfg_scope, value)));
|
||||
}
|
||||
|
||||
if name.as_str().eq_ignore_ascii_case("prefix") {
|
||||
if name.eq_ignore_ascii_case("prefix") {
|
||||
if has_prefix {
|
||||
return Err(s3_error!(InvalidArgument, "duplicate notification filter name 'prefix' ({cfg_scope})"));
|
||||
}
|
||||
has_prefix = true;
|
||||
} else if name.as_str().eq_ignore_ascii_case("suffix") {
|
||||
} else if name.eq_ignore_ascii_case("suffix") {
|
||||
if has_suffix {
|
||||
return Err(s3_error!(InvalidArgument, "duplicate notification filter name 'suffix' ({cfg_scope})"));
|
||||
}
|
||||
has_suffix = true;
|
||||
} else {
|
||||
return Err(s3_error!(InvalidArgument, "{}", invalid_filter_name_message(&cfg_scope, name.as_str())));
|
||||
return Err(s3_error!(InvalidArgument, "{}", invalid_filter_name_message(&cfg_scope, name)));
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_notification_configuration_filters(notification_configuration: &NotificationConfiguration) -> S3Result<()> {
|
||||
if let Some(queue_configs) = notification_configuration.queue_configurations.as_ref() {
|
||||
for cfg in queue_configs {
|
||||
validate_notification_filter_rules(cfg.filter.as_ref(), "QueueConfiguration", cfg.id.as_deref())?;
|
||||
}
|
||||
}
|
||||
if let Some(topic_configs) = notification_configuration.topic_configurations.as_ref() {
|
||||
for cfg in topic_configs {
|
||||
validate_notification_filter_rules(cfg.filter.as_ref(), "TopicConfiguration", cfg.id.as_deref())?;
|
||||
}
|
||||
}
|
||||
if let Some(lambda_configs) = notification_configuration.lambda_function_configurations.as_ref() {
|
||||
for cfg in lambda_configs {
|
||||
validate_notification_filter_rules(cfg.filter.as_ref(), "LambdaFunctionConfiguration", cfg.id.as_deref())?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn parse_notification_target_id(arn_str: &str) -> Result<TargetID, TargetIDError> {
|
||||
ARN::parse(arn_str)
|
||||
.map(|arn| arn.target_id)
|
||||
@@ -120,43 +101,34 @@ fn parse_notification_target_id(arn_str: &str) -> Result<TargetID, TargetIDError
|
||||
|
||||
type NotificationEventRule = (Vec<EventName>, String, String, Vec<TargetID>);
|
||||
|
||||
/// Builds the notify runtime rules for a bucket notification configuration
|
||||
/// without touching the store or the runtime rule state.
|
||||
fn build_notification_event_rules(
|
||||
notification_configuration: &NotificationConfiguration,
|
||||
) -> S3Result<Vec<NotificationEventRule>> {
|
||||
let mut event_rules = Vec::new();
|
||||
let invalid_arn = |e: TargetIDError| {
|
||||
S3Error::with_message(S3ErrorCode::InvalidArgument, format!("Invalid ARN in notification configuration: {e}"))
|
||||
};
|
||||
|
||||
process_queue_configurations(
|
||||
&mut event_rules,
|
||||
notification_configuration.queue_configurations.clone(),
|
||||
parse_notification_target_id,
|
||||
)
|
||||
.map_err(invalid_arn)?;
|
||||
process_topic_configurations(
|
||||
&mut event_rules,
|
||||
notification_configuration.topic_configurations.clone(),
|
||||
parse_notification_target_id,
|
||||
)
|
||||
.map_err(invalid_arn)?;
|
||||
process_lambda_configurations(
|
||||
&mut event_rules,
|
||||
notification_configuration.lambda_function_configurations.clone(),
|
||||
parse_notification_target_id,
|
||||
)
|
||||
.map_err(invalid_arn)?;
|
||||
|
||||
Ok(event_rules)
|
||||
fn build_notification_event_rules(destinations: Vec<NotificationDestination>) -> S3Result<Vec<NotificationEventRule>> {
|
||||
destinations
|
||||
.into_iter()
|
||||
.map(|destination| {
|
||||
let events = destination
|
||||
.events
|
||||
.iter()
|
||||
.filter_map(|event| EventName::parse(event).ok())
|
||||
.collect();
|
||||
let (mut prefix, mut suffix) = (String::new(), String::new());
|
||||
for (name, value) in destination.filters {
|
||||
if let (Some(name), Some(value)) = (name, value) {
|
||||
if name.eq_ignore_ascii_case("prefix") {
|
||||
prefix = value;
|
||||
} else if name.eq_ignore_ascii_case("suffix") {
|
||||
suffix = value;
|
||||
}
|
||||
}
|
||||
}
|
||||
let target = parse_notification_target_id(&destination.target_arn)
|
||||
.map_err(|err| s3_error!(InvalidArgument, "Invalid ARN in notification configuration: {err}"))?;
|
||||
Ok((events, prefix, suffix, vec![target]))
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn resolve_notification_region(global_region: Option<Region>, request_region: Option<Region>) -> String {
|
||||
global_region
|
||||
.or(request_region)
|
||||
.map(|region| region.to_string())
|
||||
.unwrap_or_else(|| RUSTFS_REGION.to_string())
|
||||
fn resolve_notification_region(global_region: Option<String>, request_region: Option<&str>) -> String {
|
||||
global_region.unwrap_or_else(|| request_region.unwrap_or(RUSTFS_REGION).to_owned())
|
||||
}
|
||||
|
||||
impl DefaultBucketUsecase {
|
||||
@@ -211,70 +183,74 @@ impl DefaultBucketUsecase {
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn execute_put_bucket_notification_configuration(
|
||||
/// Persists notification configuration, then updates the active rules for the bucket.
|
||||
pub(crate) async fn execute_put_bucket_notification_configuration(
|
||||
&self,
|
||||
req: S3Request<PutBucketNotificationConfigurationInput>,
|
||||
envelope: RequestEnvelope,
|
||||
bucket: &str,
|
||||
destinations: Vec<NotificationDestination>,
|
||||
prepared: S3Result<PersistedNotificationConfiguration>,
|
||||
) -> S3Result<S3Response<PutBucketNotificationConfigurationOutput>> {
|
||||
let expected_incarnation_id = bucket_config_mutation_incarnation(&req, &req.input.bucket)?;
|
||||
let request_region = req.region.clone();
|
||||
let request_context = req.extensions.get::<request_context::RequestContext>().cloned();
|
||||
) -> S3Result<gateway_dto::PutBucketNotificationConfigurationOutput> {
|
||||
RequestEnvelope::audit_only_operation(
|
||||
envelope,
|
||||
S3Operation::PutBucketNotificationConfiguration,
|
||||
StatusCode::OK,
|
||||
|envelope| async move {
|
||||
let expected_incarnation_id = bucket_config_mutation_incarnation(&envelope, bucket)?;
|
||||
let request_context = envelope.request_context().cloned();
|
||||
|
||||
let PutBucketNotificationConfigurationInput {
|
||||
bucket,
|
||||
notification_configuration,
|
||||
..
|
||||
} = req.input;
|
||||
for destination in &destinations {
|
||||
validate_notification_filter_rules(destination)?;
|
||||
}
|
||||
let persisted_notification_configuration = prepared?;
|
||||
|
||||
validate_notification_configuration_filters(¬ification_configuration)?;
|
||||
let persisted_notification_configuration = prepared?;
|
||||
let Some(store) = self.object_store() else {
|
||||
return Err(s3_error!(InternalError, "Not init"));
|
||||
};
|
||||
store
|
||||
.get_bucket_info(bucket, &BucketOptions::default())
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
let Some(store) = self.object_store() else {
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
|
||||
};
|
||||
let region =
|
||||
resolve_notification_region(self.global_region().map(|region| region.to_string()), envelope.region());
|
||||
let notify = current_notify_interface_for_context(self.context.as_deref());
|
||||
let event_rules = build_notification_event_rules(destinations)?;
|
||||
|
||||
store
|
||||
.get_bucket_info(&bucket, &BucketOptions::default())
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
// Validate before the write. Clear and add still run after the durable update.
|
||||
notify
|
||||
.validate_event_specific_rules(bucket, ®ion, &event_rules)
|
||||
.await
|
||||
.map_err(|e| s3_error!(InternalError, "Failed to add rules: {e}"))?;
|
||||
|
||||
let region = resolve_notification_region(self.global_region(), request_region);
|
||||
let notify = current_notify_interface_for_context(self.context.as_deref());
|
||||
let event_rules = build_notification_event_rules(¬ification_configuration)?;
|
||||
let data = serialize_config(&persisted_notification_configuration)?;
|
||||
update_bucket_config_for_incarnation(bucket, BUCKET_NOTIFICATION_CONFIG, data, expected_incarnation_id)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
// Reject the request before the store write so a failure cannot leave a
|
||||
// persisted configuration that the notify runtime refused to activate.
|
||||
notify
|
||||
.validate_event_specific_rules(&bucket, region.as_str(), &event_rules)
|
||||
.await
|
||||
.map_err(|e| s3_error!(InternalError, "Failed to add rules: {e}"))?;
|
||||
notify_bucket_metadata_reload(bucket.to_owned(), "put bucket notification", request_context, false).await;
|
||||
|
||||
let data = serialize_config(&persisted_notification_configuration)?;
|
||||
update_bucket_config_for_incarnation(&bucket, BUCKET_NOTIFICATION_CONFIG, data, expected_incarnation_id)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
notify
|
||||
.clear_bucket_notification_rules(bucket)
|
||||
.await
|
||||
.map_err(|e| s3_error!(InternalError, "Failed to clear rules: {e}"))?;
|
||||
warn!("notify event rules: {:?}", &event_rules);
|
||||
notify
|
||||
.add_event_specific_rules(bucket, ®ion, &event_rules)
|
||||
.await
|
||||
.map_err(|e| s3_error!(InternalError, "Failed to add rules: {e}"))?;
|
||||
|
||||
notify_bucket_metadata_reload(bucket.clone(), "put bucket notification", request_context, false).await;
|
||||
|
||||
notify
|
||||
.clear_bucket_notification_rules(&bucket)
|
||||
.await
|
||||
.map_err(|e| s3_error!(InternalError, "Failed to clear rules: {e}"))?;
|
||||
warn!("notify event rules: {:?}", &event_rules);
|
||||
notify
|
||||
.add_event_specific_rules(&bucket, region.as_str(), &event_rules)
|
||||
.await
|
||||
.map_err(|e| s3_error!(InternalError, "Failed to add rules: {e}"))?;
|
||||
|
||||
Ok(S3Response::new(PutBucketNotificationConfigurationOutput {}))
|
||||
Ok(gateway_dto::PutBucketNotificationConfigurationOutput::default())
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::app::bucket_usecase::test_support::build_request;
|
||||
use crate::app::storage_api::s3::dto::{FilterRule, FilterRuleName, QueueConfiguration, S3KeyFilter};
|
||||
use crate::app::bucket_usecase::test_support::{build_request, owned_envelope};
|
||||
use http::Method;
|
||||
|
||||
#[tokio::test]
|
||||
@@ -294,43 +270,40 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn resolve_notification_region_prefers_global_region() {
|
||||
let binding = resolve_notification_region(Some("us-east-1".parse().unwrap()), Some("ap-southeast-1".parse().unwrap()));
|
||||
assert_eq!(binding, "us-east-1");
|
||||
assert_eq!(
|
||||
resolve_notification_region(Some("us-east-1".to_owned()), Some("ap-southeast-1")),
|
||||
"us-east-1"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_notification_region_falls_back_to_request_region() {
|
||||
let binding = resolve_notification_region(None, Some("ap-southeast-1".parse().unwrap()));
|
||||
assert_eq!(binding, "ap-southeast-1");
|
||||
assert_eq!(resolve_notification_region(None, Some("ap-southeast-1")), "ap-southeast-1");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_notification_region_defaults_value() {
|
||||
let binding = resolve_notification_region(None, None);
|
||||
assert_eq!(binding, RUSTFS_REGION);
|
||||
assert_eq!(resolve_notification_region(None, None), RUSTFS_REGION);
|
||||
}
|
||||
|
||||
fn destination(kind: &'static str, filters: &[(Option<&str>, Option<&str>)]) -> NotificationDestination {
|
||||
NotificationDestination {
|
||||
kind,
|
||||
id: Some("fixture".to_owned()),
|
||||
events: vec!["s3:ObjectCreated:*".to_owned()],
|
||||
target_arn: "arn:rustfs:sqs:us-east-1:1:webhook".to_owned(),
|
||||
filters: filters
|
||||
.iter()
|
||||
.map(|&(name, value)| (name.map(str::to_owned), value.map(str::to_owned)))
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_notification_configuration_filters_rejects_invalid_filter_name() {
|
||||
let raw_name = "unsupported".repeat(100);
|
||||
let cfg = NotificationConfiguration {
|
||||
queue_configurations: Some(vec![QueueConfiguration {
|
||||
id: Some("q1".to_string()),
|
||||
queue_arn: "arn:rustfs:sqs:us-east-1:1:webhook".to_string(),
|
||||
events: vec!["s3:ObjectCreated:*".to_string().into()],
|
||||
filter: Some(NotificationConfigurationFilter {
|
||||
key: Some(S3KeyFilter {
|
||||
filter_rules: Some(vec![FilterRule {
|
||||
name: Some(FilterRuleName::from(raw_name.clone())),
|
||||
value: Some("uploads/".to_string()),
|
||||
}]),
|
||||
}),
|
||||
}),
|
||||
}]),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let err = validate_notification_configuration_filters(&cfg).unwrap_err();
|
||||
let cfg = destination("QueueConfiguration", &[(Some(&raw_name), Some("uploads/"))]);
|
||||
let err = validate_notification_filter_rules(&cfg).unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
|
||||
let msg = err.message().unwrap_or_default();
|
||||
assert!(msg.contains("len="), "error message should include summarized length");
|
||||
@@ -339,81 +312,47 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn validate_notification_configuration_filters_accepts_case_insensitive_filter_names() {
|
||||
let cfg = NotificationConfiguration {
|
||||
queue_configurations: Some(vec![QueueConfiguration {
|
||||
id: Some("q1".to_string()),
|
||||
queue_arn: "arn:rustfs:sqs:us-east-1:1:webhook".to_string(),
|
||||
events: vec!["s3:ObjectCreated:*".to_string().into()],
|
||||
filter: Some(NotificationConfigurationFilter {
|
||||
key: Some(S3KeyFilter {
|
||||
filter_rules: Some(vec![
|
||||
FilterRule {
|
||||
name: Some(FilterRuleName::from("Prefix".to_string())),
|
||||
value: Some("uploads/".to_string()),
|
||||
},
|
||||
FilterRule {
|
||||
name: Some(FilterRuleName::from("Suffix".to_string())),
|
||||
value: Some(".csv".to_string()),
|
||||
},
|
||||
]),
|
||||
}),
|
||||
}),
|
||||
}]),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
validate_notification_configuration_filters(&cfg).expect("capitalized filter names should be accepted");
|
||||
let cfg = destination(
|
||||
"QueueConfiguration",
|
||||
&[(Some("Prefix"), Some("uploads/")), (Some("Suffix"), Some(".csv"))],
|
||||
);
|
||||
validate_notification_filter_rules(&cfg).expect("capitalized filter names should be accepted");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_notification_configuration_filters_rejects_duplicate_prefix_rules() {
|
||||
let cfg = NotificationConfiguration {
|
||||
queue_configurations: Some(vec![QueueConfiguration {
|
||||
id: Some("q1".to_string()),
|
||||
queue_arn: "arn:rustfs:sqs:us-east-1:1:webhook".to_string(),
|
||||
events: vec!["s3:ObjectCreated:*".to_string().into()],
|
||||
filter: Some(NotificationConfigurationFilter {
|
||||
key: Some(S3KeyFilter {
|
||||
filter_rules: Some(vec![
|
||||
FilterRule {
|
||||
name: Some(FilterRuleName::from_static(FilterRuleName::PREFIX)),
|
||||
value: Some("uploads/".to_string()),
|
||||
},
|
||||
FilterRule {
|
||||
name: Some(FilterRuleName::from_static(FilterRuleName::PREFIX)),
|
||||
value: Some("images/".to_string()),
|
||||
},
|
||||
]),
|
||||
}),
|
||||
}),
|
||||
}]),
|
||||
..Default::default()
|
||||
};
|
||||
for kind in ["QueueConfiguration"] {
|
||||
for duplicate in ["prefix", "PREFIX"] {
|
||||
let cfg = destination(kind, &[(Some("prefix"), Some("uploads/")), (Some(duplicate), Some("images/"))]);
|
||||
let err = validate_notification_filter_rules(&cfg).unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
|
||||
assert_eq!(
|
||||
err.message(),
|
||||
Some(format!("duplicate notification filter name 'prefix' ({kind} id=fixture)").as_str())
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let err = validate_notification_configuration_filters(&cfg).unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
|
||||
#[test]
|
||||
fn validate_notification_configuration_filters_rejects_duplicate_suffix_rules() {
|
||||
for kind in ["QueueConfiguration", "TopicConfiguration"] {
|
||||
for duplicate in ["suffix", "SUFFIX"] {
|
||||
let cfg = destination(kind, &[(Some("suffix"), Some(".csv")), (Some(duplicate), Some(".log"))]);
|
||||
let err = validate_notification_filter_rules(&cfg).unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
|
||||
assert_eq!(
|
||||
err.message(),
|
||||
Some(format!("duplicate notification filter name 'suffix' ({kind} id=fixture)").as_str())
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_notification_configuration_filters_rejects_invalid_filter_value() {
|
||||
let cfg = NotificationConfiguration {
|
||||
queue_configurations: Some(vec![QueueConfiguration {
|
||||
id: Some("q1".to_string()),
|
||||
queue_arn: "arn:rustfs:sqs:us-east-1:1:webhook".to_string(),
|
||||
events: vec!["s3:ObjectCreated:*".to_string().into()],
|
||||
filter: Some(NotificationConfigurationFilter {
|
||||
key: Some(S3KeyFilter {
|
||||
filter_rules: Some(vec![FilterRule {
|
||||
name: Some(FilterRuleName::from_static(FilterRuleName::SUFFIX)),
|
||||
value: Some("../secret".to_string()),
|
||||
}]),
|
||||
}),
|
||||
}),
|
||||
}]),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let err = validate_notification_configuration_filters(&cfg).unwrap_err();
|
||||
let cfg = destination("QueueConfiguration", &[(Some("suffix"), Some("../secret"))]);
|
||||
let err = validate_notification_filter_rules(&cfg).unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
|
||||
let msg = err.message().unwrap_or_default();
|
||||
assert!(msg.contains("len="), "error message should include summarized length");
|
||||
@@ -422,109 +361,127 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn validate_notification_configuration_filters_rejects_missing_filter_name() {
|
||||
let cfg = NotificationConfiguration {
|
||||
queue_configurations: Some(vec![QueueConfiguration {
|
||||
id: Some("q1".to_string()),
|
||||
queue_arn: "arn:rustfs:sqs:us-east-1:1:webhook".to_string(),
|
||||
events: vec!["s3:ObjectCreated:*".to_string().into()],
|
||||
filter: Some(NotificationConfigurationFilter {
|
||||
key: Some(S3KeyFilter {
|
||||
filter_rules: Some(vec![FilterRule {
|
||||
name: None,
|
||||
value: Some("uploads/".to_string()),
|
||||
}]),
|
||||
}),
|
||||
}),
|
||||
}]),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let err = validate_notification_configuration_filters(&cfg).unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
|
||||
for (kind, value) in [
|
||||
("QueueConfiguration", Some("uploads/")),
|
||||
("LambdaFunctionConfiguration", None),
|
||||
] {
|
||||
let cfg = destination(kind, &[(None, value)]);
|
||||
let err = validate_notification_filter_rules(&cfg).unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
|
||||
assert_eq!(
|
||||
err.message(),
|
||||
Some(format!("invalid notification filter rule: missing Name ({kind} id=fixture)").as_str())
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_notification_configuration_filters_rejects_missing_filter_value() {
|
||||
let cfg = NotificationConfiguration {
|
||||
queue_configurations: Some(vec![QueueConfiguration {
|
||||
id: Some("q1".to_string()),
|
||||
queue_arn: "arn:rustfs:sqs:us-east-1:1:webhook".to_string(),
|
||||
events: vec!["s3:ObjectCreated:*".to_string().into()],
|
||||
filter: Some(NotificationConfigurationFilter {
|
||||
key: Some(S3KeyFilter {
|
||||
filter_rules: Some(vec![FilterRule {
|
||||
name: Some(FilterRuleName::from_static(FilterRuleName::PREFIX)),
|
||||
value: None,
|
||||
}]),
|
||||
}),
|
||||
}),
|
||||
}]),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let err = validate_notification_configuration_filters(&cfg).unwrap_err();
|
||||
let cfg = destination("QueueConfiguration", &[(Some("prefix"), None)]);
|
||||
let err = validate_notification_filter_rules(&cfg).unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
|
||||
assert_eq!(
|
||||
err.message(),
|
||||
Some("invalid notification filter rule: missing Value (QueueConfiguration id=fixture)")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_notification_configuration_filters_rejects_duplicate_suffix_rules() {
|
||||
let cfg = NotificationConfiguration {
|
||||
queue_configurations: Some(vec![QueueConfiguration {
|
||||
id: Some("q1".to_string()),
|
||||
queue_arn: "arn:rustfs:sqs:us-east-1:1:webhook".to_string(),
|
||||
events: vec!["s3:ObjectCreated:*".to_string().into()],
|
||||
filter: Some(NotificationConfigurationFilter {
|
||||
key: Some(S3KeyFilter {
|
||||
filter_rules: Some(vec![
|
||||
FilterRule {
|
||||
name: Some(FilterRuleName::from_static(FilterRuleName::SUFFIX)),
|
||||
value: Some(".csv".to_string()),
|
||||
},
|
||||
FilterRule {
|
||||
name: Some(FilterRuleName::from_static(FilterRuleName::SUFFIX)),
|
||||
value: Some(".log".to_string()),
|
||||
},
|
||||
]),
|
||||
}),
|
||||
}),
|
||||
}]),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let err = validate_notification_configuration_filters(&cfg).unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
|
||||
fn notification_filter_values_preserve_byte_and_path_boundaries() {
|
||||
let limit = "é".repeat(512);
|
||||
for value in ["", "a.b/..suffix/.prefix", &limit] {
|
||||
let cfg = destination("QueueConfiguration", &[(Some("prefix"), Some(value))]);
|
||||
validate_notification_filter_rules(&cfg).unwrap();
|
||||
}
|
||||
let too_long = format!("{limit}a");
|
||||
for value in ["a\\b", "a/./b", "a/../b", &too_long] {
|
||||
let cfg = destination("QueueConfiguration", &[(Some("unknown"), Some(value))]);
|
||||
let error = validate_notification_filter_rules(&cfg).unwrap_err();
|
||||
assert_eq!(error.code(), &S3ErrorCode::InvalidArgument);
|
||||
assert_eq!(
|
||||
error.message(),
|
||||
Some(format!("invalid notification filter value (len={}) (QueueConfiguration id=fixture)", value.len()).as_str())
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_put_bucket_notification_configuration_rejects_invalid_filter_before_store_lookup() {
|
||||
let input = PutBucketNotificationConfigurationInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.notification_configuration(NotificationConfiguration {
|
||||
queue_configurations: Some(vec![QueueConfiguration {
|
||||
id: Some("q1".to_string()),
|
||||
queue_arn: "arn:rustfs:sqs:us-east-1:1:webhook".to_string(),
|
||||
events: vec!["s3:ObjectCreated:*".to_string().into()],
|
||||
filter: Some(NotificationConfigurationFilter {
|
||||
key: Some(S3KeyFilter {
|
||||
filter_rules: Some(vec![FilterRule {
|
||||
name: Some(FilterRuleName::from("unsupported".to_string())),
|
||||
value: Some("uploads/".to_string()),
|
||||
}]),
|
||||
}),
|
||||
}),
|
||||
}]),
|
||||
..Default::default()
|
||||
})
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let req = build_request(input, Method::PUT);
|
||||
let usecase = DefaultBucketUsecase::without_context();
|
||||
|
||||
let err = usecase
|
||||
.execute_put_bucket_notification_configuration(req, Ok(PersistedNotificationConfiguration::default()))
|
||||
let bucket = "test-bucket";
|
||||
let error = usecase
|
||||
.execute_put_bucket_notification_configuration(
|
||||
owned_envelope(Method::PUT),
|
||||
bucket,
|
||||
vec![destination("QueueConfiguration", &[(Some("unsupported"), Some("uploads/"))])],
|
||||
Ok(PersistedNotificationConfiguration::default()),
|
||||
)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
|
||||
assert_eq!(error.code(), &S3ErrorCode::InvalidArgument);
|
||||
|
||||
let invalid = [
|
||||
destination("QueueConfiguration", &[(Some("unsupported"), Some("uploads/"))]),
|
||||
destination("TopicConfiguration", &[(None, None)]),
|
||||
];
|
||||
let error = usecase
|
||||
.execute_put_bucket_notification_configuration(
|
||||
owned_envelope(Method::PUT),
|
||||
bucket,
|
||||
invalid.into(),
|
||||
Err(s3_error!(MalformedXML, "persistence conversion")),
|
||||
)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(error.code(), &S3ErrorCode::InvalidArgument);
|
||||
assert_eq!(
|
||||
error.message(),
|
||||
Some(
|
||||
"invalid notification filter name (len=11) (only 'prefix'/'suffix' are supported) (QueueConfiguration id=fixture)"
|
||||
)
|
||||
);
|
||||
|
||||
let error = usecase
|
||||
.execute_put_bucket_notification_configuration(
|
||||
owned_envelope(Method::PUT),
|
||||
bucket,
|
||||
Vec::new(),
|
||||
Err(s3_error!(MalformedXML, "persistence conversion")),
|
||||
)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(error.code(), &S3ErrorCode::MalformedXML);
|
||||
assert_eq!(error.message(), Some("persistence conversion"));
|
||||
|
||||
let mut request = build_request((), Method::PUT);
|
||||
request.extensions.insert(crate::runtime_sources::ServerContextSlot::new());
|
||||
let error = usecase
|
||||
.execute_put_bucket_notification_configuration(
|
||||
RequestEnvelope::from_s3s(&request),
|
||||
bucket,
|
||||
vec![destination("QueueConfiguration", &[(None, None)])],
|
||||
Err(s3_error!(MalformedXML, "persistence conversion")),
|
||||
)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(error.code(), &S3ErrorCode::InternalError);
|
||||
assert_eq!(error.message(), Some("bucket config mutation snapshot is missing"));
|
||||
|
||||
let mut invalid_arn = destination("QueueConfiguration", &[]);
|
||||
invalid_arn.target_arn = "invalid-arn".to_owned();
|
||||
let error = usecase
|
||||
.execute_put_bucket_notification_configuration(
|
||||
owned_envelope(Method::PUT),
|
||||
bucket,
|
||||
vec![invalid_arn],
|
||||
Ok(PersistedNotificationConfiguration::default()),
|
||||
)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(error.code(), &S3ErrorCode::InternalError);
|
||||
assert_eq!(error.message(), Some("Not init"));
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod side_effect_tests;
|
||||
|
||||
@@ -0,0 +1,459 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Store and callback observations through the legacy Put entrypoint and shared use case.
|
||||
|
||||
use super::*;
|
||||
use crate::app::bucket_usecase::test_support::build_request_with_req_info;
|
||||
use crate::app::gating_test_env::{app_context_from_current_environment, run_large_stack_test, shared_gating_ambient};
|
||||
use crate::app::storage_api::bucket_usecase::access::ReqInfo;
|
||||
use crate::app::storage_api::bucket_usecase::helper::MutationCompletionProbe;
|
||||
use crate::app::storage_api::s3::dto::{
|
||||
FilterRule, LambdaFunctionConfiguration, NotificationConfiguration, NotificationConfigurationFilter,
|
||||
PutBucketNotificationConfigurationInput, QueueConfiguration, S3KeyFilter, TopicConfiguration,
|
||||
};
|
||||
use crate::app::storage_api::s3::{S3, S3Access, S3ErrorCode, S3Request};
|
||||
use crate::app::storage_api::test::ecfs::FS;
|
||||
use crate::runtime_sources::{NotifyInterface, ServerContextSlot};
|
||||
use http::Method;
|
||||
use rustfs_notify::{EventArgs, NotificationError};
|
||||
use std::sync::Mutex;
|
||||
|
||||
const EMPTY_XML: &[u8] = b"<NotificationConfiguration></NotificationConfiguration>";
|
||||
const OLD_XML: &[u8] =
|
||||
b"<NotificationConfiguration><EventBridgeConfiguration></EventBridgeConfiguration></NotificationConfiguration>";
|
||||
const NEW_XML: &[u8] = concat!(
|
||||
"<NotificationConfiguration>",
|
||||
"<CloudFunctionConfiguration><Event>s3:Future:Unrecognized</Event><Id>lambda</Id><CloudFunction>arn:rustfs:sqs:us-east-1:3:webhook</CloudFunction></CloudFunctionConfiguration>",
|
||||
"<QueueConfiguration><Event>s3:ObjectCreated:*</Event><Event>s3:Future:Unrecognized</Event>",
|
||||
"<Filter><S3Key><FilterRule><Name>Prefix</Name><Value>logs/%2F</Value></FilterRule>",
|
||||
"<FilterRule><Name>Suffix</Name><Value>.csv</Value></FilterRule></S3Key></Filter>",
|
||||
"<Id>queue</Id><Queue>arn:rustfs:sqs:us-east-1:1:webhook</Queue></QueueConfiguration>",
|
||||
"<TopicConfiguration><Event>s3:ObjectRemoved:*</Event><Id>topic</Id><Topic>arn:rustfs:sqs:us-east-1:2:webhook</Topic></TopicConfiguration>",
|
||||
"</NotificationConfiguration>"
|
||||
).as_bytes();
|
||||
|
||||
type RuntimeRule = (Vec<EventName>, String, String, Vec<TargetID>);
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
enum Callback {
|
||||
Validate,
|
||||
ReloadStarted,
|
||||
ReloadCompleted,
|
||||
Clear,
|
||||
Add,
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
enum Observation {
|
||||
Rules(Callback, String, Vec<RuntimeRule>, Vec<u8>),
|
||||
Reload(Callback, &'static str, bool),
|
||||
Clear(Vec<u8>),
|
||||
}
|
||||
|
||||
struct RecordingNotify {
|
||||
bucket: String,
|
||||
fail_at: Option<Callback>,
|
||||
calls: Arc<Mutex<Vec<Observation>>>,
|
||||
active_rules: Mutex<Vec<RuntimeRule>>,
|
||||
}
|
||||
|
||||
async fn read_notification_xml(bucket: &str) -> Vec<u8> {
|
||||
crate::app::storage_api::test::get_global_bucket_metadata_sys()
|
||||
.expect("test bucket metadata")
|
||||
.read()
|
||||
.await
|
||||
.get_config_from_disk(bucket)
|
||||
.await
|
||||
.expect("load notification bytes from disk")
|
||||
.notification_config_xml
|
||||
}
|
||||
|
||||
impl RecordingNotify {
|
||||
fn maybe_fail(&self, callback: Callback) -> Result<(), NotificationError> {
|
||||
if self.fail_at == Some(callback) {
|
||||
return Err(NotificationError::RuleConfiguration("fixture failure".into()));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn record_rules(&self, callback: Callback, bucket: &str, region: &str, rules: &[RuntimeRule]) {
|
||||
assert_eq!(bucket, self.bucket);
|
||||
let xml = read_notification_xml(bucket).await;
|
||||
self.calls
|
||||
.lock()
|
||||
.unwrap()
|
||||
.push(Observation::Rules(callback, region.to_owned(), rules.to_vec(), xml));
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl NotifyInterface for RecordingNotify {
|
||||
async fn notify(&self, _args: EventArgs) {
|
||||
panic!("bucket notification configuration must not emit an object event");
|
||||
}
|
||||
|
||||
async fn validate_event_specific_rules(
|
||||
&self,
|
||||
bucket: &str,
|
||||
region: &str,
|
||||
rules: &[RuntimeRule],
|
||||
) -> Result<(), NotificationError> {
|
||||
self.record_rules(Callback::Validate, bucket, region, rules).await;
|
||||
self.maybe_fail(Callback::Validate)
|
||||
}
|
||||
|
||||
async fn add_event_specific_rules(&self, bucket: &str, region: &str, rules: &[RuntimeRule]) -> Result<(), NotificationError> {
|
||||
self.record_rules(Callback::Add, bucket, region, rules).await;
|
||||
self.maybe_fail(Callback::Add)?;
|
||||
*self.active_rules.lock().unwrap() = rules.to_vec();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn clear_bucket_notification_rules(&self, bucket: &str) -> Result<(), NotificationError> {
|
||||
assert_eq!(bucket, self.bucket);
|
||||
let xml = read_notification_xml(bucket).await;
|
||||
self.calls.lock().unwrap().push(Observation::Clear(xml));
|
||||
self.maybe_fail(Callback::Clear)?;
|
||||
self.active_rules.lock().unwrap().clear();
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn configuration() -> NotificationConfiguration {
|
||||
NotificationConfiguration {
|
||||
queue_configurations: Some(vec![QueueConfiguration {
|
||||
id: Some("queue".into()),
|
||||
events: vec![
|
||||
"s3:ObjectCreated:*".to_owned().into(),
|
||||
"s3:Future:Unrecognized".to_owned().into(),
|
||||
],
|
||||
queue_arn: "arn:rustfs:sqs:us-east-1:1:webhook".into(),
|
||||
filter: Some(NotificationConfigurationFilter {
|
||||
key: Some(S3KeyFilter {
|
||||
filter_rules: Some(vec![
|
||||
FilterRule {
|
||||
name: Some("Prefix".to_owned().into()),
|
||||
value: Some("logs/%2F".into()),
|
||||
},
|
||||
FilterRule {
|
||||
name: Some("Suffix".to_owned().into()),
|
||||
value: Some(".csv".into()),
|
||||
},
|
||||
]),
|
||||
}),
|
||||
}),
|
||||
}]),
|
||||
topic_configurations: Some(vec![TopicConfiguration {
|
||||
id: Some("topic".into()),
|
||||
events: vec!["s3:ObjectRemoved:*".to_owned().into()],
|
||||
topic_arn: "arn:rustfs:sqs:us-east-1:2:webhook".into(),
|
||||
filter: None,
|
||||
}]),
|
||||
lambda_function_configurations: Some(vec![LambdaFunctionConfiguration {
|
||||
id: Some("lambda".into()),
|
||||
events: vec!["s3:Future:Unrecognized".to_owned().into()],
|
||||
lambda_function_arn: "arn:rustfs:sqs:us-east-1:3:webhook".into(),
|
||||
filter: None,
|
||||
}]),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn expected_rules() -> Vec<RuntimeRule> {
|
||||
[
|
||||
(vec![EventName::parse("s3:ObjectCreated:*").unwrap()], "logs/%2F", ".csv", "1"),
|
||||
(vec![EventName::parse("s3:ObjectRemoved:*").unwrap()], "", "", "2"),
|
||||
(Vec::new(), "", "", "3"),
|
||||
]
|
||||
.map(|(events, prefix, suffix, id)| {
|
||||
(
|
||||
events,
|
||||
prefix.to_owned(),
|
||||
suffix.to_owned(),
|
||||
vec![TargetID {
|
||||
id: id.to_owned(),
|
||||
name: "webhook".into(),
|
||||
}],
|
||||
)
|
||||
})
|
||||
.into()
|
||||
}
|
||||
|
||||
fn notification_request(
|
||||
bucket: &str,
|
||||
configuration: NotificationConfiguration,
|
||||
slot: &Arc<ServerContextSlot>,
|
||||
) -> S3Request<PutBucketNotificationConfigurationInput> {
|
||||
let input = PutBucketNotificationConfigurationInput::builder()
|
||||
.bucket(bucket.to_owned())
|
||||
.notification_configuration(configuration)
|
||||
.build()
|
||||
.unwrap();
|
||||
let mut request = build_request_with_req_info(
|
||||
input,
|
||||
Method::PUT,
|
||||
ReqInfo {
|
||||
bucket: Some(bucket.to_owned()),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
request.extensions.insert(slot.clone());
|
||||
request.uri = format!("/{bucket}?notification").parse().unwrap();
|
||||
request.region = Some("eu-west-2".parse().unwrap());
|
||||
request
|
||||
}
|
||||
|
||||
fn check_notification_write_order() {
|
||||
run_large_stack_test("notification-write-order", || async {
|
||||
let ambient = shared_gating_ambient().await;
|
||||
let store = ambient.object_store();
|
||||
for fail_at in [None, Some(Callback::Validate), Some(Callback::Clear), Some(Callback::Add)] {
|
||||
let bucket = format!("notify-order-{}", uuid::Uuid::new_v4().simple());
|
||||
store.make_bucket(&bucket, &MakeBucketOptions::default()).await.unwrap();
|
||||
store
|
||||
.update_bucket_metadata_config(&bucket, BUCKET_NOTIFICATION_CONFIG, OLD_XML.to_vec())
|
||||
.await
|
||||
.unwrap();
|
||||
let policy = serde_json::json!({
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{"Effect": "Allow", "Principal": {"AWS": "*"}, "Action": ["s3:PutBucketNotification"],
|
||||
"Resource": [format!("arn:aws:s3:::{bucket}")]}]
|
||||
});
|
||||
store
|
||||
.update_bucket_metadata_config(&bucket, BUCKET_POLICY_CONFIG, serde_json::to_vec(&policy).unwrap())
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let calls = Arc::new(Mutex::new(Vec::new()));
|
||||
let recorder = Arc::new(RecordingNotify {
|
||||
bucket: bucket.clone(),
|
||||
fail_at,
|
||||
calls: calls.clone(),
|
||||
active_rules: Mutex::new(expected_rules()),
|
||||
});
|
||||
let context = Arc::new(app_context_from_current_environment(&ambient).with_test_notify_interface(recorder.clone()));
|
||||
let region = context
|
||||
.region()
|
||||
.get()
|
||||
.map_or_else(|| "eu-west-2".to_owned(), |region| region.to_string());
|
||||
let slot = ServerContextSlot::new();
|
||||
assert!(slot.install(context.clone()));
|
||||
let fs = FS::with_server_ctx(slot.clone());
|
||||
let audit = MutationCompletionProbe::install(&bucket);
|
||||
let mut request = notification_request(&bucket, configuration(), &slot);
|
||||
S3Access::put_bucket_notification_configuration(&fs, &mut request)
|
||||
.await
|
||||
.unwrap();
|
||||
let result = observe_reload(&bucket, calls.clone(), S3::put_bucket_notification_configuration(&fs, request)).await;
|
||||
let expected = expected_rules();
|
||||
let mut timeline = vec![Observation::Rules(
|
||||
Callback::Validate,
|
||||
region.clone(),
|
||||
expected.clone(),
|
||||
OLD_XML.to_vec(),
|
||||
)];
|
||||
if fail_at != Some(Callback::Validate) {
|
||||
timeline.extend([
|
||||
Observation::Reload(Callback::ReloadStarted, "put bucket notification", false),
|
||||
Observation::Reload(Callback::ReloadCompleted, "put bucket notification", false),
|
||||
Observation::Clear(NEW_XML.to_vec()),
|
||||
]);
|
||||
if fail_at != Some(Callback::Clear) {
|
||||
timeline.push(Observation::Rules(Callback::Add, region.clone(), expected.clone(), NEW_XML.to_vec()));
|
||||
}
|
||||
}
|
||||
assert_eq!(*calls.lock().unwrap(), timeline);
|
||||
let stored = if fail_at == Some(Callback::Validate) {
|
||||
OLD_XML
|
||||
} else {
|
||||
NEW_XML
|
||||
};
|
||||
assert_eq!(read_notification_xml(&bucket).await, stored);
|
||||
let error_message = match fail_at {
|
||||
Some(Callback::Clear) => Some("Failed to clear rules: Rule configuration error: fixture failure"),
|
||||
Some(_) => Some("Failed to add rules: Rule configuration error: fixture failure"),
|
||||
None => None,
|
||||
};
|
||||
if let Some(message) = error_message {
|
||||
let error = result.expect_err("injected callback failure");
|
||||
assert_eq!(error.code(), &S3ErrorCode::InternalError);
|
||||
assert_eq!(error.message(), Some(message));
|
||||
} else {
|
||||
result.expect("configuration written");
|
||||
}
|
||||
{
|
||||
let audits = audit.audits.lock().unwrap();
|
||||
let [entry] = audits.as_slice() else { panic!("one notification audit: {audits:?}") };
|
||||
assert_eq!(entry.api.name.as_deref(), Some(S3Operation::PutBucketNotificationConfiguration.as_str()));
|
||||
assert_eq!(entry.api.bucket.as_deref(), Some(bucket.as_str()));
|
||||
assert_eq!(entry.api.status_code, Some(if fail_at.is_some() { 500 } else { 200 }));
|
||||
assert_eq!(entry.error.as_deref().filter(|message| !message.is_empty()), error_message);
|
||||
}
|
||||
assert!(audit.events.lock().unwrap().is_empty());
|
||||
|
||||
if fail_at.is_none() {
|
||||
assert_eq!(*recorder.active_rules.lock().unwrap(), expected);
|
||||
let mut invalid = configuration();
|
||||
invalid.lambda_function_configurations.as_mut().unwrap()[0].lambda_function_arn = "invalid-arn".into();
|
||||
let mut invalid = notification_request(&bucket, invalid, &slot);
|
||||
S3Access::put_bucket_notification_configuration(&fs, &mut invalid)
|
||||
.await
|
||||
.unwrap();
|
||||
calls.lock().unwrap().clear();
|
||||
let err = observe_reload(&bucket, calls.clone(), S3::put_bucket_notification_configuration(&fs, invalid))
|
||||
.await
|
||||
.expect_err("the last destination has an invalid ARN");
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
|
||||
assert!(
|
||||
err.message()
|
||||
.unwrap()
|
||||
.starts_with("Invalid ARN in notification configuration: ")
|
||||
);
|
||||
assert!(
|
||||
calls.lock().unwrap().is_empty(),
|
||||
"validation happens after all destinations have been built"
|
||||
);
|
||||
assert_eq!(read_notification_xml(&bucket).await, NEW_XML);
|
||||
let usecase = DefaultBucketUsecase::with_context(Some(context));
|
||||
let output = usecase
|
||||
.execute_get_bucket_notification_configuration(gateway_dto::GetBucketNotificationConfigurationInput {
|
||||
bucket: super::super::test_support::gateway_bucket(&bucket),
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(output.queue_configurations[0].events[1].as_str(), "s3:Future:Unrecognized");
|
||||
|
||||
calls.lock().unwrap().clear();
|
||||
let mut empty = notification_request(&bucket, NotificationConfiguration::default(), &slot);
|
||||
S3Access::put_bucket_notification_configuration(&fs, &mut empty)
|
||||
.await
|
||||
.unwrap();
|
||||
observe_reload(&bucket, calls.clone(), S3::put_bucket_notification_configuration(&fs, empty))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
*calls.lock().unwrap(),
|
||||
[
|
||||
Observation::Rules(Callback::Validate, region.clone(), Vec::new(), NEW_XML.to_vec()),
|
||||
Observation::Reload(Callback::ReloadStarted, "put bucket notification", false),
|
||||
Observation::Reload(Callback::ReloadCompleted, "put bucket notification", false),
|
||||
Observation::Clear(EMPTY_XML.to_vec()),
|
||||
Observation::Rules(Callback::Add, region, Vec::new(), EMPTY_XML.to_vec()),
|
||||
]
|
||||
);
|
||||
assert_eq!(read_notification_xml(&bucket).await, EMPTY_XML);
|
||||
assert!(recorder.active_rules.lock().unwrap().is_empty());
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn legacy_put_notification_keeps_callback_order_and_durable_bytes() {
|
||||
temp_env::with_var(rustfs_config::ENV_AUDIT_ENABLE, Some("true"), || {
|
||||
crate::server::refresh_audit_module_enabled();
|
||||
check_notification_write_order();
|
||||
});
|
||||
crate::server::refresh_audit_module_enabled();
|
||||
}
|
||||
|
||||
async fn observe_reload<T>(bucket: &str, calls: Arc<Mutex<Vec<Observation>>>, operation: impl Future<Output = T>) -> T {
|
||||
super::super::BUCKET_METADATA_RELOAD_PROBE
|
||||
.scope(
|
||||
(
|
||||
bucket.to_owned(),
|
||||
Box::new(move |completed, operation, scanner| {
|
||||
let callback = if completed {
|
||||
Callback::ReloadCompleted
|
||||
} else {
|
||||
Callback::ReloadStarted
|
||||
};
|
||||
calls.lock().unwrap().push(Observation::Reload(callback, operation, scanner));
|
||||
}),
|
||||
),
|
||||
operation,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn legacy_put_notification_does_not_write_after_the_bucket_is_recreated() {
|
||||
run_large_stack_test("notification-stale-incarnation", || async {
|
||||
let ambient = shared_gating_ambient().await;
|
||||
let store = ambient.object_store();
|
||||
let bucket = format!("notify-stale-{}", uuid::Uuid::new_v4().simple());
|
||||
store.make_bucket(&bucket, &MakeBucketOptions::default()).await.unwrap();
|
||||
let first = store.bucket_incarnation_id(&bucket).await.unwrap();
|
||||
let policy = serde_json::json!({
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{"Effect": "Allow", "Principal": {"AWS": "*"}, "Action": ["s3:PutBucketNotification"],
|
||||
"Resource": [format!("arn:aws:s3:::{bucket}")]}]
|
||||
});
|
||||
store
|
||||
.update_bucket_metadata_config(&bucket, BUCKET_POLICY_CONFIG, serde_json::to_vec(&policy).unwrap())
|
||||
.await
|
||||
.unwrap();
|
||||
let calls = Arc::new(Mutex::new(Vec::new()));
|
||||
let recorder = Arc::new(RecordingNotify {
|
||||
bucket: bucket.clone(),
|
||||
fail_at: None,
|
||||
calls: calls.clone(),
|
||||
active_rules: Mutex::new(Vec::new()),
|
||||
});
|
||||
let context = Arc::new(app_context_from_current_environment(&ambient).with_test_notify_interface(recorder));
|
||||
let slot = ServerContextSlot::new();
|
||||
assert!(slot.install(context));
|
||||
let fs = FS::with_server_ctx(slot.clone());
|
||||
let mut request = notification_request(&bucket, configuration(), &slot);
|
||||
S3Access::put_bucket_notification_configuration(&fs, &mut request)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(bucket_config_mutation_incarnation(&request, &bucket).unwrap(), Some(first));
|
||||
|
||||
let mismatch = DefaultBucketUsecase::without_context()
|
||||
.execute_put_bucket_notification_configuration(
|
||||
RequestEnvelope::from_s3s(&request),
|
||||
"other-bucket",
|
||||
vec![NotificationDestination {
|
||||
kind: "QueueConfiguration",
|
||||
id: None,
|
||||
events: Vec::new(),
|
||||
target_arn: String::new(),
|
||||
filters: vec![(None, None)],
|
||||
}],
|
||||
Err(rustfs_s3_types::s3_error!(MalformedXML, "persistence conversion")),
|
||||
)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(mismatch.code(), &rustfs_s3_types::S3ErrorCode::InternalError);
|
||||
assert_eq!(mismatch.message(), Some("bucket config mutation snapshot does not match request bucket"));
|
||||
store.delete_bucket(&bucket, &DeleteBucketOptions::default()).await.unwrap();
|
||||
store.make_bucket(&bucket, &MakeBucketOptions::default()).await.unwrap();
|
||||
assert_ne!(first, store.bucket_incarnation_id(&bucket).await.unwrap());
|
||||
store
|
||||
.update_bucket_metadata_config(&bucket, BUCKET_NOTIFICATION_CONFIG, OLD_XML.to_vec())
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let result = observe_reload(&bucket, calls.clone(), S3::put_bucket_notification_configuration(&fs, request)).await;
|
||||
assert_eq!(read_notification_xml(&bucket).await, OLD_XML);
|
||||
assert_eq!(result.unwrap_err().code(), &S3ErrorCode::NoSuchBucket);
|
||||
assert!(
|
||||
matches!(calls.lock().unwrap().as_slice(), [Observation::Rules(Callback::Validate, _, _, xml)] if xml == OLD_XML)
|
||||
);
|
||||
});
|
||||
}
|
||||
@@ -14,14 +14,213 @@
|
||||
|
||||
//! Bucket Object Lock configuration use-case operations.
|
||||
//!
|
||||
//! NOT responsible for: the lock validators, which object writes share (`storage::ecfs_extend`).
|
||||
//! Default-retention policy here applies to bucket configuration. Object writes enforce their
|
||||
//! own retention rules.
|
||||
|
||||
use super::*;
|
||||
use rustfs_gateway_types::persistence::parse_object_lock_dto;
|
||||
use crate::app::storage_api::bucket_usecase::bucket::VersioningConfigExt as _;
|
||||
use crate::app::storage_api::bucket_usecase::bucket::metadata::OBJECT_LOCK_CONFIG;
|
||||
use rustfs_gateway_types::persistence::{PersistedDefaultRetention, parse_object_lock_dto};
|
||||
use rustfs_s3_types::{S3Error, S3ErrorCode, S3Result, s3_error};
|
||||
|
||||
const LOG_SUBSYSTEM_OBJECT_LOCK: &str = "object_lock";
|
||||
|
||||
const MAXIMUM_RETENTION_DAYS: i32 = 36_500;
|
||||
const MAXIMUM_RETENTION_YEARS: i32 = 100;
|
||||
|
||||
fn invalid_object_lock_configuration(message: impl Into<String>) -> S3Error {
|
||||
S3Error::with_message(S3ErrorCode::MalformedXML, message.into())
|
||||
}
|
||||
|
||||
fn propagate_object_lock_peer_reload(result: std::result::Result<(), StorageError>) -> S3Result<()> {
|
||||
result.map_err(|err| {
|
||||
S3Error::with_message(S3ErrorCode::InternalError, format!("Failed to publish Object Lock metadata: {err}"))
|
||||
})
|
||||
}
|
||||
|
||||
fn invalid_retention_period(message: impl Into<String>) -> S3Error {
|
||||
let mut err = S3Error::with_message(S3ErrorCode::Custom("InvalidRetentionPeriod".into()), message.into());
|
||||
err.set_status_code(StatusCode::BAD_REQUEST);
|
||||
err
|
||||
}
|
||||
|
||||
fn validate_default_retention_configuration(default_retention: &PersistedDefaultRetention) -> S3Result<()> {
|
||||
let Some(mode) = default_retention.mode.as_deref() else {
|
||||
return Err(invalid_object_lock_configuration("retention mode must be specified"));
|
||||
};
|
||||
|
||||
match mode {
|
||||
"COMPLIANCE" | "GOVERNANCE" => {}
|
||||
_ => {
|
||||
return Err(invalid_object_lock_configuration(format!("unknown retention mode {mode}")));
|
||||
}
|
||||
}
|
||||
|
||||
match (default_retention.days, default_retention.years) {
|
||||
(Some(days), None) => {
|
||||
if days <= 0 {
|
||||
return Err(invalid_retention_period(
|
||||
"Default retention period must be a positive integer value for 'Days'",
|
||||
));
|
||||
}
|
||||
if days > MAXIMUM_RETENTION_DAYS {
|
||||
return Err(invalid_retention_period(format!("Default retention period too large for 'Days' {days}",)));
|
||||
}
|
||||
}
|
||||
(None, Some(years)) => {
|
||||
if years <= 0 {
|
||||
return Err(invalid_retention_period(
|
||||
"Default retention period must be a positive integer value for 'Years'",
|
||||
));
|
||||
}
|
||||
if years > MAXIMUM_RETENTION_YEARS {
|
||||
return Err(invalid_retention_period(format!(
|
||||
"Default retention period too large for 'Years' {years}",
|
||||
)));
|
||||
}
|
||||
}
|
||||
(Some(_), Some(_)) => {
|
||||
return Err(invalid_object_lock_configuration("either Days or Years must be specified, not both"));
|
||||
}
|
||||
(None, None) => {
|
||||
return Err(invalid_object_lock_configuration("either Days or Years must be specified"));
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_object_lock_configuration_input(input_cfg: &ObjectLockConfiguration) -> S3Result<()> {
|
||||
let enabled = input_cfg.object_lock_enabled.as_deref();
|
||||
if enabled != Some("Enabled") {
|
||||
return Err(invalid_object_lock_configuration(
|
||||
"only 'Enabled' value is allowed to ObjectLockEnabled element",
|
||||
));
|
||||
}
|
||||
|
||||
if let Some(rule) = input_cfg.rule.as_ref() {
|
||||
let Some(default_retention) = rule.default_retention.as_ref() else {
|
||||
return Err(invalid_object_lock_configuration("Rule must include DefaultRetention"));
|
||||
};
|
||||
validate_default_retention_configuration(default_retention)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
impl DefaultBucketUsecase {
|
||||
/// Persists the lock configuration, then enables versioning under the same authorization.
|
||||
///
|
||||
/// Each write is fenced independently. A versioning write or peer reload can fail after
|
||||
/// the lock configuration is durable.
|
||||
pub(crate) async fn execute_put_object_lock_configuration(
|
||||
&self,
|
||||
envelope: RequestEnvelope,
|
||||
bucket: &str,
|
||||
config: Option<ObjectLockConfiguration>,
|
||||
) -> S3Result<()> {
|
||||
RequestEnvelope::audit_only_operation(envelope, S3Operation::PutObjectLockConfiguration, StatusCode::OK, |envelope| {
|
||||
self.put_object_lock_configuration(envelope, bucket, config)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
async fn put_object_lock_configuration(
|
||||
&self,
|
||||
envelope: RequestEnvelope,
|
||||
bucket: &str,
|
||||
config: Option<ObjectLockConfiguration>,
|
||||
) -> S3Result<()> {
|
||||
let expected_incarnation_id = bucket_config_mutation_incarnation(&envelope, bucket)?;
|
||||
let Some(config) = config else {
|
||||
return Err(s3_error!(InvalidArgument, "Invalid argument."));
|
||||
};
|
||||
self.ensure_config_bucket(bucket).await?;
|
||||
validate_object_lock_configuration_input(&config)?;
|
||||
|
||||
match metadata_sys::get_object_lock_config(bucket).await {
|
||||
Ok(_) => {}
|
||||
Err(StorageError::ConfigNotFound) => {
|
||||
if !BucketVersioningSys::enabled(bucket).await {
|
||||
return Err(s3_error!(
|
||||
InvalidBucketState,
|
||||
"Object Lock configuration cannot be enabled on existing buckets"
|
||||
));
|
||||
}
|
||||
}
|
||||
Err(err) => {
|
||||
warn!(
|
||||
component = LOG_COMPONENT_APP,
|
||||
subsystem = LOG_SUBSYSTEM_OBJECT_LOCK,
|
||||
event = "object_lock_config_load_failed",
|
||||
bucket = %bucket,
|
||||
error = ?err,
|
||||
"Failed to load bucket object lock configuration"
|
||||
);
|
||||
return Err(s3_error!(InternalError, "Failed to get bucket ObjectLockConfiguration"));
|
||||
}
|
||||
};
|
||||
|
||||
let data = serialize_config(&config)?;
|
||||
let object_lock_config = String::from_utf8(data.clone()).map_err(to_internal_error)?;
|
||||
let updated_at = update_bucket_config_for_incarnation(bucket, OBJECT_LOCK_CONFIG, data, expected_incarnation_id)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
#[cfg(test)]
|
||||
tests::after_lock_write(bucket).await;
|
||||
|
||||
// Versioning is a separate write and must stay fenced to the original incarnation.
|
||||
let versioning_config = BucketVersioningSys::get(bucket).await.map_err(ApiError::from)?;
|
||||
if !versioning_config.enabled() {
|
||||
let versioning_config = PersistedVersioningConfiguration {
|
||||
status: Some("Enabled".to_owned()),
|
||||
..Default::default()
|
||||
};
|
||||
let data = serialize_config(&versioning_config)?;
|
||||
update_bucket_config_for_incarnation(bucket, BUCKET_VERSIONING_CONFIG, data, expected_incarnation_id)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
}
|
||||
|
||||
let peer_reload = match current_notification_system_for_context(self.context.as_deref()) {
|
||||
Some(notification_sys) => notification_sys.load_bucket_metadata(bucket).await,
|
||||
None => Ok(()),
|
||||
};
|
||||
#[cfg(test)]
|
||||
let peer_reload = peer_reload.and(tests::peer_reload_outcome(bucket).await);
|
||||
propagate_object_lock_peer_reload(peer_reload)?;
|
||||
|
||||
let site_result = site_replication_bucket_meta_hook(SRBucketMeta {
|
||||
bucket: bucket.to_owned(),
|
||||
r#type: "object-lock-config".to_owned(),
|
||||
object_lock_config: Some(object_lock_config),
|
||||
updated_at: Some(updated_at),
|
||||
api_version: Some(SITE_REPL_API_VERSION.to_owned()),
|
||||
..Default::default()
|
||||
})
|
||||
.await;
|
||||
#[cfg(test)]
|
||||
let site_result = site_result.and_then(|()| tests::site_hook_outcome());
|
||||
if let Err(err) = site_result {
|
||||
warn!(
|
||||
component = LOG_COMPONENT_APP,
|
||||
subsystem = LOG_SUBSYSTEM_OBJECT_LOCK,
|
||||
event = "put_object_lock_configuration",
|
||||
bucket = %bucket,
|
||||
result = "site_replication_hook_failed",
|
||||
error = ?err,
|
||||
"storage object lock state"
|
||||
);
|
||||
}
|
||||
|
||||
rustfs_scanner::record_dirty_usage_bucket_from_producer(
|
||||
bucket,
|
||||
rustfs_scanner::SegmentInvalidationProducerIdentity::BucketMetadata,
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// The stored Object Lock configuration as the gateway's answer holds it.
|
||||
///
|
||||
/// # Errors
|
||||
@@ -75,6 +274,365 @@ impl DefaultBucketUsecase {
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
use rustfs_gateway_types::persistence::PersistedObjectLockRule;
|
||||
|
||||
use crate::app::bucket_usecase::test_support::build_request;
|
||||
use crate::app::gating_test_env::{run_large_stack_test, shared_gating_ambient};
|
||||
use crate::app::runtime_sources::ServerContextSlot;
|
||||
use crate::app::storage_api::bucket_usecase::helper::MutationCompletionProbe;
|
||||
use crate::app::storage_api::owned_parity::authorize_bucket_config_mutation;
|
||||
use crate::site_replication::hooks::BUCKET_META_HOOK_CALLS;
|
||||
use http::Method;
|
||||
use rustfs_credentials::Credentials;
|
||||
use serial_test::serial;
|
||||
use std::cell::{Cell, RefCell};
|
||||
|
||||
const ORIGINAL: &[u8] = b"<ObjectLockConfiguration><ObjectLockEnabled>Enabled</ObjectLockEnabled></ObjectLockConfiguration>";
|
||||
const CHANGED: &[u8] = b"<ObjectLockConfiguration><ObjectLockEnabled>Enabled</ObjectLockEnabled><Rule><DefaultRetention><Days>3</Days><Mode>GOVERNANCE</Mode></DefaultRetention></Rule></ObjectLockConfiguration>";
|
||||
const SUSPENDED: &[u8] = b"<VersioningConfiguration><Status>Suspended</Status></VersioningConfiguration>";
|
||||
const ENABLED: &[u8] = b"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>";
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
enum Scenario {
|
||||
FirstFence,
|
||||
SecondFence,
|
||||
PeerFailure,
|
||||
SiteFailure,
|
||||
Success,
|
||||
}
|
||||
|
||||
type MetadataBytes = (Vec<u8>, Vec<u8>);
|
||||
|
||||
struct CompletionProbe {
|
||||
scenario: Scenario,
|
||||
store: Arc<ECStore>,
|
||||
observations: RefCell<Vec<MetadataBytes>>,
|
||||
recreated_scanner_generation: Cell<Option<u64>>,
|
||||
}
|
||||
|
||||
tokio::task_local! {
|
||||
static STATE: CompletionProbe;
|
||||
}
|
||||
|
||||
async fn stored_lock_and_versioning(bucket: &str) -> MetadataBytes {
|
||||
let metadata = metadata_sys::get_global_bucket_metadata_sys()
|
||||
.expect("metadata system")
|
||||
.read()
|
||||
.await
|
||||
.get_config_from_disk(bucket)
|
||||
.await
|
||||
.expect("read metadata from disk");
|
||||
(metadata.object_lock_config_xml, metadata.versioning_config_xml)
|
||||
}
|
||||
|
||||
async fn recreate(store: &Arc<ECStore>, bucket: &str) {
|
||||
store
|
||||
.delete_bucket(bucket, &DeleteBucketOptions::default())
|
||||
.await
|
||||
.expect("delete original bucket");
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("recreate bucket");
|
||||
seed(bucket).await;
|
||||
}
|
||||
|
||||
async fn seed(bucket: &str) {
|
||||
metadata_sys::update(bucket, OBJECT_LOCK_CONFIG, ORIGINAL.to_vec())
|
||||
.await
|
||||
.expect("store original lock");
|
||||
metadata_sys::update(bucket, BUCKET_VERSIONING_CONFIG, SUSPENDED.to_vec())
|
||||
.await
|
||||
.expect("store suspended versioning");
|
||||
}
|
||||
|
||||
pub(super) async fn after_lock_write(bucket: &str) {
|
||||
if let Ok((Scenario::SecondFence, store)) = STATE.try_with(|state| (state.scenario, Arc::clone(&state.store))) {
|
||||
let bytes = stored_lock_and_versioning(bucket).await;
|
||||
STATE.with(|state| state.observations.borrow_mut().push(bytes));
|
||||
recreate(&store, bucket).await;
|
||||
STATE.with(|state| {
|
||||
state
|
||||
.recreated_scanner_generation
|
||||
.set(rustfs_scanner::scanner_dirty_usage_bucket_generation(bucket));
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn peer_reload_outcome(bucket: &str) -> Result<(), StorageError> {
|
||||
let Ok(scenario) = STATE.try_with(|state| state.scenario) else {
|
||||
return Ok(());
|
||||
};
|
||||
let bytes = stored_lock_and_versioning(bucket).await;
|
||||
STATE.with(|state| state.observations.borrow_mut().push(bytes));
|
||||
if scenario == Scenario::PeerFailure {
|
||||
return Err(StorageError::other("test peer reload failure"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn site_hook_outcome() -> S3Result<()> {
|
||||
if matches!(STATE.try_with(|state| state.scenario), Ok(Scenario::SiteFailure)) {
|
||||
return Err(s3_error!(InternalError, "test site replication failure"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn put_lock_with_observed_completion(scenario: Scenario) {
|
||||
let context = shared_gating_ambient().await;
|
||||
let store = context.object_store();
|
||||
let bucket = format!(
|
||||
"lock-cfg-{}-{}",
|
||||
match scenario {
|
||||
Scenario::FirstFence => "first",
|
||||
Scenario::SecondFence => "second",
|
||||
Scenario::PeerFailure => "peer",
|
||||
Scenario::SiteFailure => "site",
|
||||
Scenario::Success => "success",
|
||||
},
|
||||
uuid::Uuid::new_v4().simple()
|
||||
);
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
seed(&bucket).await;
|
||||
|
||||
let server = ServerContextSlot::new();
|
||||
assert!(server.install(Arc::clone(&context)));
|
||||
let mut request = build_request((), Method::PUT);
|
||||
request.uri = format!("/{bucket}?object-lock").parse().expect("request URI");
|
||||
request.extensions.insert(Arc::clone(&server));
|
||||
request.extensions.insert(ReqInfo {
|
||||
cred: Some(Credentials {
|
||||
access_key: "bucket-lock-owner".to_owned(),
|
||||
..Default::default()
|
||||
}),
|
||||
is_owner: true,
|
||||
..Default::default()
|
||||
});
|
||||
let mut envelope = RequestEnvelope::from_s3s(&request);
|
||||
authorize_bucket_config_mutation(
|
||||
&server,
|
||||
&mut envelope,
|
||||
&bucket,
|
||||
Action::S3Action(S3Action::PutBucketObjectLockConfigurationAction),
|
||||
)
|
||||
.await
|
||||
.expect("capture the authorized bucket incarnation");
|
||||
|
||||
if scenario == Scenario::FirstFence {
|
||||
recreate(&store, &bucket).await;
|
||||
}
|
||||
let scanner_before = rustfs_scanner::scanner_dirty_usage_bucket_generation(&bucket);
|
||||
let audits = MutationCompletionProbe::install(&bucket);
|
||||
let usecase = DefaultBucketUsecase::with_context(Some(context));
|
||||
let config = rustfs_gateway_types::persistence::parse_object_lock(CHANGED).expect("fixture parses");
|
||||
let (result, hooks, observed, recreated_scanner_generation) = STATE
|
||||
.scope(
|
||||
CompletionProbe {
|
||||
scenario,
|
||||
store,
|
||||
observations: RefCell::default(),
|
||||
recreated_scanner_generation: Cell::default(),
|
||||
},
|
||||
async {
|
||||
let (result, hooks) = BUCKET_META_HOOK_CALLS
|
||||
.scope(RefCell::default(), async {
|
||||
let result = usecase
|
||||
.execute_put_object_lock_configuration(envelope, &bucket, Some(config))
|
||||
.await;
|
||||
(result, BUCKET_META_HOOK_CALLS.with(RefCell::take))
|
||||
})
|
||||
.await;
|
||||
let (observed, recreated_scanner_generation) =
|
||||
STATE.with(|state| (state.observations.take(), state.recreated_scanner_generation.get()));
|
||||
(result, hooks, observed, recreated_scanner_generation)
|
||||
},
|
||||
)
|
||||
.await;
|
||||
let stored = stored_lock_and_versioning(&bucket).await;
|
||||
let scanner_after = rustfs_scanner::scanner_dirty_usage_bucket_generation(&bucket);
|
||||
let audits_seen = audits.audits.lock().expect("audit entries");
|
||||
let seen: Vec<_> = audits_seen
|
||||
.iter()
|
||||
.map(|entry| (entry.api.name.as_deref(), entry.api.status_code))
|
||||
.collect();
|
||||
let batch = |lock: &[u8], version: &[u8]| (lock.to_vec(), version.to_vec());
|
||||
let (error, versioning, callbacks, status) = match scenario {
|
||||
Scenario::FirstFence => (Some(S3ErrorCode::NoSuchBucket), batch(ORIGINAL, SUSPENDED), vec![], 404),
|
||||
Scenario::SecondFence => (
|
||||
Some(S3ErrorCode::NoSuchBucket),
|
||||
batch(ORIGINAL, SUSPENDED),
|
||||
vec![batch(CHANGED, SUSPENDED)],
|
||||
404,
|
||||
),
|
||||
Scenario::PeerFailure => (
|
||||
Some(S3ErrorCode::InternalError),
|
||||
batch(CHANGED, ENABLED),
|
||||
vec![batch(CHANGED, ENABLED)],
|
||||
500,
|
||||
),
|
||||
Scenario::SiteFailure | Scenario::Success => (None, batch(CHANGED, ENABLED), vec![batch(CHANGED, ENABLED)], 200),
|
||||
};
|
||||
assert_eq!(
|
||||
(result.err().map(|error| error.code().clone()), stored, observed, seen),
|
||||
(
|
||||
error,
|
||||
versioning,
|
||||
callbacks,
|
||||
vec![(Some(S3Operation::PutObjectLockConfiguration.as_str()), Some(status))]
|
||||
),
|
||||
"{scenario:?}",
|
||||
);
|
||||
if scenario == Scenario::SiteFailure || scenario == Scenario::Success {
|
||||
let [hook] = hooks.as_slice() else { panic!("{scenario:?} must call site replication once") };
|
||||
assert_eq!(
|
||||
(&hook.bucket, hook.item_type.as_str(), hook.scanner_generation),
|
||||
(&bucket, "object-lock-config", scanner_before)
|
||||
);
|
||||
assert!(scanner_after > scanner_before, "{scenario:?} must invalidate scanner usage");
|
||||
} else {
|
||||
assert!(hooks.is_empty(), "{scenario:?} must stop before site replication");
|
||||
let expected_generation = if scenario == Scenario::SecondFence {
|
||||
recreated_scanner_generation
|
||||
} else {
|
||||
scanner_before
|
||||
};
|
||||
assert_eq!(expected_generation, scanner_after, "{scenario:?} must stop before scanner invalidation");
|
||||
}
|
||||
assert!(audits.events.lock().expect("event entries").is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn put_lock_configuration_fences_the_first_write() {
|
||||
run_large_stack_test("bucket-lock-first-write", || put_lock_with_observed_completion(Scenario::FirstFence));
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn put_lock_configuration_fences_the_versioning_write() {
|
||||
run_large_stack_test("bucket-lock-version-write", || put_lock_with_observed_completion(Scenario::SecondFence));
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn put_lock_configuration_propagates_peer_reload_failure() {
|
||||
run_large_stack_test("bucket-lock-peer-failure", || put_lock_with_observed_completion(Scenario::PeerFailure));
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn put_lock_configuration_continues_after_site_hook_failure() {
|
||||
run_large_stack_test("bucket-lock-site-failure", || put_lock_with_observed_completion(Scenario::SiteFailure));
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn put_lock_configuration_completes_durable_writes_before_callbacks() {
|
||||
run_large_stack_test("bucket-lock-callback-order", || put_lock_with_observed_completion(Scenario::Success));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_validate_object_lock_configuration_rejects_disabled_status() {
|
||||
let cfg = ObjectLockConfiguration {
|
||||
object_lock_enabled: Some("Disabled".to_owned()),
|
||||
rule: None,
|
||||
};
|
||||
|
||||
let err = validate_object_lock_configuration_input(&cfg).unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::MalformedXML);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_validate_object_lock_configuration_rejects_invalid_default_retention_mode() {
|
||||
let cfg = ObjectLockConfiguration {
|
||||
object_lock_enabled: Some("Enabled".to_owned()),
|
||||
rule: Some(PersistedObjectLockRule {
|
||||
default_retention: Some(PersistedDefaultRetention {
|
||||
mode: Some("abc".to_owned()),
|
||||
days: Some(1),
|
||||
years: None,
|
||||
}),
|
||||
}),
|
||||
};
|
||||
|
||||
let err = validate_object_lock_configuration_input(&cfg).unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::MalformedXML);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_validate_object_lock_configuration_rejects_days_and_years_together() {
|
||||
let cfg = ObjectLockConfiguration {
|
||||
object_lock_enabled: Some("Enabled".to_owned()),
|
||||
rule: Some(PersistedObjectLockRule {
|
||||
default_retention: Some(PersistedDefaultRetention {
|
||||
mode: Some("GOVERNANCE".to_owned()),
|
||||
days: Some(1),
|
||||
years: Some(1),
|
||||
}),
|
||||
}),
|
||||
};
|
||||
|
||||
let err = validate_object_lock_configuration_input(&cfg).unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::MalformedXML);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_validate_object_lock_configuration_rejects_missing_default_retention() {
|
||||
let cfg = ObjectLockConfiguration {
|
||||
object_lock_enabled: Some("Enabled".to_owned()),
|
||||
rule: Some(PersistedObjectLockRule { default_retention: None }),
|
||||
};
|
||||
|
||||
let err = validate_object_lock_configuration_input(&cfg).unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::MalformedXML);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_validate_object_lock_configuration_rejects_zero_days() {
|
||||
let cfg = ObjectLockConfiguration {
|
||||
object_lock_enabled: Some("Enabled".to_owned()),
|
||||
rule: Some(PersistedObjectLockRule {
|
||||
default_retention: Some(PersistedDefaultRetention {
|
||||
mode: Some("GOVERNANCE".to_owned()),
|
||||
days: Some(0),
|
||||
years: None,
|
||||
}),
|
||||
}),
|
||||
};
|
||||
|
||||
let err = validate_object_lock_configuration_input(&cfg).unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::Custom("InvalidRetentionPeriod".into()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_validate_object_lock_configuration_rejects_too_many_years() {
|
||||
let cfg = ObjectLockConfiguration {
|
||||
object_lock_enabled: Some("Enabled".to_owned()),
|
||||
rule: Some(PersistedObjectLockRule {
|
||||
default_retention: Some(PersistedDefaultRetention {
|
||||
mode: Some("COMPLIANCE".to_owned()),
|
||||
days: None,
|
||||
years: Some(101),
|
||||
}),
|
||||
}),
|
||||
};
|
||||
|
||||
let err = validate_object_lock_configuration_input(&cfg).unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::Custom("InvalidRetentionPeriod".into()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_object_lock_peer_reload_failure_is_returned_to_the_client() {
|
||||
let err = propagate_object_lock_peer_reload(Err(StorageError::other("peer reload failed")))
|
||||
.expect_err("peer reload failure must prevent a successful Object Lock update response");
|
||||
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
assert!(err.message().is_some_and(|message| message.contains("peer reload failed")));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_get_object_lock_configuration_returns_internal_error_when_store_uninitialized() {
|
||||
let input = gateway_dto::GetObjectLockConfigurationInput {
|
||||
|
||||
@@ -15,43 +15,42 @@
|
||||
//! Bucket policy and policy status use-case operations.
|
||||
|
||||
use super::*;
|
||||
use crate::app::storage_api::s3::dto::{
|
||||
DeleteBucketPolicyInput, DeleteBucketPolicyOutput, PutBucketPolicyInput, PutBucketPolicyOutput,
|
||||
};
|
||||
use crate::app::storage_api::s3::{S3Error, S3ErrorCode, S3Request, S3Response, S3Result, s3_error};
|
||||
use rustfs_policy::policy::Validator;
|
||||
use rustfs_s3_types::s3_error;
|
||||
|
||||
impl DefaultBucketUsecase {
|
||||
pub async fn execute_delete_bucket_policy(
|
||||
pub(crate) async fn execute_delete_bucket_policy(
|
||||
&self,
|
||||
req: S3Request<DeleteBucketPolicyInput>,
|
||||
) -> S3Result<S3Response<DeleteBucketPolicyOutput>> {
|
||||
record_s3_op(S3Operation::DeleteBucketPolicy);
|
||||
let expected_incarnation_id = bucket_config_mutation_incarnation(&req, &req.input.bucket)?;
|
||||
let request_context = req.extensions.get::<request_context::RequestContext>().cloned();
|
||||
let DeleteBucketPolicyInput { bucket, .. } = req.input;
|
||||
envelope: RequestEnvelope,
|
||||
bucket: &str,
|
||||
) -> rustfs_s3_types::S3Result<()> {
|
||||
RequestEnvelope::audit_only_operation(envelope, S3Operation::DeleteBucketPolicy, StatusCode::OK, |envelope| async move {
|
||||
let expected_incarnation_id = bucket_config_mutation_incarnation(&envelope, bucket)?;
|
||||
let request_context = envelope.request_context().cloned();
|
||||
|
||||
let Some(store) = self.object_store() else {
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
|
||||
};
|
||||
let Some(store) = self.object_store() else {
|
||||
return Err(rustfs_s3_types::s3_error!(InternalError, "Not init"));
|
||||
};
|
||||
|
||||
store
|
||||
.get_bucket_info(&bucket, &BucketOptions::default())
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
store
|
||||
.get_bucket_info(bucket, &BucketOptions::default())
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
delete_bucket_config_for_incarnation(&bucket, BUCKET_POLICY_CONFIG, expected_incarnation_id)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
delete_bucket_config_for_incarnation(bucket, BUCKET_POLICY_CONFIG, expected_incarnation_id)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
notify_bucket_metadata_reload(bucket.clone(), "delete bucket policy", request_context, false).await;
|
||||
notify_bucket_metadata_reload(bucket.to_owned(), "delete bucket policy", request_context, false).await;
|
||||
|
||||
let item = sr_bucket_meta_item(bucket.clone(), "policy");
|
||||
if let Err(err) = site_replication_bucket_meta_hook(item).await {
|
||||
warn!(bucket = %bucket, error = ?err, "site replication bucket policy delete hook failed");
|
||||
}
|
||||
let item = sr_bucket_meta_item(bucket.to_owned(), "policy");
|
||||
if let Err(err) = site_replication_bucket_meta_hook(item).await {
|
||||
warn!(bucket = %bucket, error = ?err, "site replication bucket policy delete hook failed");
|
||||
}
|
||||
|
||||
Ok(S3Response::new(DeleteBucketPolicyOutput {}))
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
/// The stored policy document, verbatim.
|
||||
@@ -166,75 +165,78 @@ impl DefaultBucketUsecase {
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn execute_put_bucket_policy(
|
||||
pub(crate) async fn execute_put_bucket_policy(
|
||||
&self,
|
||||
req: S3Request<PutBucketPolicyInput>,
|
||||
) -> S3Result<S3Response<PutBucketPolicyOutput>> {
|
||||
record_s3_op(S3Operation::PutBucketPolicy);
|
||||
let expected_incarnation_id = bucket_config_mutation_incarnation(&req, &req.input.bucket)?;
|
||||
let request_context = req.extensions.get::<request_context::RequestContext>().cloned();
|
||||
let PutBucketPolicyInput { bucket, policy, .. } = req.input;
|
||||
envelope: RequestEnvelope,
|
||||
bucket: &str,
|
||||
policy: &str,
|
||||
) -> rustfs_s3_types::S3Result<()> {
|
||||
RequestEnvelope::audit_only_operation(envelope, S3Operation::PutBucketPolicy, StatusCode::OK, |envelope| async move {
|
||||
let expected_incarnation_id = bucket_config_mutation_incarnation(&envelope, bucket)?;
|
||||
let request_context = envelope.request_context().cloned();
|
||||
|
||||
let Some(store) = self.object_store() else {
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
|
||||
};
|
||||
let Some(store) = self.object_store() else {
|
||||
return Err(s3_error!(InternalError, "Not init"));
|
||||
};
|
||||
|
||||
store
|
||||
.get_bucket_info(&bucket, &BucketOptions::default())
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
store
|
||||
.get_bucket_info(bucket, &BucketOptions::default())
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
let cfg: BucketPolicy =
|
||||
serde_json::from_str(&policy).map_err(|e| s3_error!(InvalidArgument, "parse policy failed {:?}", e))?;
|
||||
let cfg: BucketPolicy =
|
||||
serde_json::from_str(policy).map_err(|e| s3_error!(InvalidArgument, "parse policy failed {:?}", e))?;
|
||||
|
||||
if let Err(err) = cfg.is_valid() {
|
||||
warn!("put_bucket_policy err input {:?}, {:?}", &policy, err);
|
||||
return Err(s3_error!(MalformedPolicy));
|
||||
}
|
||||
if cfg.is_valid().is_err() {
|
||||
return Err(s3_error!(MalformedPolicy));
|
||||
}
|
||||
|
||||
let is_public_policy = cfg
|
||||
.statements
|
||||
.iter()
|
||||
.any(|statement| matches!(statement.effect, Effect::Allow) && statement.principal.is_match("*"));
|
||||
let is_public_policy = cfg
|
||||
.statements
|
||||
.iter()
|
||||
.any(|statement| matches!(statement.effect, Effect::Allow) && statement.principal.is_match("*"));
|
||||
|
||||
if is_public_policy {
|
||||
match metadata_sys::get_public_access_block_config(&bucket).await {
|
||||
Ok((config, _)) => {
|
||||
if config.block_public_policy.unwrap_or(false) {
|
||||
return Err(s3_error!(AccessDenied, "Access Denied"));
|
||||
if is_public_policy {
|
||||
match metadata_sys::get_public_access_block_config(bucket).await {
|
||||
Ok((config, _)) => {
|
||||
if config.block_public_policy.unwrap_or(false) {
|
||||
return Err(s3_error!(AccessDenied, "Access Denied"));
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(err) => {
|
||||
if err != StorageError::ConfigNotFound {
|
||||
warn!(
|
||||
component = LOG_COMPONENT_APP,
|
||||
subsystem = LOG_SUBSYSTEM_BUCKET,
|
||||
event = "bucket_public_access_block_load_failed",
|
||||
bucket = %bucket,
|
||||
error = ?err,
|
||||
"Failed to load bucket public access block configuration"
|
||||
);
|
||||
return Err(ApiError::from(err).into());
|
||||
Err(err) => {
|
||||
if err != StorageError::ConfigNotFound {
|
||||
warn!(
|
||||
component = LOG_COMPONENT_APP,
|
||||
subsystem = LOG_SUBSYSTEM_BUCKET,
|
||||
event = "bucket_public_access_block_load_failed",
|
||||
bucket = %bucket,
|
||||
error = ?err,
|
||||
"Failed to load bucket public access block configuration"
|
||||
);
|
||||
return Err(ApiError::from(err).into());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let data = policy.as_bytes().to_vec();
|
||||
let data = policy.as_bytes().to_vec();
|
||||
|
||||
update_bucket_config_for_incarnation(&bucket, BUCKET_POLICY_CONFIG, data, expected_incarnation_id)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
update_bucket_config_for_incarnation(bucket, BUCKET_POLICY_CONFIG, data, expected_incarnation_id)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
notify_bucket_metadata_reload(bucket.clone(), "put bucket policy", request_context, false).await;
|
||||
notify_bucket_metadata_reload(bucket.to_owned(), "put bucket policy", request_context, false).await;
|
||||
|
||||
let mut item = sr_bucket_meta_item(bucket.clone(), "policy");
|
||||
item.policy = Some(serde_json::from_str(&policy).map_err(|e| s3_error!(InvalidArgument, "parse policy failed {:?}", e))?);
|
||||
if let Err(err) = site_replication_bucket_meta_hook(item).await {
|
||||
warn!(bucket = %bucket, error = ?err, "site replication bucket policy hook failed");
|
||||
}
|
||||
let mut item = sr_bucket_meta_item(bucket.to_owned(), "policy");
|
||||
item.policy =
|
||||
Some(serde_json::from_str(policy).map_err(|e| s3_error!(InvalidArgument, "parse policy failed {:?}", e))?);
|
||||
if let Err(err) = site_replication_bucket_meta_hook(item).await {
|
||||
warn!(bucket = %bucket, error = ?err, "site replication bucket policy hook failed");
|
||||
}
|
||||
|
||||
Ok(S3Response::new(PutBucketPolicyOutput {}))
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
@@ -242,6 +244,8 @@ impl DefaultBucketUsecase {
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::app::bucket_usecase::test_support::{build_request, s3_op_total};
|
||||
use crate::app::storage_api::s3::S3ErrorCode;
|
||||
use crate::app::storage_api::s3::dto::{DeleteBucketPolicyInput, PutBucketPolicyInput};
|
||||
use http::Method;
|
||||
|
||||
#[tokio::test]
|
||||
@@ -252,10 +256,12 @@ mod tests {
|
||||
.unwrap();
|
||||
|
||||
let req = build_request(input, Method::DELETE);
|
||||
let usecase = DefaultBucketUsecase::without_context();
|
||||
let fs = crate::storage::ecfs::FS::new();
|
||||
let before = s3_op_total(S3Operation::DeleteBucketPolicy);
|
||||
|
||||
let err = usecase.execute_delete_bucket_policy(req).await.unwrap_err();
|
||||
let err = crate::app::storage_api::s3::S3::delete_bucket_policy(&fs, req)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
assert_eq!(s3_op_total(S3Operation::DeleteBucketPolicy), before + 1);
|
||||
}
|
||||
@@ -301,10 +307,12 @@ mod tests {
|
||||
.unwrap();
|
||||
|
||||
let req = build_request(input, Method::PUT);
|
||||
let usecase = DefaultBucketUsecase::without_context();
|
||||
let fs = crate::storage::ecfs::FS::new();
|
||||
let before = s3_op_total(S3Operation::PutBucketPolicy);
|
||||
|
||||
let err = usecase.execute_put_bucket_policy(req).await.unwrap_err();
|
||||
let err = crate::app::storage_api::s3::S3::put_bucket_policy(&fs, req)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
assert_eq!(s3_op_total(S3Operation::PutBucketPolicy), before + 1);
|
||||
}
|
||||
|
||||
@@ -15,100 +15,97 @@
|
||||
//! Bucket public access block use-case operations.
|
||||
|
||||
use super::*;
|
||||
use crate::app::storage_api::s3::dto::{
|
||||
DeletePublicAccessBlockInput, DeletePublicAccessBlockOutput, GetPublicAccessBlockInput, PutPublicAccessBlockInput,
|
||||
PutPublicAccessBlockOutput,
|
||||
};
|
||||
use crate::app::storage_api::s3::{S3Error, S3ErrorCode, S3Request, S3Response, S3Result};
|
||||
use rustfs_s3_types::{S3Error, S3ErrorCode, S3Result};
|
||||
|
||||
impl DefaultBucketUsecase {
|
||||
#[instrument(level = "debug", skip(self, req))]
|
||||
pub async fn execute_delete_public_access_block(
|
||||
&self,
|
||||
req: S3Request<DeletePublicAccessBlockInput>,
|
||||
) -> S3Result<S3Response<DeletePublicAccessBlockOutput>> {
|
||||
let expected_incarnation_id = bucket_config_mutation_incarnation(&req, &req.input.bucket)?;
|
||||
let request_context = req.extensions.get::<request_context::RequestContext>().cloned();
|
||||
let DeletePublicAccessBlockInput { bucket, .. } = req.input;
|
||||
#[instrument(level = "debug", skip(self, envelope))]
|
||||
pub(crate) async fn execute_delete_public_access_block(&self, envelope: RequestEnvelope, bucket: &str) -> S3Result<()> {
|
||||
RequestEnvelope::audit_only_operation(
|
||||
envelope,
|
||||
S3Operation::DeletePublicAccessBlock,
|
||||
StatusCode::NO_CONTENT,
|
||||
|envelope| async move {
|
||||
let expected_incarnation_id = bucket_config_mutation_incarnation(&envelope, bucket)?;
|
||||
let request_context = envelope.request_context().cloned();
|
||||
|
||||
let Some(store) = self.object_store() else {
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
|
||||
};
|
||||
let Some(store) = self.object_store() else {
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init"));
|
||||
};
|
||||
store
|
||||
.get_bucket_info(bucket, &BucketOptions::default())
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
store
|
||||
.get_bucket_info(&bucket, &BucketOptions::default())
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
delete_bucket_config_for_incarnation(&bucket, BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG, expected_incarnation_id)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
notify_bucket_metadata_reload(bucket.clone(), "delete public access block", request_context, false).await;
|
||||
|
||||
Ok(S3Response::with_status(DeletePublicAccessBlockOutput::default(), StatusCode::NO_CONTENT))
|
||||
delete_bucket_config_for_incarnation(bucket, BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG, expected_incarnation_id)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
notify_bucket_metadata_reload(bucket.to_owned(), "delete public access block", request_context, false).await;
|
||||
Ok(())
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self, req))]
|
||||
pub async fn execute_get_public_access_block(
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
pub(crate) async fn execute_get_public_access_block(
|
||||
&self,
|
||||
req: S3Request<GetPublicAccessBlockInput>,
|
||||
bucket: &str,
|
||||
) -> S3Result<PersistedPublicAccessBlockConfiguration> {
|
||||
let GetPublicAccessBlockInput { bucket, .. } = req.input;
|
||||
|
||||
let Some(store) = self.object_store() else {
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init"));
|
||||
};
|
||||
|
||||
store
|
||||
.get_bucket_info(&bucket, &BucketOptions::default())
|
||||
.get_bucket_info(bucket, &BucketOptions::default())
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
let config = match metadata_sys::get_public_access_block_config(&bucket).await {
|
||||
Ok((config, _)) => config,
|
||||
Err(err) => {
|
||||
if err == StorageError::ConfigNotFound {
|
||||
return Err(S3Error::with_message(
|
||||
S3ErrorCode::Custom("NoSuchPublicAccessBlockConfiguration".into()),
|
||||
"Public access block configuration does not exist".to_string(),
|
||||
));
|
||||
}
|
||||
return Err(ApiError::from(err).into());
|
||||
match metadata_sys::get_public_access_block_config(bucket).await {
|
||||
Ok((config, _)) => Ok(config),
|
||||
Err(StorageError::ConfigNotFound) => {
|
||||
let mut error = S3Error::with_message(
|
||||
S3ErrorCode::Custom("NoSuchPublicAccessBlockConfiguration".into()),
|
||||
"Public access block configuration does not exist",
|
||||
);
|
||||
// The legacy edge used the custom code with the server-error status.
|
||||
error.set_status_code(StatusCode::INTERNAL_SERVER_ERROR);
|
||||
Err(error)
|
||||
}
|
||||
};
|
||||
|
||||
Ok(config)
|
||||
Err(err) => Err(ApiError::from(err).into()),
|
||||
}
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self, req))]
|
||||
pub async fn execute_put_public_access_block(
|
||||
#[instrument(level = "debug", skip(self, envelope, prepared))]
|
||||
pub(crate) async fn execute_put_public_access_block(
|
||||
&self,
|
||||
req: S3Request<PutPublicAccessBlockInput>,
|
||||
envelope: RequestEnvelope,
|
||||
bucket: &str,
|
||||
prepared: S3Result<PersistedPublicAccessBlockConfiguration>,
|
||||
) -> S3Result<S3Response<PutPublicAccessBlockOutput>> {
|
||||
let expected_incarnation_id = bucket_config_mutation_incarnation(&req, &req.input.bucket)?;
|
||||
let request_context = req.extensions.get::<request_context::RequestContext>().cloned();
|
||||
let PutPublicAccessBlockInput { bucket, .. } = req.input;
|
||||
) -> S3Result<()> {
|
||||
RequestEnvelope::audit_only_operation(
|
||||
envelope,
|
||||
S3Operation::PutPublicAccessBlock,
|
||||
StatusCode::OK,
|
||||
|envelope| async move {
|
||||
let expected_incarnation_id = bucket_config_mutation_incarnation(&envelope, bucket)?;
|
||||
let request_context = envelope.request_context().cloned();
|
||||
|
||||
let Some(store) = self.object_store() else {
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
|
||||
};
|
||||
let Some(store) = self.object_store() else {
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init"));
|
||||
};
|
||||
store
|
||||
.get_bucket_info(bucket, &BucketOptions::default())
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
store
|
||||
.get_bucket_info(&bucket, &BucketOptions::default())
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
let public_access_block_configuration = prepared?;
|
||||
let data = serialize_config(&public_access_block_configuration)?;
|
||||
update_bucket_config_for_incarnation(&bucket, BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG, data, expected_incarnation_id)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
notify_bucket_metadata_reload(bucket.clone(), "put public access block", request_context, false).await;
|
||||
|
||||
Ok(S3Response::new(PutPublicAccessBlockOutput::default()))
|
||||
let data = serialize_config(&prepared?)?;
|
||||
update_bucket_config_for_incarnation(bucket, BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG, data, expected_incarnation_id)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
notify_bucket_metadata_reload(bucket.to_owned(), "put public access block", request_context, false).await;
|
||||
Ok(())
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
@@ -116,50 +113,193 @@ impl DefaultBucketUsecase {
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::app::bucket_usecase::test_support::build_request;
|
||||
use crate::app::storage_api::s3::dto::PublicAccessBlockConfiguration;
|
||||
use crate::app::gating_test_env::{run_large_stack_test, shared_gating_ambient};
|
||||
use crate::app::runtime_sources::ServerContextSlot;
|
||||
use crate::app::storage_api::bucket_usecase::helper::MutationCompletionProbe;
|
||||
use crate::app::storage_api::owned_parity::authorize_bucket_config_mutation;
|
||||
use http::Method;
|
||||
use rustfs_credentials::Credentials;
|
||||
use rustfs_policy::policy::action::{Action, S3Action};
|
||||
use serial_test::serial;
|
||||
use std::sync::Mutex;
|
||||
|
||||
const ORIGINAL_XML: &[u8] =
|
||||
b"<PublicAccessBlockConfiguration><BlockPublicPolicy>true</BlockPublicPolicy></PublicAccessBlockConfiguration>";
|
||||
|
||||
async fn public_access_block_mutation(put: bool, stale: bool) {
|
||||
let (method, action, operation, reload, status) = if put {
|
||||
(
|
||||
Method::PUT,
|
||||
S3Action::PutBucketPublicAccessBlockAction,
|
||||
S3Operation::PutPublicAccessBlock,
|
||||
"put public access block",
|
||||
200,
|
||||
)
|
||||
} else {
|
||||
(
|
||||
Method::DELETE,
|
||||
S3Action::DeleteBucketPublicAccessBlockAction,
|
||||
S3Operation::DeletePublicAccessBlock,
|
||||
"delete public access block",
|
||||
204,
|
||||
)
|
||||
};
|
||||
let context = shared_gating_ambient().await;
|
||||
let store = context.object_store();
|
||||
let bucket = format!("public-access-block-{}", uuid::Uuid::new_v4().simple());
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
|
||||
let server = ServerContextSlot::new();
|
||||
assert!(server.install(Arc::clone(&context)));
|
||||
let mut request = build_request((), method);
|
||||
request.uri = format!("/{bucket}?publicAccessBlock").parse().expect("request URI");
|
||||
request.extensions.insert(Arc::clone(&server));
|
||||
request.extensions.insert(ReqInfo {
|
||||
cred: Some(Credentials {
|
||||
access_key: "public-access-block-owner".to_owned(),
|
||||
..Default::default()
|
||||
}),
|
||||
is_owner: true,
|
||||
..Default::default()
|
||||
});
|
||||
request.extensions.insert(request_context::RequestContext {
|
||||
request_id: "public-access-block-fixture".to_owned(),
|
||||
x_amz_request_id: "public-access-block-fixture".to_owned(),
|
||||
trace_id: None,
|
||||
span_id: None,
|
||||
start_time: std::time::Instant::now(),
|
||||
});
|
||||
let mut envelope = RequestEnvelope::from_s3s(&request);
|
||||
authorize_bucket_config_mutation(&server, &mut envelope, &bucket, Action::S3Action(action))
|
||||
.await
|
||||
.expect("bucket owner authorization installs the incarnation snapshot");
|
||||
|
||||
if stale {
|
||||
store
|
||||
.delete_bucket(&bucket, &DeleteBucketOptions::default())
|
||||
.await
|
||||
.expect("delete authorized bucket");
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("recreate bucket");
|
||||
}
|
||||
metadata_sys::update(&bucket, BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG, ORIGINAL_XML.to_vec())
|
||||
.await
|
||||
.expect("seed config");
|
||||
|
||||
let usecase = DefaultBucketUsecase::with_context(Some(context));
|
||||
let audits = MutationCompletionProbe::install(&bucket);
|
||||
let reloads = Arc::new(Mutex::new(Vec::new()));
|
||||
let record = Arc::clone(&reloads);
|
||||
let result = BUCKET_METADATA_RELOAD_PROBE
|
||||
.scope(
|
||||
(
|
||||
bucket.clone(),
|
||||
Box::new(move |completed, label, maintenance| {
|
||||
record.lock().expect("reload observer").push((completed, label, maintenance));
|
||||
}),
|
||||
),
|
||||
async {
|
||||
if put {
|
||||
let config = PersistedPublicAccessBlockConfiguration {
|
||||
block_public_policy: Some(false),
|
||||
..Default::default()
|
||||
};
|
||||
usecase.execute_put_public_access_block(envelope, &bucket, Ok(config)).await
|
||||
} else {
|
||||
usecase.execute_delete_public_access_block(envelope, &bucket).await
|
||||
}
|
||||
},
|
||||
)
|
||||
.await;
|
||||
let stored = metadata_sys::get_global_bucket_metadata_sys()
|
||||
.expect("initialized metadata system")
|
||||
.read()
|
||||
.await
|
||||
.get_config_from_disk(&bucket)
|
||||
.await
|
||||
.expect("persisted bucket metadata")
|
||||
.public_access_block_config_xml;
|
||||
let reloads = reloads.lock().expect("recorded reloads");
|
||||
if stale {
|
||||
assert_eq!(
|
||||
result.expect_err("stale authorization must not mutate replacement").code(),
|
||||
&S3ErrorCode::NoSuchBucket
|
||||
);
|
||||
assert_eq!(stored, ORIGINAL_XML);
|
||||
assert!(reloads.is_empty());
|
||||
} else {
|
||||
result.expect("authorized mutation");
|
||||
if put {
|
||||
assert_eq!(stored, b"<PublicAccessBlockConfiguration><BlockPublicPolicy>false</BlockPublicPolicy></PublicAccessBlockConfiguration>");
|
||||
} else {
|
||||
assert!(stored.is_empty());
|
||||
}
|
||||
assert_eq!(
|
||||
*reloads,
|
||||
[(false, reload, false), (true, reload, false)],
|
||||
"metadata reload completes without scanner maintenance"
|
||||
);
|
||||
let entries = audits.audits.lock().expect("audit entries");
|
||||
assert_eq!(entries.len(), 1);
|
||||
assert_eq!(entries[0].api.name.as_deref(), Some(operation.as_str()));
|
||||
assert_eq!(entries[0].api.status_code, Some(status));
|
||||
assert!(audits.events.lock().expect("event entries").is_empty());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn put_public_access_block_rejects_stale_authorization() {
|
||||
run_large_stack_test("public-access-block-put-stale", || public_access_block_mutation(true, true));
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn delete_public_access_block_rejects_stale_authorization() {
|
||||
run_large_stack_test("public-access-block-delete-stale", || public_access_block_mutation(false, true));
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn put_public_access_block_reloads_and_audits() {
|
||||
run_large_stack_test("public-access-block-put-completion", || public_access_block_mutation(true, false));
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn delete_public_access_block_reloads_and_audits() {
|
||||
run_large_stack_test("public-access-block-delete-completion", || public_access_block_mutation(false, false));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_delete_public_access_block_returns_internal_error_when_store_uninitialized() {
|
||||
let input = DeletePublicAccessBlockInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let req = build_request(input, Method::DELETE);
|
||||
let envelope = RequestEnvelope::from_s3s(&build_request((), Method::DELETE));
|
||||
let usecase = DefaultBucketUsecase::without_context();
|
||||
|
||||
let err = usecase.execute_delete_public_access_block(req).await.unwrap_err();
|
||||
let err = usecase
|
||||
.execute_delete_public_access_block(envelope, "test-bucket")
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_get_public_access_block_returns_internal_error_when_store_uninitialized() {
|
||||
let input = GetPublicAccessBlockInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let req = build_request(input, Method::GET);
|
||||
let usecase = DefaultBucketUsecase::without_context();
|
||||
|
||||
let err = usecase.execute_get_public_access_block(req).await.unwrap_err();
|
||||
let err = usecase.execute_get_public_access_block("test-bucket").await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_put_public_access_block_returns_internal_error_when_store_uninitialized() {
|
||||
let input = PutPublicAccessBlockInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.public_access_block_configuration(PublicAccessBlockConfiguration::default())
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let req = build_request(input, Method::PUT);
|
||||
let envelope = RequestEnvelope::from_s3s(&build_request((), Method::PUT));
|
||||
let usecase = DefaultBucketUsecase::without_context();
|
||||
|
||||
let err = usecase
|
||||
.execute_put_public_access_block(req, Ok(PersistedPublicAccessBlockConfiguration::default()))
|
||||
.execute_put_public_access_block(envelope, "test-bucket", Ok(PersistedPublicAccessBlockConfiguration::default()))
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
|
||||
@@ -15,11 +15,8 @@
|
||||
//! Bucket replication configuration use-case operations.
|
||||
|
||||
use super::*;
|
||||
use crate::app::storage_api::s3::dto::{
|
||||
DeleteBucketReplicationInput, DeleteBucketReplicationOutput, PutBucketReplicationInput, PutBucketReplicationOutput,
|
||||
};
|
||||
use crate::app::storage_api::s3::{S3Error, S3ErrorCode, S3Request, S3Response, S3Result, s3_error};
|
||||
use rustfs_gateway_types::persistence::parse_replication_dto;
|
||||
use rustfs_s3_types::{S3Error, S3ErrorCode, S3Result, s3_error};
|
||||
|
||||
fn validate_replication_config_targets(targets: &BucketTargets, config: &PersistedReplicationConfiguration) -> S3Result<()> {
|
||||
let configured_arns = targets
|
||||
@@ -168,6 +165,15 @@ async fn write_replication_targets_after_config_delete(
|
||||
removed: usize,
|
||||
expected_incarnation_id: Option<uuid::Uuid>,
|
||||
) -> S3Result<()> {
|
||||
#[cfg(test)]
|
||||
if let Some(fail) = tests::TARGET_CLEANUP_FAILURE
|
||||
.try_with(std::cell::RefCell::take)
|
||||
.ok()
|
||||
.flatten()
|
||||
{
|
||||
fail.await?;
|
||||
}
|
||||
|
||||
let json_targets = serde_json::to_vec(&targets).map_err(to_internal_error)?;
|
||||
update_bucket_config_for_incarnation(bucket, BUCKET_TARGETS_FILE, json_targets, expected_incarnation_id)
|
||||
.await
|
||||
@@ -210,38 +216,56 @@ async fn restore_replication_config_after_target_cleanup_failure(
|
||||
}
|
||||
|
||||
impl DefaultBucketUsecase {
|
||||
/// `site_peers` is the set of remote site-replication peer deployment ids
|
||||
/// (empty when site replication is disabled). The interface layer reads it
|
||||
/// from the persisted state and fails closed on a read error, so this
|
||||
/// usecase stays a pure function of its inputs (layer rule: app never
|
||||
/// imports interface).
|
||||
pub async fn execute_delete_bucket_replication(
|
||||
/// Removes operator rules and targets unique to those rules.
|
||||
///
|
||||
/// Preparation runs inside the audit scope and resolves the peer contract
|
||||
/// before any bucket lookup or mutation.
|
||||
pub(crate) async fn execute_delete_bucket_replication(
|
||||
&self,
|
||||
req: S3Request<DeleteBucketReplicationInput>,
|
||||
envelope: RequestEnvelope,
|
||||
bucket: &str,
|
||||
prepare: impl Future<Output = S3Result<(HashSet<String>, OperatorRuleContract)>> + Send,
|
||||
) -> S3Result<()> {
|
||||
// Preserve the success status recorded by the legacy audit for an empty response.
|
||||
RequestEnvelope::audit_only_operation(
|
||||
envelope,
|
||||
S3Operation::DeleteBucketReplication,
|
||||
StatusCode::OK,
|
||||
|envelope| async move {
|
||||
let (site_peers, contract) = prepare.await?;
|
||||
self.delete_bucket_replication(envelope, bucket, site_peers, contract).await
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn delete_bucket_replication(
|
||||
&self,
|
||||
envelope: RequestEnvelope,
|
||||
bucket: &str,
|
||||
site_peers: HashSet<String>,
|
||||
contract: OperatorRuleContract,
|
||||
) -> S3Result<S3Response<DeleteBucketReplicationOutput>> {
|
||||
let expected_incarnation_id = bucket_config_mutation_incarnation(&req, &req.input.bucket)?;
|
||||
let request_context = req.extensions.get::<request_context::RequestContext>().cloned();
|
||||
let DeleteBucketReplicationInput { bucket, .. } = req.input;
|
||||
) -> S3Result<()> {
|
||||
let expected_incarnation_id = bucket_config_mutation_incarnation(&envelope, bucket)?;
|
||||
let request_context = envelope.request_context().cloned();
|
||||
|
||||
let Some(store) = self.object_store() else {
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init"));
|
||||
};
|
||||
|
||||
store
|
||||
.get_bucket_info(&bucket, &BucketOptions::default())
|
||||
.get_bucket_info(bucket, &BucketOptions::default())
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
let targets_guard = lock_bucket_targets_metadata(&bucket).await;
|
||||
let replication_config = match metadata_sys::get_replication_config(&bucket).await {
|
||||
let targets_guard = lock_bucket_targets_metadata(bucket).await;
|
||||
let replication_config = match metadata_sys::get_replication_config(bucket).await {
|
||||
Ok((config, _)) => Some(config),
|
||||
Err(StorageError::ConfigNotFound) => None,
|
||||
Err(err) => return Err(ApiError::from(err).into()),
|
||||
};
|
||||
let (remaining_config, updated_targets) = if let Some(config) = replication_config.as_ref() {
|
||||
let (remaining, removable_arns) = split_replication_config_for_user_delete(config.clone(), &site_peers, contract);
|
||||
let targets = replication_targets_without_arns(&bucket, &removable_arns).await?;
|
||||
let targets = replication_targets_without_arns(bucket, &removable_arns).await?;
|
||||
(remaining, targets)
|
||||
} else {
|
||||
(None, None)
|
||||
@@ -252,23 +276,23 @@ impl DefaultBucketUsecase {
|
||||
// S3 delete (issue #1948); only the operator-authored rules go.
|
||||
Some(remaining) => {
|
||||
let data = serialize_config(&remaining)?;
|
||||
update_bucket_config_for_incarnation(&bucket, BUCKET_REPLICATION_CONFIG, data, expected_incarnation_id)
|
||||
update_bucket_config_for_incarnation(bucket, BUCKET_REPLICATION_CONFIG, data, expected_incarnation_id)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
}
|
||||
None => {
|
||||
delete_bucket_config_for_incarnation(&bucket, BUCKET_REPLICATION_CONFIG, expected_incarnation_id)
|
||||
delete_bucket_config_for_incarnation(bucket, BUCKET_REPLICATION_CONFIG, expected_incarnation_id)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
}
|
||||
}
|
||||
if let Some((targets, removed)) = updated_targets
|
||||
&& let Err(err) =
|
||||
write_replication_targets_after_config_delete(&bucket, &targets, removed, expected_incarnation_id).await
|
||||
write_replication_targets_after_config_delete(bucket, &targets, removed, expected_incarnation_id).await
|
||||
{
|
||||
if let Some(config) = replication_config.as_ref() {
|
||||
return Err(restore_replication_config_after_target_cleanup_failure(
|
||||
&bucket,
|
||||
bucket,
|
||||
config,
|
||||
err,
|
||||
expected_incarnation_id,
|
||||
@@ -279,16 +303,16 @@ impl DefaultBucketUsecase {
|
||||
}
|
||||
drop(targets_guard);
|
||||
|
||||
notify_bucket_metadata_reload(bucket.clone(), "delete bucket replication", request_context, true).await;
|
||||
notify_bucket_metadata_reload(bucket.to_owned(), "delete bucket replication", request_context, true).await;
|
||||
|
||||
let item = sr_bucket_meta_item(bucket.clone(), "replication-config");
|
||||
let item = sr_bucket_meta_item(bucket.to_owned(), "replication-config");
|
||||
if let Err(err) = site_replication_bucket_meta_hook(item).await {
|
||||
warn!(bucket = %bucket, error = ?err, "site replication bucket replication-config delete hook failed");
|
||||
}
|
||||
|
||||
info!(bucket = %bucket, "deleted bucket replication config");
|
||||
|
||||
Ok(S3Response::new(DeleteBucketReplicationOutput::default()))
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// The stored replication configuration as the gateway's answer holds it.
|
||||
@@ -335,36 +359,52 @@ impl DefaultBucketUsecase {
|
||||
})
|
||||
}
|
||||
|
||||
/// See [`Self::execute_delete_bucket_replication`] for `site_peers` and
|
||||
/// `contract`. `replication_configuration` is the request body in the
|
||||
/// engine's shape: the s3s edge (`storage::ecfs`) converts it, and this use
|
||||
/// case never reads `req.input.replication_configuration`.
|
||||
pub async fn execute_put_bucket_replication(
|
||||
/// Replaces operator rules after the interface resolves the peer contract.
|
||||
pub(crate) async fn execute_put_bucket_replication(
|
||||
&self,
|
||||
req: S3Request<PutBucketReplicationInput>,
|
||||
envelope: RequestEnvelope,
|
||||
bucket: &str,
|
||||
prepare: impl Future<Output = S3Result<(PersistedReplicationConfiguration, HashSet<String>, OperatorRuleContract)>> + Send,
|
||||
) -> S3Result<()> {
|
||||
RequestEnvelope::audit_only_operation(
|
||||
envelope,
|
||||
S3Operation::PutBucketReplication,
|
||||
StatusCode::OK,
|
||||
|envelope| async move {
|
||||
let (replication_configuration, site_peers, contract) = prepare.await?;
|
||||
self.put_bucket_replication(envelope, bucket, replication_configuration, site_peers, contract)
|
||||
.await
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn put_bucket_replication(
|
||||
&self,
|
||||
envelope: RequestEnvelope,
|
||||
bucket: &str,
|
||||
replication_configuration: PersistedReplicationConfiguration,
|
||||
site_peers: HashSet<String>,
|
||||
contract: OperatorRuleContract,
|
||||
) -> S3Result<S3Response<PutBucketReplicationOutput>> {
|
||||
let expected_incarnation_id = bucket_config_mutation_incarnation(&req, &req.input.bucket)?;
|
||||
let request_context = req.extensions.get::<request_context::RequestContext>().cloned();
|
||||
let PutBucketReplicationInput { bucket, .. } = req.input;
|
||||
) -> S3Result<()> {
|
||||
let expected_incarnation_id = bucket_config_mutation_incarnation(&envelope, bucket)?;
|
||||
let request_context = envelope.request_context().cloned();
|
||||
info!(bucket = %bucket, "updating bucket replication config");
|
||||
|
||||
validate_replication_config_capabilities(&replication_configuration)?;
|
||||
|
||||
let Some(store) = self.object_store() else {
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init"));
|
||||
};
|
||||
|
||||
store
|
||||
.get_bucket_info(&bucket, &BucketOptions::default())
|
||||
.get_bucket_info(bucket, &BucketOptions::default())
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
let targets_guard = lock_bucket_targets_metadata(&bucket).await;
|
||||
validate_bucket_replication_update(&bucket, &replication_configuration).await?;
|
||||
let existing_config = match metadata_sys::get_replication_config(&bucket).await {
|
||||
let targets_guard = lock_bucket_targets_metadata(bucket).await;
|
||||
validate_bucket_replication_update(bucket, &replication_configuration).await?;
|
||||
let existing_config = match metadata_sys::get_replication_config(bucket).await {
|
||||
Ok((config, _)) => Some(config),
|
||||
Err(StorageError::ConfigNotFound) => None,
|
||||
Err(err) => return Err(ApiError::from(err).into()),
|
||||
@@ -372,14 +412,14 @@ impl DefaultBucketUsecase {
|
||||
let replication_configuration =
|
||||
merge_user_replication_config_update(replication_configuration, existing_config, &site_peers, contract);
|
||||
let data = serialize_config(&replication_configuration)?;
|
||||
update_bucket_config_for_incarnation(&bucket, BUCKET_REPLICATION_CONFIG, data, expected_incarnation_id)
|
||||
update_bucket_config_for_incarnation(bucket, BUCKET_REPLICATION_CONFIG, data, expected_incarnation_id)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
drop(targets_guard);
|
||||
|
||||
notify_bucket_metadata_reload(bucket.clone(), "put bucket replication", request_context, true).await;
|
||||
notify_bucket_metadata_reload(bucket.to_owned(), "put bucket replication", request_context, true).await;
|
||||
|
||||
let mut item = sr_bucket_meta_item(bucket.clone(), "replication-config");
|
||||
let mut item = sr_bucket_meta_item(bucket.to_owned(), "replication-config");
|
||||
item.replication_config = Some(
|
||||
serialize_config(&replication_configuration).and_then(|bytes| String::from_utf8(bytes).map_err(to_internal_error))?,
|
||||
);
|
||||
@@ -387,19 +427,29 @@ impl DefaultBucketUsecase {
|
||||
warn!(bucket = %bucket, error = ?err, "site replication bucket replication-config hook failed");
|
||||
}
|
||||
|
||||
Ok(S3Response::new(PutBucketReplicationOutput::default()))
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::app::bucket_usecase::test_support::build_request;
|
||||
use crate::app::bucket_usecase::test_support::{build_request, owned_envelope};
|
||||
use crate::app::gating_test_env::{run_large_stack_test, shared_gating_ambient};
|
||||
use crate::app::runtime_sources::ServerContextSlot;
|
||||
use crate::app::storage_api::owned_parity::{ReqInfo, authorize_bucket_config_mutation};
|
||||
use crate::site_replication::hooks::{BUCKET_META_HOOK_CALLS, BucketMetaHookCall};
|
||||
use futures::{FutureExt, future::BoxFuture};
|
||||
use http::Method;
|
||||
use rustfs_credentials::Credentials;
|
||||
use rustfs_gateway_types::dto::Status;
|
||||
use rustfs_gateway_types::persistence::{
|
||||
PersistedEncryptionConfiguration, PersistedReplicationDestination, PersistedReplicationRule,
|
||||
};
|
||||
use rustfs_policy::policy::action::{Action, S3Action};
|
||||
use serial_test::serial;
|
||||
use std::cell::RefCell;
|
||||
use std::sync::Mutex;
|
||||
|
||||
fn replication_rule_for_target(arn: &str) -> PersistedReplicationRule {
|
||||
PersistedReplicationRule {
|
||||
@@ -834,16 +884,11 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_delete_bucket_replication_returns_internal_error_when_store_uninitialized() {
|
||||
let input = DeleteBucketReplicationInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let req = build_request(input, Method::DELETE);
|
||||
let usecase = DefaultBucketUsecase::without_context();
|
||||
|
||||
let err = usecase
|
||||
.execute_delete_bucket_replication(req, HashSet::new(), OperatorRuleContract::Derived)
|
||||
.execute_delete_bucket_replication(owned_envelope(Method::DELETE), "test-bucket", async {
|
||||
Ok((HashSet::new(), OperatorRuleContract::Derived))
|
||||
})
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
@@ -869,17 +914,11 @@ mod tests {
|
||||
role: "arn:aws:iam::123456789012:role/test".to_string(),
|
||||
rules: vec![replication_rule_for_target("arn:rustfs:replication:us-east-1:target:bucket")],
|
||||
};
|
||||
let input = PutBucketReplicationInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.replication_configuration(Default::default())
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let req = build_request(input, Method::PUT);
|
||||
let usecase = DefaultBucketUsecase::without_context();
|
||||
|
||||
let err = usecase
|
||||
.execute_put_bucket_replication(req, configuration, HashSet::new(), OperatorRuleContract::Derived)
|
||||
.execute_put_bucket_replication(owned_envelope(Method::PUT), "test-bucket", async {
|
||||
Ok((configuration, HashSet::new(), OperatorRuleContract::Derived))
|
||||
})
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
@@ -893,19 +932,10 @@ mod tests {
|
||||
role: String::new(),
|
||||
rules: vec![rule],
|
||||
};
|
||||
let input = PutBucketReplicationInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.replication_configuration(Default::default())
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let err = DefaultBucketUsecase::without_context()
|
||||
.execute_put_bucket_replication(
|
||||
build_request(input, Method::PUT),
|
||||
configuration,
|
||||
HashSet::new(),
|
||||
OperatorRuleContract::Derived,
|
||||
)
|
||||
.execute_put_bucket_replication(owned_envelope(Method::PUT), "test-bucket", async {
|
||||
Ok((configuration, HashSet::new(), OperatorRuleContract::Derived))
|
||||
})
|
||||
.await
|
||||
.expect_err("unsupported fields must be rejected before store access");
|
||||
|
||||
@@ -915,4 +945,316 @@ mod tests {
|
||||
Some("replication field Destination.Account is not supported by this RustFS version")
|
||||
);
|
||||
}
|
||||
|
||||
const PEER_ARN: &str = "arn:rustfs:replication::peer-dep:bucket";
|
||||
const USER_ARN: &str = "arn:rustfs:replication:us-east-1:user:bucket";
|
||||
const SITE_RULE: &str = "site-repl-peer-dep";
|
||||
|
||||
tokio::task_local! {
|
||||
pub(super) static TARGET_CLEANUP_FAILURE: RefCell<Option<BoxFuture<'static, S3Result<()>>>>;
|
||||
}
|
||||
|
||||
fn put_configuration() -> PersistedReplicationConfiguration {
|
||||
PersistedReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![
|
||||
replication_rule_with_id(USER_ARN, "new-operator", 3),
|
||||
replication_rule_with_id(PEER_ARN, "shared-peer-target", 4),
|
||||
],
|
||||
}
|
||||
}
|
||||
|
||||
async fn seed_replication(store: &ECStore, bucket: &str, operator: &str) -> (Vec<u8>, Vec<u8>) {
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket for replication");
|
||||
metadata_sys::update(
|
||||
bucket,
|
||||
BUCKET_VERSIONING_CONFIG,
|
||||
b"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>".to_vec(),
|
||||
)
|
||||
.await
|
||||
.expect("enable versioning");
|
||||
let config = PersistedReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![
|
||||
replication_rule_with_id(USER_ARN, operator, 1),
|
||||
replication_rule_with_id(PEER_ARN, SITE_RULE, 2),
|
||||
],
|
||||
};
|
||||
let targets = BucketTargets {
|
||||
targets: [PEER_ARN, USER_ARN]
|
||||
.into_iter()
|
||||
.map(|arn| BucketTarget {
|
||||
source_bucket: bucket.to_owned(),
|
||||
arn: arn.to_owned(),
|
||||
target_type: BucketTargetType::ReplicationService,
|
||||
..Default::default()
|
||||
})
|
||||
.collect(),
|
||||
};
|
||||
let bytes = serialize(&config).expect("seed configuration bytes");
|
||||
let targets = serde_json::to_vec(&targets).expect("seed target bytes");
|
||||
metadata_sys::update(bucket, BUCKET_TARGETS_FILE, targets.clone())
|
||||
.await
|
||||
.expect("seed target metadata");
|
||||
metadata_sys::update(bucket, BUCKET_REPLICATION_CONFIG, bytes.clone())
|
||||
.await
|
||||
.expect("seed replication configuration");
|
||||
(bytes, targets)
|
||||
}
|
||||
|
||||
async fn persisted_replication(bucket: &str) -> (Vec<u8>, Vec<u8>) {
|
||||
let stored = metadata_sys::get_global_bucket_metadata_sys()
|
||||
.expect("initialized metadata system")
|
||||
.read()
|
||||
.await
|
||||
.get_config_from_disk(bucket)
|
||||
.await
|
||||
.expect("disk metadata");
|
||||
(stored.replication_config_xml, stored.bucket_targets_config_json)
|
||||
}
|
||||
|
||||
async fn authorize_replication(context: &Arc<AppContext>, bucket: &str, method: Method) -> RequestEnvelope {
|
||||
let server = ServerContextSlot::new();
|
||||
assert!(server.install(Arc::clone(context)));
|
||||
let mut request = build_request((), method);
|
||||
request.uri = format!("/{bucket}?replication").parse().expect("replication URI");
|
||||
request.extensions.insert(Arc::clone(&server));
|
||||
request.extensions.insert(ReqInfo {
|
||||
cred: Some(Credentials {
|
||||
access_key: "replication-owner".to_owned(),
|
||||
..Default::default()
|
||||
}),
|
||||
is_owner: true,
|
||||
..Default::default()
|
||||
});
|
||||
let mut envelope = RequestEnvelope::from_s3s(&request);
|
||||
authorize_bucket_config_mutation(
|
||||
&server,
|
||||
&mut envelope,
|
||||
bucket,
|
||||
Action::S3Action(S3Action::PutReplicationConfigurationAction),
|
||||
)
|
||||
.await
|
||||
.expect("snapshot authorized bucket");
|
||||
envelope
|
||||
}
|
||||
|
||||
async fn edit_replication(
|
||||
usecase: &DefaultBucketUsecase,
|
||||
envelope: RequestEnvelope,
|
||||
bucket: &str,
|
||||
put: bool,
|
||||
) -> S3Result<()> {
|
||||
let peers = HashSet::from(["peer-dep".to_owned()]);
|
||||
if put {
|
||||
usecase
|
||||
.execute_put_bucket_replication(envelope, bucket, async {
|
||||
Ok((put_configuration(), peers, OperatorRuleContract::Derived))
|
||||
})
|
||||
.await
|
||||
} else {
|
||||
usecase
|
||||
.execute_delete_bucket_replication(envelope, bucket, async { Ok((peers, OperatorRuleContract::Derived)) })
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
struct Reload {
|
||||
completed: bool,
|
||||
maintenance: bool,
|
||||
target_lock_available: bool,
|
||||
hook_calls: usize,
|
||||
}
|
||||
|
||||
async fn observe_replication_edit(
|
||||
bucket: &str,
|
||||
edit: impl Future<Output = S3Result<()>>,
|
||||
) -> (S3Result<()>, Vec<Reload>, Vec<BucketMetaHookCall>) {
|
||||
let reloads = Arc::new(Mutex::new(Vec::new()));
|
||||
let record = Arc::clone(&reloads);
|
||||
let target_bucket = bucket.to_owned();
|
||||
let (result, hooks) = BUCKET_META_HOOK_CALLS
|
||||
.scope(
|
||||
RefCell::default(),
|
||||
BUCKET_METADATA_RELOAD_PROBE.scope(
|
||||
(
|
||||
bucket.to_owned(),
|
||||
Box::new(move |completed, _, maintenance| {
|
||||
record.lock().expect("reloads").push(Reload {
|
||||
completed,
|
||||
maintenance,
|
||||
target_lock_available: lock_bucket_targets_metadata(&target_bucket).now_or_never().is_some(),
|
||||
hook_calls: BUCKET_META_HOOK_CALLS.with(|calls| calls.borrow().len()),
|
||||
});
|
||||
}),
|
||||
),
|
||||
async {
|
||||
let result = edit.await;
|
||||
(result, BUCKET_META_HOOK_CALLS.with(RefCell::take))
|
||||
},
|
||||
),
|
||||
)
|
||||
.await;
|
||||
let reloads = std::mem::take(&mut *reloads.lock().expect("reloads"));
|
||||
(result, reloads, hooks)
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn owned_replication_preserves_peer_rules_and_finishes_reload_before_site_hook() {
|
||||
run_large_stack_test("owned-replication-peer-rules", || async {
|
||||
let context = shared_gating_ambient().await;
|
||||
let bucket = format!("replication-peer-{}", uuid::Uuid::new_v4().simple());
|
||||
let (original, _) = seed_replication(&context.object_store(), &bucket, "old-operator").await;
|
||||
let usecase = DefaultBucketUsecase::with_context(Some(Arc::clone(&context)));
|
||||
for (put, method) in [(true, Method::PUT), (false, Method::DELETE)] {
|
||||
let envelope = authorize_replication(&context, &bucket, method).await;
|
||||
let (result, reloads, hooks) =
|
||||
observe_replication_edit(&bucket, edit_replication(&usecase, envelope, &bucket, put)).await;
|
||||
result.expect("replication edit");
|
||||
assert_eq!(
|
||||
reloads,
|
||||
[
|
||||
Reload {
|
||||
completed: false,
|
||||
maintenance: true,
|
||||
target_lock_available: true,
|
||||
hook_calls: 0
|
||||
},
|
||||
Reload {
|
||||
completed: true,
|
||||
maintenance: true,
|
||||
target_lock_available: true,
|
||||
hook_calls: 0
|
||||
},
|
||||
]
|
||||
);
|
||||
|
||||
let (xml, target_bytes) = persisted_replication(&bucket).await;
|
||||
assert_ne!(xml, original);
|
||||
let config = rustfs_gateway_types::persistence::parse_replication(&xml).expect("stored XML");
|
||||
let ids: HashSet<_> = config.rules.iter().filter_map(|rule| rule.id.as_deref()).collect();
|
||||
let targets: BucketTargets = serde_json::from_slice(&target_bytes).expect("stored targets");
|
||||
let arns: HashSet<_> = targets.targets.iter().map(|target| target.arn.as_str()).collect();
|
||||
if put {
|
||||
assert_eq!(ids, HashSet::from([SITE_RULE, "new-operator", "shared-peer-target"]));
|
||||
assert_eq!(arns, HashSet::from([PEER_ARN, USER_ARN]));
|
||||
} else {
|
||||
assert_eq!(ids, HashSet::from([SITE_RULE]));
|
||||
assert_eq!(arns, HashSet::from([PEER_ARN]));
|
||||
}
|
||||
assert_eq!(hooks.len(), 1);
|
||||
assert_eq!(
|
||||
(hooks[0].bucket.as_str(), hooks[0].item_type.as_str()),
|
||||
(bucket.as_str(), "replication-config")
|
||||
);
|
||||
assert_eq!(hooks[0].replication_config.as_deref().map(str::as_bytes), put.then_some(xml.as_slice()));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn owned_replication_stale_edits_leave_replacement_bytes_and_skip_callbacks() {
|
||||
run_large_stack_test("owned-replication-stale", || async {
|
||||
let context = shared_gating_ambient().await;
|
||||
let store = context.object_store();
|
||||
let usecase = DefaultBucketUsecase::with_context(Some(Arc::clone(&context)));
|
||||
for (method, preserve_peer) in [(Method::PUT, true), (Method::DELETE, true), (Method::DELETE, false)] {
|
||||
let bucket = format!("replication-stale-{}", uuid::Uuid::new_v4().simple());
|
||||
seed_replication(&store, &bucket, "old-operator").await;
|
||||
let put = method == Method::PUT;
|
||||
let envelope = authorize_replication(&context, &bucket, method).await;
|
||||
store
|
||||
.delete_bucket(&bucket, &DeleteBucketOptions::default())
|
||||
.await
|
||||
.expect("delete authorized bucket");
|
||||
let replacement = seed_replication(&store, &bucket, "replacement").await;
|
||||
let edit = async {
|
||||
if preserve_peer {
|
||||
edit_replication(&usecase, envelope, &bucket, put).await
|
||||
} else {
|
||||
usecase
|
||||
.execute_delete_bucket_replication(envelope, &bucket, async {
|
||||
Ok((HashSet::new(), OperatorRuleContract::Derived))
|
||||
})
|
||||
.await
|
||||
}
|
||||
};
|
||||
let (result, reloads, hooks) = observe_replication_edit(&bucket, edit).await;
|
||||
assert_eq!(
|
||||
(
|
||||
result.err().map(|err| err.code().clone()),
|
||||
persisted_replication(&bucket).await,
|
||||
reloads.len(),
|
||||
hooks.len()
|
||||
),
|
||||
(Some(S3ErrorCode::NoSuchBucket), replacement, 0, 0),
|
||||
"stale {}",
|
||||
if put { "Put" } else { "Delete" }
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn owned_replication_cleanup_error_restores_original_and_never_overwrites_a_replacement() {
|
||||
run_large_stack_test("owned-replication-cleanup", || async {
|
||||
let context = shared_gating_ambient().await;
|
||||
let store = context.object_store();
|
||||
let usecase = DefaultBucketUsecase::with_context(Some(Arc::clone(&context)));
|
||||
for (replace, fail_target_write) in [(false, true), (true, true), (true, false)] {
|
||||
let bucket = format!("replication-cleanup-{}", uuid::Uuid::new_v4().simple());
|
||||
let original = seed_replication(&store, &bucket, "old-operator").await;
|
||||
let envelope = authorize_replication(&context, &bucket, Method::DELETE).await;
|
||||
let expected_storage = Arc::new(Mutex::new(original));
|
||||
let output = Arc::clone(&expected_storage);
|
||||
let replacement_bucket = bucket.clone();
|
||||
let old_store = Arc::clone(&store);
|
||||
let fail = Box::pin(async move {
|
||||
let (after_delete, _) = persisted_replication(&replacement_bucket).await;
|
||||
assert!(!String::from_utf8_lossy(&after_delete).contains("old-operator"));
|
||||
assert!(lock_bucket_targets_metadata(&replacement_bucket).now_or_never().is_none());
|
||||
if replace {
|
||||
old_store
|
||||
.delete_bucket(&replacement_bucket, &DeleteBucketOptions::default())
|
||||
.await
|
||||
.expect("delete original before restoration");
|
||||
let replacement = seed_replication(&old_store, &replacement_bucket, "replacement").await;
|
||||
*output.lock().expect("replacement bytes") = replacement;
|
||||
}
|
||||
if fail_target_write {
|
||||
Err(s3_error!(InternalError, "replication target cleanup failed"))
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
});
|
||||
let (result, reloads, hooks) = TARGET_CLEANUP_FAILURE
|
||||
.scope(
|
||||
RefCell::new(Some(fail)),
|
||||
observe_replication_edit(&bucket, edit_replication(&usecase, envelope, &bucket, false)),
|
||||
)
|
||||
.await;
|
||||
let expected = if fail_target_write {
|
||||
S3ErrorCode::InternalError
|
||||
} else {
|
||||
S3ErrorCode::NoSuchBucket
|
||||
};
|
||||
let stored = persisted_replication(&bucket).await;
|
||||
assert_eq!(
|
||||
(result.as_ref().err().map(|error| error.code().clone()), stored),
|
||||
(Some(expected), expected_storage.lock().expect("expected bytes").clone())
|
||||
);
|
||||
if fail_target_write {
|
||||
assert_eq!(result.unwrap_err().message(), Some("replication target cleanup failed"));
|
||||
}
|
||||
assert!(reloads.is_empty() && hooks.is_empty());
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12,4 +12,46 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Bucket request-payment configuration use-case operations (reserved for the next batch).
|
||||
//! Bucket request-payment configuration shared by the S3 edges.
|
||||
|
||||
use super::BUCKET_REQUEST_PAYMENT_CONFIG;
|
||||
use super::{
|
||||
ApiError, DefaultBucketUsecase, StorageError, bucket_config_mutation_incarnation, metadata_sys, serialize,
|
||||
update_bucket_config_for_incarnation,
|
||||
};
|
||||
use crate::app::RequestEnvelope;
|
||||
use rustfs_gateway_types::persistence::PersistedRequestPaymentConfiguration;
|
||||
use rustfs_s3_types::S3Result;
|
||||
|
||||
impl DefaultBucketUsecase {
|
||||
pub(crate) async fn execute_get_bucket_request_payment(
|
||||
&self,
|
||||
bucket: &str,
|
||||
) -> S3Result<PersistedRequestPaymentConfiguration> {
|
||||
self.ensure_config_bucket(bucket).await?;
|
||||
match metadata_sys::get_request_payment_config(bucket).await {
|
||||
Ok((config, _)) => Ok(config),
|
||||
Err(StorageError::ConfigNotFound) => Ok(PersistedRequestPaymentConfiguration {
|
||||
payer: "BucketOwner".to_owned(),
|
||||
}),
|
||||
Err(err) => Err(ApiError::from(err).into()),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn execute_put_bucket_request_payment(
|
||||
&self,
|
||||
envelope: RequestEnvelope,
|
||||
bucket: &str,
|
||||
configuration: impl FnOnce() -> S3Result<PersistedRequestPaymentConfiguration> + Send,
|
||||
) -> S3Result<()> {
|
||||
let expected_incarnation_id = bucket_config_mutation_incarnation(&envelope, bucket)?;
|
||||
self.ensure_config_bucket(bucket).await?;
|
||||
|
||||
let config = configuration()?;
|
||||
let data = serialize(&config).map_err(|err| rustfs_s3_types::s3_error!(MalformedXML, "{err}"))?;
|
||||
update_bucket_config_for_incarnation(bucket, BUCKET_REQUEST_PAYMENT_CONFIG, data, expected_incarnation_id)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,58 +15,63 @@
|
||||
//! Bucket tagging use-case operations.
|
||||
|
||||
use super::*;
|
||||
use crate::app::storage_api::s3::dto;
|
||||
use crate::app::storage_api::s3::dto::{
|
||||
DeleteBucketTaggingInput, DeleteBucketTaggingOutput, GetBucketTaggingInput, PutBucketTaggingInput, PutBucketTaggingOutput,
|
||||
};
|
||||
use crate::app::storage_api::s3::{S3Error, S3ErrorCode, S3Request, S3Response, S3Result};
|
||||
use crate::app::storage_api::bucket_usecase::bucket::metadata::{MAX_BUCKET_TAGS, validate_tag_set_entries};
|
||||
use rustfs_s3_types::s3_error;
|
||||
|
||||
impl DefaultBucketUsecase {
|
||||
#[instrument(level = "debug", skip(self, req))]
|
||||
pub async fn execute_delete_bucket_tagging(
|
||||
#[instrument(level = "debug", skip(self, envelope))]
|
||||
pub(crate) async fn execute_delete_bucket_tagging(
|
||||
&self,
|
||||
req: S3Request<DeleteBucketTaggingInput>,
|
||||
) -> S3Result<S3Response<DeleteBucketTaggingOutput>> {
|
||||
let expected_incarnation_id = bucket_config_mutation_incarnation(&req, &req.input.bucket)?;
|
||||
let request_context = req.extensions.get::<request_context::RequestContext>().cloned();
|
||||
let DeleteBucketTaggingInput { bucket, .. } = req.input;
|
||||
envelope: RequestEnvelope,
|
||||
bucket: &str,
|
||||
) -> rustfs_s3_types::S3Result<()> {
|
||||
RequestEnvelope::audit_only_operation(envelope, S3Operation::DeleteBucketTagging, StatusCode::OK, |envelope| async move {
|
||||
let expected_incarnation_id = bucket_config_mutation_incarnation(&envelope, bucket)?;
|
||||
let request_context = envelope.request_context().cloned();
|
||||
|
||||
metadata_sys::update_s3_tagging_config(&bucket, None, expected_incarnation_id)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
metadata_sys::update_s3_tagging_config(bucket, None, expected_incarnation_id)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
notify_bucket_metadata_reload(bucket.clone(), "delete bucket tagging", request_context, false).await;
|
||||
notify_bucket_metadata_reload(bucket.to_owned(), "delete bucket tagging", request_context, false).await;
|
||||
|
||||
let item = sr_bucket_meta_item(bucket.clone(), "tags");
|
||||
if let Err(err) = site_replication_bucket_meta_hook(item).await {
|
||||
warn!(bucket = %bucket, error = ?err, "site replication bucket tagging delete hook failed");
|
||||
}
|
||||
let item = sr_bucket_meta_item(bucket.to_owned(), "tags");
|
||||
if let Err(err) = site_replication_bucket_meta_hook(item).await {
|
||||
warn!(bucket = %bucket, error = ?err, "site replication bucket tagging delete hook failed");
|
||||
}
|
||||
|
||||
rustfs_scanner::record_dirty_usage_bucket_from_producer(
|
||||
&bucket,
|
||||
rustfs_scanner::SegmentInvalidationProducerIdentity::BucketMetadata,
|
||||
);
|
||||
Ok(S3Response::new(DeleteBucketTaggingOutput {}))
|
||||
rustfs_scanner::record_dirty_usage_bucket_from_producer(
|
||||
bucket,
|
||||
rustfs_scanner::SegmentInvalidationProducerIdentity::BucketMetadata,
|
||||
);
|
||||
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self, req))]
|
||||
pub async fn execute_get_bucket_tagging(&self, req: S3Request<GetBucketTaggingInput>) -> S3Result<PersistedTagging> {
|
||||
let GetBucketTaggingInput { bucket, .. } = req.input;
|
||||
|
||||
/// Returns the stored bucket tagging configuration after checking the bucket exists.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Returns `InternalError` if the store is uninitialized, `NoSuchTagSet`
|
||||
/// if the configuration is absent, or the storage error from either lookup.
|
||||
#[instrument(level = "debug", skip(self, bucket))]
|
||||
pub async fn execute_get_bucket_tagging(&self, bucket: &str) -> rustfs_s3_types::S3Result<PersistedTagging> {
|
||||
let Some(store) = self.object_store() else {
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
|
||||
return Err(rustfs_s3_types::s3_error!(InternalError, "Not init"));
|
||||
};
|
||||
|
||||
store
|
||||
.get_bucket_info(&bucket, &BucketOptions::default())
|
||||
.get_bucket_info(bucket, &BucketOptions::default())
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
let tagging = match metadata_sys::get_tagging_config(&bucket).await {
|
||||
let tagging = match metadata_sys::get_tagging_config(bucket).await {
|
||||
Ok((tags, _)) => tags,
|
||||
Err(err) => {
|
||||
if err == StorageError::ConfigNotFound {
|
||||
return Err(S3Error::with_message(S3ErrorCode::NoSuchTagSet, "The TagSet does not exist".to_string()));
|
||||
return Err(rustfs_s3_types::s3_error!(NoSuchTagSet, "The TagSet does not exist"));
|
||||
}
|
||||
warn!(
|
||||
component = LOG_COMPONENT_APP,
|
||||
@@ -83,51 +88,50 @@ impl DefaultBucketUsecase {
|
||||
Ok(tagging)
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self, req))]
|
||||
pub async fn execute_put_bucket_tagging(
|
||||
#[instrument(level = "debug", skip(self, envelope, tagging))]
|
||||
pub(crate) async fn execute_put_bucket_tagging(
|
||||
&self,
|
||||
req: S3Request<PutBucketTaggingInput>,
|
||||
envelope: RequestEnvelope,
|
||||
bucket: &str,
|
||||
tagging: PersistedTagging,
|
||||
) -> S3Result<S3Response<PutBucketTaggingOutput>> {
|
||||
let expected_incarnation_id = bucket_config_mutation_incarnation(&req, &req.input.bucket)?;
|
||||
let request_context = req.extensions.get::<request_context::RequestContext>().cloned();
|
||||
let PutBucketTaggingInput { bucket, .. } = req.input;
|
||||
let s3_tag_set: Vec<dto::Tag> = tagging
|
||||
.tag_set
|
||||
.iter()
|
||||
.map(|tag| dto::Tag {
|
||||
key: tag.key.clone(),
|
||||
value: tag.value.clone(),
|
||||
})
|
||||
.collect();
|
||||
crate::app::storage_api::bucket_usecase::s3_api::tagging::validate_bucket_tag_set(&s3_tag_set)?;
|
||||
) -> rustfs_s3_types::S3Result<()> {
|
||||
RequestEnvelope::audit_only_operation(envelope, S3Operation::PutBucketTagging, StatusCode::OK, |envelope| async move {
|
||||
let expected_incarnation_id = bucket_config_mutation_incarnation(&envelope, bucket)?;
|
||||
let request_context = envelope.request_context().cloned();
|
||||
|
||||
let Some(store) = self.object_store() else {
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
|
||||
};
|
||||
if tagging.tag_set.len() > MAX_BUCKET_TAGS {
|
||||
return Err(s3_error!(InvalidTag, "Cannot have more than 50 tags per bucket"));
|
||||
}
|
||||
validate_tag_set_entries(&tagging.tag_set).map_err(|err| s3_error!(InvalidTag, "{err}"))?;
|
||||
|
||||
store
|
||||
.get_bucket_info(&bucket, &BucketOptions::default())
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
let Some(store) = self.object_store() else {
|
||||
return Err(s3_error!(InternalError, "Not init"));
|
||||
};
|
||||
|
||||
metadata_sys::update_s3_tagging_config(&bucket, Some(tagging.clone()), expected_incarnation_id)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
store
|
||||
.get_bucket_info(bucket, &BucketOptions::default())
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
notify_bucket_metadata_reload(bucket.clone(), "put bucket tagging", request_context, false).await;
|
||||
metadata_sys::update_s3_tagging_config(bucket, Some(tagging.clone()), expected_incarnation_id)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
let mut item = sr_bucket_meta_item(bucket.clone(), "tags");
|
||||
item.tags = Some(serialize_config(&tagging).and_then(|bytes| String::from_utf8(bytes).map_err(to_internal_error))?);
|
||||
if let Err(err) = site_replication_bucket_meta_hook(item).await {
|
||||
warn!(bucket = %bucket, error = ?err, "site replication bucket tagging hook failed");
|
||||
}
|
||||
notify_bucket_metadata_reload(bucket.to_owned(), "put bucket tagging", request_context, false).await;
|
||||
|
||||
rustfs_scanner::record_dirty_usage_bucket_from_producer(
|
||||
&bucket,
|
||||
rustfs_scanner::SegmentInvalidationProducerIdentity::BucketMetadata,
|
||||
);
|
||||
Ok(S3Response::new(PutBucketTaggingOutput::default()))
|
||||
let mut item = sr_bucket_meta_item(bucket.to_owned(), "tags");
|
||||
item.tags = Some(serialize_config(&tagging).and_then(|bytes| String::from_utf8(bytes).map_err(to_internal_error))?);
|
||||
if let Err(err) = site_replication_bucket_meta_hook(item).await {
|
||||
warn!(bucket = %bucket, error = ?err, "site replication bucket tagging hook failed");
|
||||
}
|
||||
|
||||
rustfs_scanner::record_dirty_usage_bucket_from_producer(
|
||||
bucket,
|
||||
rustfs_scanner::SegmentInvalidationProducerIdentity::BucketMetadata,
|
||||
);
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
@@ -135,21 +139,16 @@ impl DefaultBucketUsecase {
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::app::bucket_usecase::test_support::build_request;
|
||||
use crate::app::storage_api::s3::dto::{Tag, Tagging};
|
||||
use crate::app::storage_api::s3::dto::{self, PutBucketTaggingInput, Tag, Tagging};
|
||||
use http::Method;
|
||||
use rustfs_s3_types::S3ErrorCode;
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_get_bucket_tagging_returns_internal_error_when_store_uninitialized() {
|
||||
let input = GetBucketTaggingInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let req = build_request(input, Method::GET);
|
||||
let usecase = DefaultBucketUsecase::without_context();
|
||||
|
||||
let err = usecase.execute_get_bucket_tagging(req).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
let err = usecase.execute_get_bucket_tagging("test-bucket").await.unwrap_err();
|
||||
assert_eq!(err.code(), &rustfs_s3_types::S3ErrorCode::InternalError);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -214,7 +213,9 @@ mod tests {
|
||||
req.extensions.insert(ReqInfo::default());
|
||||
req.extensions.insert(slot.clone());
|
||||
fs.put_bucket_tagging(&mut req).await.expect("permission allows tag updates");
|
||||
let result = usecase.execute_put_bucket_tagging(req, persisted).await;
|
||||
let result = usecase
|
||||
.execute_put_bucket_tagging(RequestEnvelope::from_s3s(&req), &req.input.bucket, persisted)
|
||||
.await;
|
||||
if valid {
|
||||
result.expect("valid tags are persisted");
|
||||
} else {
|
||||
@@ -245,9 +246,25 @@ mod tests {
|
||||
let usecase = DefaultBucketUsecase::without_context();
|
||||
|
||||
let err = usecase
|
||||
.execute_put_bucket_tagging(req, PersistedTagging::default())
|
||||
.execute_put_bucket_tagging(RequestEnvelope::from_s3s(&req), &req.input.bucket, PersistedTagging::default())
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
|
||||
let tag = rustfs_gateway_types::cors_tagging::PersistedTag {
|
||||
key: Some("team".to_owned()),
|
||||
value: Some("storage".to_owned()),
|
||||
};
|
||||
for (tag_set, message) in [
|
||||
(vec![tag.clone(), tag.clone()], "Cannot provide multiple Tags with the same key"),
|
||||
(vec![tag; MAX_BUCKET_TAGS + 1], "Cannot have more than 50 tags per bucket"),
|
||||
] {
|
||||
let err = usecase
|
||||
.execute_put_bucket_tagging(RequestEnvelope::from_s3s(&req), &req.input.bucket, PersistedTagging { tag_set })
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidTag);
|
||||
assert_eq!(err.message(), Some(message));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12,4 +12,228 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Bucket website configuration use-case operations (reserved for the next batch).
|
||||
//! Bucket website configuration shared by the S3 edges.
|
||||
|
||||
use super::BUCKET_WEBSITE_CONFIG;
|
||||
use super::{
|
||||
ApiError, DefaultBucketUsecase, StorageError, bucket_config_mutation_incarnation, delete_bucket_config_for_incarnation,
|
||||
metadata_sys, serialize, update_bucket_config_for_incarnation,
|
||||
};
|
||||
use crate::app::RequestEnvelope;
|
||||
use crate::app::storage_api::bucket_usecase::validate_website_configuration;
|
||||
use rustfs_gateway_types::persistence::PersistedWebsiteConfiguration;
|
||||
use rustfs_s3_types::S3Result;
|
||||
|
||||
impl DefaultBucketUsecase {
|
||||
pub(crate) async fn execute_get_bucket_website(&self, bucket: &str) -> S3Result<PersistedWebsiteConfiguration> {
|
||||
self.ensure_config_bucket(bucket).await?;
|
||||
match metadata_sys::get_website_config(bucket).await {
|
||||
Ok((config, _)) => Ok(config),
|
||||
Err(StorageError::ConfigNotFound) => Err(rustfs_s3_types::s3_error!(
|
||||
NoSuchWebsiteConfiguration,
|
||||
"The specified bucket does not have a website configuration."
|
||||
)),
|
||||
Err(err) => Err(ApiError::from(err).into()),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn execute_put_bucket_website(
|
||||
&self,
|
||||
envelope: RequestEnvelope,
|
||||
bucket: &str,
|
||||
configuration: impl FnOnce() -> S3Result<PersistedWebsiteConfiguration> + Send,
|
||||
) -> S3Result<()> {
|
||||
let expected_incarnation_id = bucket_config_mutation_incarnation(&envelope, bucket)?;
|
||||
self.ensure_config_bucket(bucket).await?;
|
||||
|
||||
let config = configuration()?;
|
||||
validate_website_configuration(&config)?;
|
||||
let data = serialize(&config).map_err(|err| rustfs_s3_types::s3_error!(MalformedXML, "{err}"))?;
|
||||
update_bucket_config_for_incarnation(bucket, BUCKET_WEBSITE_CONFIG, data, expected_incarnation_id)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) async fn execute_delete_bucket_website(&self, envelope: RequestEnvelope, bucket: &str) -> S3Result<()> {
|
||||
let expected_incarnation_id = bucket_config_mutation_incarnation(&envelope, bucket)?;
|
||||
self.ensure_config_bucket(bucket).await?;
|
||||
|
||||
delete_bucket_config_for_incarnation(bucket, BUCKET_WEBSITE_CONFIG, expected_incarnation_id)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::super::{BUCKET_ACCELERATE_CONFIG, BUCKET_REQUEST_PAYMENT_CONFIG};
|
||||
use super::*;
|
||||
use crate::app::bucket_usecase::test_support::build_request;
|
||||
use crate::app::gating_test_env::{run_large_stack_test, shared_gating_ambient};
|
||||
use crate::app::runtime_sources::ServerContextSlot;
|
||||
use crate::app::storage_api::owned_parity::{ReqInfo, authorize_bucket_config_mutation};
|
||||
use rustfs_credentials::Credentials;
|
||||
use rustfs_gateway_types::persistence::{parse_accelerate, parse_request_payment, parse_website};
|
||||
use rustfs_policy::policy::action::{Action, S3Action};
|
||||
use rustfs_s3_types::S3ErrorCode;
|
||||
use serial_test::serial;
|
||||
use std::sync::Arc;
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn core_config_puts_defer_conversion_until_after_bucket_lookup() {
|
||||
run_large_stack_test("core-config-lookup-before-conversion", || async {
|
||||
let ambient = shared_gating_ambient().await;
|
||||
let usecase = DefaultBucketUsecase::with_context(Some(ambient));
|
||||
let bucket = format!("core-config-missing-{}", uuid::Uuid::new_v4().simple());
|
||||
let request = build_request((), http::Method::PUT);
|
||||
let mut converted = [false; 3];
|
||||
let accelerate = usecase
|
||||
.execute_put_bucket_accelerate_configuration(RequestEnvelope::from_s3s(&request), &bucket, || {
|
||||
converted[0] = true;
|
||||
Err(rustfs_s3_types::s3_error!(MalformedXML, "invalid acceleration"))
|
||||
})
|
||||
.await;
|
||||
let payment = usecase
|
||||
.execute_put_bucket_request_payment(RequestEnvelope::from_s3s(&request), &bucket, || {
|
||||
converted[1] = true;
|
||||
Err(rustfs_s3_types::s3_error!(MalformedXML, "invalid request payment"))
|
||||
})
|
||||
.await;
|
||||
let website = usecase
|
||||
.execute_put_bucket_website(RequestEnvelope::from_s3s(&request), &bucket, || {
|
||||
converted[2] = true;
|
||||
Err(rustfs_s3_types::s3_error!(MalformedXML, "invalid website"))
|
||||
})
|
||||
.await;
|
||||
assert_eq!(
|
||||
[accelerate, payment, website].map(|result| result.err().map(|error| error.code().clone())),
|
||||
[
|
||||
Some(S3ErrorCode::NoSuchBucket),
|
||||
Some(S3ErrorCode::NoSuchBucket),
|
||||
Some(S3ErrorCode::NoSuchBucket)
|
||||
],
|
||||
);
|
||||
assert_eq!(converted, [false; 3]);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn core_config_mutations_reject_stale_incarnations() {
|
||||
use super::super::{BucketOperations, DeleteBucketOptions, MakeBucketOptions};
|
||||
const OLD_ACCELERATE: &[u8] = b"<AccelerateConfiguration><Status>Suspended</Status></AccelerateConfiguration>";
|
||||
const NEW_ACCELERATE: &[u8] = b"<AccelerateConfiguration><Status>Enabled</Status></AccelerateConfiguration>";
|
||||
const OLD_PAYMENT: &[u8] = b"<RequestPaymentConfiguration><Payer>BucketOwner</Payer></RequestPaymentConfiguration>";
|
||||
const NEW_PAYMENT: &[u8] = b"<RequestPaymentConfiguration><Payer>Requester</Payer></RequestPaymentConfiguration>";
|
||||
const OLD_WEBSITE: &[u8] =
|
||||
b"<WebsiteConfiguration><IndexDocument><Suffix>old.html</Suffix></IndexDocument></WebsiteConfiguration>";
|
||||
const NEW_WEBSITE: &[u8] =
|
||||
b"<WebsiteConfiguration><IndexDocument><Suffix>new.html</Suffix></IndexDocument></WebsiteConfiguration>";
|
||||
|
||||
run_large_stack_test("core-config-stale-incarnation", || async {
|
||||
let ambient = shared_gating_ambient().await;
|
||||
let store = ambient.object_store();
|
||||
let server = ServerContextSlot::new();
|
||||
assert!(server.install(Arc::clone(&ambient)));
|
||||
let usecase = DefaultBucketUsecase::with_context(Some(Arc::clone(&ambient)));
|
||||
|
||||
let mut results = Vec::new();
|
||||
for (method, query, file, before) in [
|
||||
(http::Method::PUT, "accelerate", BUCKET_ACCELERATE_CONFIG, OLD_ACCELERATE),
|
||||
(http::Method::PUT, "requestPayment", BUCKET_REQUEST_PAYMENT_CONFIG, OLD_PAYMENT),
|
||||
(http::Method::PUT, "website", BUCKET_WEBSITE_CONFIG, OLD_WEBSITE),
|
||||
(http::Method::DELETE, "website", BUCKET_WEBSITE_CONFIG, OLD_WEBSITE),
|
||||
] {
|
||||
let bucket = format!("core-config-stale-{}", uuid::Uuid::new_v4().simple());
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create original bucket");
|
||||
let mut request = build_request((), method.clone());
|
||||
request.uri = format!("/{bucket}?{query}").parse().expect("bucket request URI");
|
||||
request.extensions.insert(Arc::clone(&server));
|
||||
request.extensions.insert(ReqInfo {
|
||||
cred: Some(Credentials {
|
||||
access_key: "core-config-owner".to_owned(),
|
||||
..Default::default()
|
||||
}),
|
||||
is_owner: true,
|
||||
..Default::default()
|
||||
});
|
||||
let mut envelope = RequestEnvelope::from_s3s(&request);
|
||||
authorize_bucket_config_mutation(
|
||||
&server,
|
||||
&mut envelope,
|
||||
&bucket,
|
||||
Action::S3Action(S3Action::PutBucketPolicyAction),
|
||||
)
|
||||
.await
|
||||
.expect("owner authorization captures the original incarnation");
|
||||
store
|
||||
.delete_bucket(&bucket, &DeleteBucketOptions::default())
|
||||
.await
|
||||
.expect("delete original bucket");
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create replacement bucket");
|
||||
metadata_sys::update(&bucket, file, before.to_vec())
|
||||
.await
|
||||
.expect("seed replacement configuration");
|
||||
|
||||
let result = match (method, query) {
|
||||
(http::Method::PUT, "accelerate") => {
|
||||
usecase
|
||||
.execute_put_bucket_accelerate_configuration(envelope, &bucket, || {
|
||||
Ok(parse_accelerate(NEW_ACCELERATE).expect("changed accelerate document"))
|
||||
})
|
||||
.await
|
||||
}
|
||||
(http::Method::PUT, "requestPayment") => {
|
||||
usecase
|
||||
.execute_put_bucket_request_payment(envelope, &bucket, || {
|
||||
Ok(parse_request_payment(NEW_PAYMENT).expect("changed request payment document"))
|
||||
})
|
||||
.await
|
||||
}
|
||||
(http::Method::PUT, "website") => {
|
||||
usecase
|
||||
.execute_put_bucket_website(envelope, &bucket, || {
|
||||
Ok(parse_website(NEW_WEBSITE).expect("changed website document"))
|
||||
})
|
||||
.await
|
||||
}
|
||||
(http::Method::DELETE, "website") => usecase.execute_delete_bucket_website(envelope, &bucket).await,
|
||||
_ => unreachable!("only core bucket configuration mutations"),
|
||||
};
|
||||
|
||||
let stored = metadata_sys::get_global_bucket_metadata_sys()
|
||||
.expect("initialized metadata system")
|
||||
.read()
|
||||
.await
|
||||
.get_config_from_disk(&bucket)
|
||||
.await
|
||||
.expect("replacement metadata");
|
||||
let after = match file {
|
||||
BUCKET_ACCELERATE_CONFIG => stored.accelerate_config_xml,
|
||||
BUCKET_REQUEST_PAYMENT_CONFIG => stored.request_payment_config_xml,
|
||||
BUCKET_WEBSITE_CONFIG => stored.website_config_xml,
|
||||
_ => unreachable!("only core bucket configurations"),
|
||||
};
|
||||
results.push((query, result.err().map(|err| err.code().clone()), after == before));
|
||||
}
|
||||
assert_eq!(
|
||||
results,
|
||||
["accelerate", "requestPayment", "website", "website"].map(|query| (
|
||||
query,
|
||||
Some(S3ErrorCode::NoSuchBucket),
|
||||
true
|
||||
)),
|
||||
"each operation must reject the stale authorization and preserve the replacement bytes"
|
||||
);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -664,6 +664,9 @@ impl AuthorizationRequest for RequestEnvelope {
|
||||
fn requested_object_tags(&self) -> Option<&RequestedObjectTags> {
|
||||
self.requested_object_tags()
|
||||
}
|
||||
fn take_requested_object_tags(&mut self) -> Option<RequestedObjectTags> {
|
||||
self.requested_object_tags.take()
|
||||
}
|
||||
fn install_requested_retention(&mut self, retention: RequestedObjectLockRetention) {
|
||||
self.requested_object_lock_retention = Some(retention);
|
||||
}
|
||||
@@ -691,6 +694,9 @@ impl AuthorizationRequest for RequestEnvelope {
|
||||
fn object_tag_conditions(&self) -> Option<&ObjectTagConditions> {
|
||||
self.object_tag_conditions()
|
||||
}
|
||||
fn take_object_tag_conditions(&mut self) -> Option<ObjectTagConditions> {
|
||||
self.object_tag_conditions.take()
|
||||
}
|
||||
fn install_object_tag_conditions(&mut self, conditions: ObjectTagConditions) {
|
||||
self.install_object_tag_conditions(conditions);
|
||||
}
|
||||
|
||||
@@ -377,6 +377,11 @@ impl AppContext {
|
||||
self
|
||||
}
|
||||
|
||||
pub(crate) fn with_test_notify_interface(mut self, notify: Arc<dyn NotifyInterface>) -> Self {
|
||||
self.notify = notify;
|
||||
self
|
||||
}
|
||||
|
||||
pub(crate) fn with_test_notification_system_interface(
|
||||
mut self,
|
||||
notification_system: Arc<dyn NotificationSystemInterface>,
|
||||
|
||||
@@ -332,7 +332,7 @@ async fn delete_objects_resolves_bucket_versioning_once_per_request() {
|
||||
let lookups_before = VERSIONING_CONFIG_LOOKUPS.load(Ordering::SeqCst);
|
||||
let snapshots_before = DELETE_CONFIG_SNAPSHOT_LOADS.load(Ordering::SeqCst);
|
||||
let response = usecase
|
||||
.execute_delete_objects(req)
|
||||
.execute_delete_objects_from_s3s(req)
|
||||
.await
|
||||
.expect("delete objects should succeed");
|
||||
|
||||
@@ -389,7 +389,7 @@ async fn delete_objects_treats_empty_directory_version_as_absent() {
|
||||
});
|
||||
|
||||
let response = usecase
|
||||
.execute_delete_objects(req)
|
||||
.execute_delete_objects_from_s3s(req)
|
||||
.await
|
||||
.expect("empty VersionId should be treated as an absent version");
|
||||
let deleted = response
|
||||
@@ -438,7 +438,7 @@ async fn delete_objects_preserves_explicit_null_directory_version() {
|
||||
});
|
||||
|
||||
let response = usecase
|
||||
.execute_delete_objects(req)
|
||||
.execute_delete_objects_from_s3s(req)
|
||||
.await
|
||||
.expect("explicit null VersionId should delete the directory's null version");
|
||||
let deleted = response
|
||||
@@ -839,7 +839,7 @@ async fn delete_objects_replication_config_failure_does_not_call_storage_delete(
|
||||
});
|
||||
|
||||
let err = usecase
|
||||
.execute_delete_objects(req)
|
||||
.execute_delete_objects_from_s3s(req)
|
||||
.await
|
||||
.expect_err("replication config failure must reject DeleteObjects");
|
||||
|
||||
|
||||
@@ -35,7 +35,7 @@ pub(crate) mod object_traffic_health;
|
||||
pub mod object_usecase;
|
||||
pub(crate) mod runtime_sources;
|
||||
mod select_object;
|
||||
mod select_request;
|
||||
pub(crate) mod select_request;
|
||||
pub(crate) mod storage_api;
|
||||
pub(crate) mod table_list_isolation;
|
||||
pub(crate) mod trailer_adapter;
|
||||
|
||||
@@ -49,7 +49,9 @@ use super::storage_api::multipart_usecase::io::{DecryptReader, EncryptReader, Ha
|
||||
use super::storage_api::multipart_usecase::io::{
|
||||
EtagAlgorithm, HashReader, WriteEncryption, WritePlan, compression_metadata_value,
|
||||
};
|
||||
use super::storage_api::multipart_usecase::object_utils::{to_s3s_etag, to_s3s_xml_etag};
|
||||
#[cfg(test)]
|
||||
use super::storage_api::multipart_usecase::object_utils::to_s3s_etag;
|
||||
use super::storage_api::multipart_usecase::object_utils::to_s3s_xml_etag;
|
||||
use super::storage_api::multipart_usecase::options::{
|
||||
copy_src_opts, extract_metadata_from_mime, get_complete_multipart_upload_opts_with_replication_authorization,
|
||||
get_content_sha256_with_query, get_opts, has_replication_retention_update, namespace_reserved_user_metadata,
|
||||
@@ -57,8 +59,7 @@ use super::storage_api::multipart_usecase::options::{
|
||||
};
|
||||
use super::storage_api::multipart_usecase::request_context::spawn_traced_join;
|
||||
use super::storage_api::multipart_usecase::s3_api::multipart::{
|
||||
ListMultipartUploadsParams, build_list_parts_output, parse_list_multipart_uploads_params, parse_list_parts_params,
|
||||
parse_upload_part_number,
|
||||
ListMultipartUploadsParams, parse_list_multipart_uploads_params, parse_list_parts_params, parse_upload_part_number,
|
||||
};
|
||||
use super::storage_api::multipart_usecase::set_disk::is_valid_storage_class;
|
||||
use super::storage_api::multipart_usecase::sse::{
|
||||
@@ -73,7 +74,6 @@ use super::storage_api::multipart_usecase::{
|
||||
StorageObjectInfo as ObjectInfo, StorageObjectOptions as ObjectOptions, StoragePutObjReader as PutObjReader,
|
||||
validate_new_object_key,
|
||||
};
|
||||
use super::trailer_adapter::trailer_source;
|
||||
use crate::app::object::{
|
||||
ConcurrencyManager, ForegroundWriteAdmission, get_concurrency_manager, put_object_body_read_timeout,
|
||||
reject_non_wildcard_write_if_none_match, reject_oversize_single_upload, reject_veeam_sosapi_mutation,
|
||||
@@ -83,8 +83,7 @@ use crate::app::object_data_cache::{
|
||||
invalidate_object_data_cache_before_mutation,
|
||||
};
|
||||
use crate::app::object_usecase::{
|
||||
apply_quota_admission, build_put_like_object_lock_metadata, map_quota_check_outcome, s3s_body_error_to_io,
|
||||
validate_existing_object_lock_for_write,
|
||||
apply_quota_admission, map_quota_check_outcome, s3s_body_error_to_io, validate_existing_object_lock_for_write,
|
||||
};
|
||||
use crate::app::runtime_sources::{
|
||||
AppContext, current_app_context, current_object_data_cache_for_context, current_object_store_handle_for_context,
|
||||
@@ -103,7 +102,6 @@ use bytes::Bytes;
|
||||
use futures::StreamExt;
|
||||
use http::{HeaderMap, HeaderValue, Uri};
|
||||
use metrics::counter;
|
||||
use rustfs_io_metrics::record_s3_op;
|
||||
use rustfs_s3_ops::S3Operation;
|
||||
use rustfs_targets::EventName;
|
||||
use rustfs_utils::CompressionAlgorithm;
|
||||
@@ -127,13 +125,21 @@ use urlencoding::encode;
|
||||
use uuid::Uuid;
|
||||
|
||||
mod abort;
|
||||
pub(crate) use abort::{AbortMultipartUploadTarget, reject_abort_capabilities};
|
||||
mod complete;
|
||||
mod create;
|
||||
mod create_values;
|
||||
pub(crate) use create_values::{CreateMultipartUploadFields, CreateMultipartUploadResult};
|
||||
mod list;
|
||||
#[cfg(feature = "gateway-stack")]
|
||||
pub(crate) use list::ListPartsInfo;
|
||||
pub(crate) use list::{ListPartsFields, ListPartsTarget};
|
||||
#[cfg(test)]
|
||||
mod test_support;
|
||||
mod upload_part;
|
||||
mod upload_part_copy;
|
||||
mod upload_values;
|
||||
pub(crate) use upload_values::{UploadPartTarget, UploadPartWriteInput, UploadPartWriteOutput};
|
||||
|
||||
#[cfg(test)]
|
||||
fn merge_part_encryption_metadata(
|
||||
@@ -535,7 +541,7 @@ mod tests {
|
||||
.build()
|
||||
.expect("valid create request DTO");
|
||||
let error = make_usecase()
|
||||
.execute_create_multipart_upload(build_request(create, Method::POST))
|
||||
.execute_create_multipart_upload_from_s3s(build_request(create, Method::POST))
|
||||
.await
|
||||
.expect_err("reserved key must be rejected before accessing storage");
|
||||
assert_eq!(error.code(), &S3ErrorCode::InvalidArgument);
|
||||
|
||||
@@ -15,76 +15,179 @@
|
||||
//! AbortMultipartUpload.
|
||||
|
||||
use super::*;
|
||||
use crate::app::storage_api::s3::dto::{AbortMultipartUploadInput, AbortMultipartUploadOutput};
|
||||
use crate::app::storage_api::s3::{S3Error, S3ErrorCode, S3Request, S3Response, S3Result};
|
||||
use crate::app::RequestEnvelope;
|
||||
#[cfg(any(feature = "gateway-stack", test))]
|
||||
use crate::app::storage_api::multipart_usecase::contract::multipart::{MultipartAbortAuthorization, MultipartOwner};
|
||||
use http::StatusCode;
|
||||
#[cfg(any(feature = "gateway-stack", test))]
|
||||
use rustfs_gateway_types::{BucketName, ObjectKey, RecordedUpload, UploadIdClaim, resolve_upload, resolve_upload_async};
|
||||
use rustfs_s3_types::{S3Error, S3ErrorCode, S3Result};
|
||||
#[cfg(any(feature = "gateway-stack", test))]
|
||||
use std::sync::OnceLock;
|
||||
|
||||
impl DefaultMultipartUsecase {
|
||||
#[instrument(level = "debug", skip(self, req))]
|
||||
pub async fn execute_abort_multipart_upload(
|
||||
&self,
|
||||
req: S3Request<AbortMultipartUploadInput>,
|
||||
) -> S3Result<S3Response<AbortMultipartUploadOutput>> {
|
||||
reject_presigned_multipart_max_total_object_size_for_other_operation(
|
||||
&req.headers,
|
||||
req.uri.query(),
|
||||
req.extensions.get::<VerifiedSigV4Request>().is_some(),
|
||||
)?;
|
||||
reject_presigned_put_max_content_length_for_other_operation(
|
||||
&req.headers,
|
||||
req.uri.query(),
|
||||
req.extensions.get::<VerifiedPresignedRequest>().is_some(),
|
||||
)?;
|
||||
record_s3_op(S3Operation::AbortMultipartUpload);
|
||||
let mut opts = ObjectOptions::default();
|
||||
apply_bucket_generation_guard(&req, &req.input.bucket, &mut opts)?;
|
||||
let AbortMultipartUploadInput {
|
||||
bucket, key, upload_id, ..
|
||||
} = req.input;
|
||||
pub(crate) enum AbortMultipartUploadTarget {
|
||||
Legacy {
|
||||
bucket: String,
|
||||
key: String,
|
||||
upload_id: String,
|
||||
},
|
||||
#[cfg(any(feature = "gateway-stack", test))]
|
||||
Gateway {
|
||||
bucket: BucketName,
|
||||
key: ObjectKey,
|
||||
upload_id: UploadIdClaim,
|
||||
},
|
||||
}
|
||||
|
||||
reject_veeam_sosapi_mutation(|| self.object_store(), &bucket, &key).await?;
|
||||
|
||||
let Some(store) = self.object_store() else {
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
|
||||
};
|
||||
|
||||
// Per the AWS S3 API reference, a malformed upload ID is reported as
|
||||
// NoSuchUpload (404), the same as an upload that was aborted or
|
||||
// completed. See multipart_upload_api_error.
|
||||
match store
|
||||
.abort_multipart_upload(bucket.as_str(), key.as_str(), upload_id.as_str(), &opts)
|
||||
.await
|
||||
{
|
||||
Ok(_) => {
|
||||
rustfs_scanner::record_dirty_usage_object_from_producer(
|
||||
&bucket,
|
||||
&key,
|
||||
rustfs_scanner::SegmentInvalidationProducerIdentity::AbortMultipartUpload,
|
||||
);
|
||||
Ok(S3Response::new(AbortMultipartUploadOutput { ..Default::default() }))
|
||||
}
|
||||
Err(err) => Err(multipart_upload_api_error(err)),
|
||||
impl AbortMultipartUploadTarget {
|
||||
fn resource(&self) -> (&str, &str) {
|
||||
match self {
|
||||
Self::Legacy { bucket, key, .. } => (bucket, key),
|
||||
#[cfg(any(feature = "gateway-stack", test))]
|
||||
Self::Gateway { bucket, key, .. } => (bucket.as_str(), key.as_str()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::super::test_support::*;
|
||||
use super::*;
|
||||
pub(crate) fn reject_abort_capabilities(envelope: &RequestEnvelope) -> S3Result<()> {
|
||||
reject_presigned_multipart_max_total_object_size_for_other_operation(
|
||||
envelope.headers(),
|
||||
envelope.uri().query(),
|
||||
envelope.verified_sigv4().is_some(),
|
||||
)?;
|
||||
reject_presigned_put_max_content_length_for_other_operation(
|
||||
envelope.headers(),
|
||||
envelope.uri().query(),
|
||||
envelope.verified_presigned().is_some(),
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_abort_multipart_upload_returns_internal_error_when_store_uninitialized() {
|
||||
let input = AbortMultipartUploadInput::builder()
|
||||
.bucket("bucket".to_string())
|
||||
.key("object".to_string())
|
||||
.upload_id("upload-id".to_string())
|
||||
.build()
|
||||
.unwrap();
|
||||
let req = build_request(input, Method::DELETE);
|
||||
let before = s3_op_total(S3Operation::AbortMultipartUpload);
|
||||
impl DefaultMultipartUsecase {
|
||||
#[instrument(level = "debug", skip(self, envelope, target))]
|
||||
pub(crate) async fn execute_abort_multipart_upload(
|
||||
&self,
|
||||
envelope: RequestEnvelope,
|
||||
target: S3Result<AbortMultipartUploadTarget>,
|
||||
) -> S3Result<()> {
|
||||
RequestEnvelope::audit_only_operation(
|
||||
envelope,
|
||||
S3Operation::AbortMultipartUpload,
|
||||
StatusCode::OK,
|
||||
|envelope| async move {
|
||||
let target = target?;
|
||||
let (bucket, key) = target.resource();
|
||||
let mut opts = ObjectOptions::default();
|
||||
apply_bucket_generation_guard(&envelope, bucket, &mut opts)?;
|
||||
#[cfg(any(feature = "gateway-stack", test))]
|
||||
if matches!(&target, AbortMultipartUploadTarget::Gateway { .. }) && opts.expected_bucket_incarnation_id.is_none()
|
||||
{
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "bucket generation guard is missing"));
|
||||
}
|
||||
|
||||
let err = make_usecase().execute_abort_multipart_upload(req).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
assert_eq!(s3_op_total(S3Operation::AbortMultipartUpload), before + 1);
|
||||
reject_veeam_sosapi_mutation(|| self.object_store(), bucket, key).await?;
|
||||
let Some(store) = self.object_store() else {
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init"));
|
||||
};
|
||||
|
||||
match &target {
|
||||
AbortMultipartUploadTarget::Legacy { upload_id, .. } => store
|
||||
.abort_multipart_upload(bucket, key, upload_id, &opts)
|
||||
.await
|
||||
.map_err(multipart_upload_api_error)?,
|
||||
#[cfg(any(feature = "gateway-stack", test))]
|
||||
AbortMultipartUploadTarget::Gateway { bucket, key, upload_id } => {
|
||||
abort_gateway(&store, bucket, key, upload_id, opts).await?
|
||||
}
|
||||
}
|
||||
rustfs_scanner::record_dirty_usage_object_from_producer(
|
||||
bucket,
|
||||
key,
|
||||
rustfs_scanner::SegmentInvalidationProducerIdentity::AbortMultipartUpload,
|
||||
);
|
||||
Ok(())
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(any(feature = "gateway-stack", test))]
|
||||
struct AbortOwner(MultipartOwner);
|
||||
|
||||
#[cfg(any(feature = "gateway-stack", test))]
|
||||
impl RecordedUpload for AbortOwner {
|
||||
fn bucket(&self) -> &str {
|
||||
&self.0.bucket
|
||||
}
|
||||
|
||||
fn key(&self) -> &str {
|
||||
&self.0.object
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(any(feature = "gateway-stack", test))]
|
||||
struct AbortAuthorization {
|
||||
bucket: BucketName,
|
||||
key: ObjectKey,
|
||||
upload_id: UploadIdClaim,
|
||||
observed: OnceLock<MultipartOwner>,
|
||||
}
|
||||
|
||||
#[cfg(any(feature = "gateway-stack", test))]
|
||||
impl MultipartAbortAuthorization for AbortAuthorization {
|
||||
type Error = StorageError;
|
||||
|
||||
fn authorize(&self, owner: Option<&MultipartOwner>) -> Result<(), Self::Error> {
|
||||
let (_, owner) =
|
||||
resolve_upload(&self.upload_id, &self.bucket, &self.key, |_| owner.cloned().map(AbortOwner)).map_err(|_| {
|
||||
StorageError::InvalidUploadID(self.bucket.as_str().to_owned(), self.key.as_str().to_owned(), String::new())
|
||||
})?;
|
||||
self.observed.get_or_init(|| owner.0);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(any(feature = "gateway-stack", test))]
|
||||
async fn abort_gateway(
|
||||
store: &ECStore,
|
||||
bucket: &BucketName,
|
||||
key: &ObjectKey,
|
||||
upload_id: &UploadIdClaim,
|
||||
mut opts: ObjectOptions,
|
||||
) -> S3Result<()> {
|
||||
let authorization = Arc::new(AbortAuthorization {
|
||||
bucket: bucket.clone(),
|
||||
key: key.clone(),
|
||||
upload_id: upload_id.clone(),
|
||||
observed: OnceLock::new(),
|
||||
});
|
||||
opts.multipart_abort_authorization = Some(authorization.clone());
|
||||
// The outer exchange checks the claim's shape. Only the inner exchange, on the
|
||||
// selected metadata while holding the upload lock, authorizes deletion.
|
||||
resolve_upload_async(upload_id, bucket, key, |candidate| async move {
|
||||
store
|
||||
.abort_multipart_upload(bucket.as_str(), key.as_str(), candidate, &opts)
|
||||
.await
|
||||
.map_err(multipart_upload_api_error)?;
|
||||
let owner = authorization.observed.get().cloned().ok_or_else(|| {
|
||||
S3Error::with_message(S3ErrorCode::InternalError, "Abort authorization did not observe the upload owner")
|
||||
})?;
|
||||
Ok::<_, S3Error>(Some(AbortOwner(owner)))
|
||||
})
|
||||
.await?
|
||||
.map_err(|_| {
|
||||
// An empty id has the legacy missing-upload response. Other impossible shapes
|
||||
// have the malformed-upload response. Neither reaches the storage lookup.
|
||||
let error = if upload_id.is_explicitly_empty() {
|
||||
StorageError::InvalidUploadID(bucket.as_str().to_owned(), key.as_str().to_owned(), String::new())
|
||||
} else {
|
||||
StorageError::MalformedUploadID(String::new())
|
||||
};
|
||||
multipart_upload_api_error(error)
|
||||
})?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
|
||||
@@ -0,0 +1,205 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::super::test_support::*;
|
||||
use super::*;
|
||||
use crate::app::storage_api::multipart_usecase::access::{AuthorizationRequest, load_bucket_generation_from_store};
|
||||
use crate::app::storage_api::multipart_usecase::helper::MutationCompletionProbe;
|
||||
use crate::app::storage_api::multipart_usecase::request_context::RequestContext;
|
||||
use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, DeleteBucketOptions, MakeBucketOptions};
|
||||
use crate::runtime_sources::ServerContextSlot;
|
||||
use rustfs_scanner::{SegmentInvalidationProducerIdentity, set_scanner_dirty_usage_mutation_observer};
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_abort_multipart_upload_returns_internal_error_when_store_uninitialized() {
|
||||
let request = build_request((), Method::DELETE);
|
||||
let envelope = RequestEnvelope::from_s3s(&request);
|
||||
let before = s3_op_total(S3Operation::AbortMultipartUpload);
|
||||
let target = AbortMultipartUploadTarget::Legacy {
|
||||
bucket: "bucket".to_owned(),
|
||||
key: "object".to_owned(),
|
||||
upload_id: "upload-id".to_owned(),
|
||||
};
|
||||
|
||||
let err = make_usecase()
|
||||
.execute_abort_multipart_upload(envelope, Ok(target))
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
assert_eq!(err.message(), Some("Not init"));
|
||||
assert_eq!(s3_op_total(S3Operation::AbortMultipartUpload), before);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn native_abort_requires_current_generation_and_audits_each_result() {
|
||||
crate::app::gating_test_env::run_large_stack_test("abort-multipart-generation", || async {
|
||||
let context = crate::app::gating_test_env::shared_gating_ambient().await;
|
||||
let store = context.object_store();
|
||||
let usecase = DefaultMultipartUsecase::with_context(Some(context.clone()));
|
||||
let slot = ServerContextSlot::new();
|
||||
assert!(slot.install(context));
|
||||
let bucket = format!("abort-multipart-{}", Uuid::new_v4().simple());
|
||||
let other = format!("abort-multipart-{}", Uuid::new_v4().simple());
|
||||
let key = "object";
|
||||
let opts = ObjectOptions::default();
|
||||
let request_context = RequestContext::fallback();
|
||||
let request = || {
|
||||
let mut req = build_request((), Method::DELETE);
|
||||
req.uri = format!("/{bucket}/{key}").parse().expect("fixture URI");
|
||||
req.extensions.insert(slot.clone());
|
||||
req.extensions.insert(request_context.clone());
|
||||
req
|
||||
};
|
||||
for name in [&bucket, &other] {
|
||||
store
|
||||
.make_bucket(name, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
}
|
||||
let original = store
|
||||
.bucket_incarnation_id(&bucket)
|
||||
.await
|
||||
.expect("previous bucket incarnation");
|
||||
let mut stale = request();
|
||||
let captured = load_bucket_generation_from_store(&store, &stale, &bucket)
|
||||
.await
|
||||
.expect("capture prior bucket");
|
||||
stale.install_bucket_generation(captured);
|
||||
store
|
||||
.delete_bucket(&bucket, &DeleteBucketOptions::default())
|
||||
.await
|
||||
.expect("delete original bucket");
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("recreate bucket");
|
||||
assert_ne!(
|
||||
store
|
||||
.bucket_incarnation_id(&bucket)
|
||||
.await
|
||||
.expect("current bucket incarnation"),
|
||||
original
|
||||
);
|
||||
|
||||
let upload = store
|
||||
.new_multipart_upload(&bucket, key, &opts)
|
||||
.await
|
||||
.expect("create current upload");
|
||||
let mut reader = PutObjReader::from_vec(b"current part".to_vec());
|
||||
store
|
||||
.put_object_part(&bucket, key, &upload.upload_id, 1, &mut reader, &opts)
|
||||
.await
|
||||
.expect("stage current part");
|
||||
let target = || AbortMultipartUploadTarget::Gateway {
|
||||
bucket: BucketName::new(bucket.clone()).expect("bucket name"),
|
||||
key: ObjectKey::new(key).expect("object key"),
|
||||
upload_id: UploadIdClaim::from_wire(upload.upload_id.as_str()),
|
||||
};
|
||||
|
||||
let dirty = Arc::new(AtomicUsize::new(0));
|
||||
let observer = {
|
||||
let dirty = dirty.clone();
|
||||
let bucket = bucket.clone();
|
||||
Arc::new(move |observed_bucket: &str, observed_key: &str, producer| {
|
||||
if observed_bucket == bucket
|
||||
&& observed_key == key
|
||||
&& producer == SegmentInvalidationProducerIdentity::AbortMultipartUpload
|
||||
{
|
||||
dirty.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
}) as rustfs_scanner::ScannerDirtyUsageMutationObserver
|
||||
};
|
||||
struct RestoreObserver(Option<rustfs_scanner::ScannerDirtyUsageMutationObserver>);
|
||||
impl Drop for RestoreObserver {
|
||||
fn drop(&mut self) {
|
||||
set_scanner_dirty_usage_mutation_observer(self.0.take());
|
||||
}
|
||||
}
|
||||
let _observer = RestoreObserver(set_scanner_dirty_usage_mutation_observer(Some(observer)));
|
||||
let probe = MutationCompletionProbe::install(&bucket);
|
||||
|
||||
let missing = request();
|
||||
let mut wrong = request();
|
||||
let captured = load_bucket_generation_from_store(&store, &wrong, &other)
|
||||
.await
|
||||
.expect("capture unrelated bucket");
|
||||
wrong.install_bucket_generation(captured);
|
||||
for (req, code) in [
|
||||
(stale, S3ErrorCode::NoSuchBucket),
|
||||
(missing, S3ErrorCode::InternalError),
|
||||
(wrong, S3ErrorCode::InternalError),
|
||||
] {
|
||||
let err = usecase
|
||||
.execute_abort_multipart_upload(RequestEnvelope::from_s3s(&req), Ok(target()))
|
||||
.await
|
||||
.expect_err("Abort requires current generation");
|
||||
assert_eq!(err.code(), &code);
|
||||
let parts = store
|
||||
.list_object_parts(&bucket, key, &upload.upload_id, None, 10, &opts)
|
||||
.await
|
||||
.expect("current upload persists");
|
||||
assert_eq!(parts.parts.len(), 1, "current staged part persists");
|
||||
}
|
||||
assert_eq!(dirty.load(Ordering::Relaxed), 0);
|
||||
assert_eq!(probe.audits.lock().unwrap().len(), 3);
|
||||
|
||||
let mut current = request();
|
||||
let captured = load_bucket_generation_from_store(&store, ¤t, &bucket)
|
||||
.await
|
||||
.expect("capture current bucket");
|
||||
current.install_bucket_generation(captured);
|
||||
usecase
|
||||
.execute_abort_multipart_upload(RequestEnvelope::from_s3s(¤t), Ok(target()))
|
||||
.await
|
||||
.expect("current generation allows Abort");
|
||||
assert!(matches!(
|
||||
store.get_multipart_info(&bucket, key, &upload.upload_id, &opts).await,
|
||||
Err(StorageError::InvalidUploadID(..))
|
||||
));
|
||||
assert_eq!(dirty.load(Ordering::Relaxed), 1);
|
||||
{
|
||||
let audits = probe.audits.lock().unwrap();
|
||||
assert_eq!(audits.len(), 4);
|
||||
assert_eq!(audits.last().expect("successful audit").api.status_code, Some(200));
|
||||
}
|
||||
|
||||
let legacy = store
|
||||
.new_multipart_upload(&bucket, key, &opts)
|
||||
.await
|
||||
.expect("create legacy-path upload");
|
||||
usecase
|
||||
.execute_abort_multipart_upload(
|
||||
RequestEnvelope::from_s3s(¤t),
|
||||
Ok(AbortMultipartUploadTarget::Legacy {
|
||||
bucket: bucket.clone(),
|
||||
key: key.to_owned(),
|
||||
upload_id: legacy.upload_id,
|
||||
}),
|
||||
)
|
||||
.await
|
||||
.expect("legacy target with the same authorized generation");
|
||||
assert_eq!(dirty.load(Ordering::Relaxed), 2);
|
||||
|
||||
let audits = probe.audits.lock().unwrap();
|
||||
assert_eq!(audits.len(), 5, "one audit per owned Abort");
|
||||
assert_eq!(audits.last().expect("legacy success audit").api.status_code, Some(200));
|
||||
for audit in audits.iter() {
|
||||
assert_eq!(audit.request_id.as_deref(), Some(request_context.request_id.as_str()));
|
||||
assert_eq!(audit.api.name.as_deref(), Some(S3Operation::AbortMultipartUpload.as_str()));
|
||||
}
|
||||
assert!(probe.events.lock().unwrap().is_empty());
|
||||
});
|
||||
}
|
||||
@@ -1026,7 +1026,7 @@ mod tests {
|
||||
..Default::default()
|
||||
});
|
||||
let upload_id = usecase
|
||||
.execute_create_multipart_upload(create_request)
|
||||
.execute_create_multipart_upload_from_s3s(create_request)
|
||||
.await
|
||||
.expect("authorized REPLICA-only create should succeed")
|
||||
.output
|
||||
@@ -1124,7 +1124,7 @@ mod tests {
|
||||
.headers
|
||||
.insert("x-amz-checksum-type", HeaderValue::from_static("COMPOSITE"));
|
||||
let upload_id = usecase
|
||||
.execute_create_multipart_upload(create)
|
||||
.execute_create_multipart_upload_from_s3s(create)
|
||||
.await
|
||||
.expect("SSE-C create")
|
||||
.output
|
||||
@@ -1514,7 +1514,7 @@ mod tests {
|
||||
.headers
|
||||
.insert(http::header::IF_NONE_MATCH, HeaderValue::from_static("*"));
|
||||
let first_upload_id = usecase
|
||||
.execute_create_multipart_upload(initial_create_req)
|
||||
.execute_create_multipart_upload_from_s3s(initial_create_req)
|
||||
.await
|
||||
.expect("initial conditional create multipart upload should succeed")
|
||||
.output
|
||||
@@ -1561,13 +1561,13 @@ mod tests {
|
||||
.headers
|
||||
.insert(http::header::IF_NONE_MATCH, HeaderValue::from_static("*"));
|
||||
let create_err = usecase
|
||||
.execute_create_multipart_upload(second_create_req)
|
||||
.execute_create_multipart_upload_from_s3s(second_create_req)
|
||||
.await
|
||||
.expect_err("create multipart upload with If-None-Match: * on existing key must fail");
|
||||
assert_eq!(create_err.code(), &crate::app::storage_api::s3::S3ErrorCode::PreconditionFailed);
|
||||
|
||||
let second_upload_id = usecase
|
||||
.execute_create_multipart_upload(build_request(create_input, Method::POST))
|
||||
.execute_create_multipart_upload_from_s3s(build_request(create_input, Method::POST))
|
||||
.await
|
||||
.expect("unconditional create multipart upload should succeed")
|
||||
.output
|
||||
@@ -1770,7 +1770,7 @@ mod tests {
|
||||
.headers
|
||||
.insert(http::header::IF_NONE_MATCH, HeaderValue::from_static("*"));
|
||||
let dm_upload_id = usecase
|
||||
.execute_create_multipart_upload(dm_create_req)
|
||||
.execute_create_multipart_upload_from_s3s(dm_create_req)
|
||||
.await
|
||||
.expect("If-None-Match: * create must succeed past a delete marker")
|
||||
.output
|
||||
@@ -2255,9 +2255,16 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
async fn completion_absent_binding_refusal_order_baseline() {
|
||||
fn completion_absent_binding_refusal_order_baseline() {
|
||||
crate::app::gating_test_env::run_large_stack_test(
|
||||
"completion-absent-binding-refusal-order",
|
||||
completion_absent_binding_refusal_order_baseline_inner,
|
||||
);
|
||||
}
|
||||
|
||||
async fn completion_absent_binding_refusal_order_baseline_inner() {
|
||||
use crate::app::storage_api::test::bucket::quota::BucketQuota;
|
||||
use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions};
|
||||
|
||||
|
||||
@@ -15,16 +15,15 @@
|
||||
//! CreateMultipartUpload.
|
||||
|
||||
use super::*;
|
||||
use crate::app::storage_api::s3::dto::{
|
||||
ChecksumAlgorithm, ChecksumType, CreateMultipartUploadInput, CreateMultipartUploadOutput,
|
||||
};
|
||||
use crate::app::storage_api::s3::{S3Error, S3ErrorCode, S3Request, S3Response, S3Result, s3_error};
|
||||
use crate::app::RequestEnvelope;
|
||||
use crate::app::object::shared::build_put_like_object_lock_metadata_native;
|
||||
use rustfs_s3_types::{S3Error, S3ErrorCode, S3Result, s3_error};
|
||||
|
||||
fn create_multipart_upload_metadata(
|
||||
input_metadata: Option<HashMap<String, String>>,
|
||||
headers: &HeaderMap,
|
||||
tagging: Option<String>,
|
||||
storage_class: Option<&crate::app::storage_api::s3::dto::StorageClass>,
|
||||
storage_class: Option<&str>,
|
||||
) -> HashMap<String, String> {
|
||||
let mut metadata = input_metadata.unwrap_or_default();
|
||||
namespace_reserved_user_metadata(&mut metadata);
|
||||
@@ -34,7 +33,7 @@ fn create_multipart_upload_metadata(
|
||||
metadata.insert(AMZ_OBJECT_TAGGING.to_owned(), tags);
|
||||
}
|
||||
if let Some(storage_class) = storage_class {
|
||||
metadata.insert(AMZ_STORAGE_CLASS.to_owned(), storage_class.as_str().to_owned());
|
||||
metadata.insert(AMZ_STORAGE_CLASS.to_owned(), storage_class.to_owned());
|
||||
}
|
||||
|
||||
metadata
|
||||
@@ -63,32 +62,37 @@ fn should_advertise_session_compression(multipart_enabled: bool, ciphertext_pass
|
||||
}
|
||||
|
||||
impl DefaultMultipartUsecase {
|
||||
#[instrument(level = "debug", skip(self, req))]
|
||||
#[instrument(level = "debug", skip(self, envelope, input, query_version_id))]
|
||||
#[hotpath::measure(impl_type = "MultipartUsecase")]
|
||||
pub async fn execute_create_multipart_upload(
|
||||
pub(crate) async fn execute_create_multipart_upload(
|
||||
&self,
|
||||
req: S3Request<CreateMultipartUploadInput>,
|
||||
) -> S3Result<S3Response<CreateMultipartUploadOutput>> {
|
||||
validate_new_object_key(&req.input.key, "CreateMultipartUpload")?;
|
||||
envelope: RequestEnvelope,
|
||||
input: CreateMultipartUploadFields,
|
||||
query_version_id: Option<String>,
|
||||
) -> S3Result<CreateMultipartUploadResult> {
|
||||
validate_new_object_key(&input.key, "CreateMultipartUpload")?;
|
||||
let headers = envelope.headers();
|
||||
let multipart_max_total_object_size = parse_presigned_multipart_max_total_object_size(
|
||||
&req.headers,
|
||||
req.uri.query(),
|
||||
req.extensions.get::<VerifiedSigV4Request>().is_some(),
|
||||
headers,
|
||||
envelope.uri().query(),
|
||||
envelope.verified_sigv4().is_some(),
|
||||
)?;
|
||||
reject_presigned_put_max_content_length_for_other_operation(
|
||||
&req.headers,
|
||||
req.uri.query(),
|
||||
req.extensions.get::<VerifiedPresignedRequest>().is_some(),
|
||||
headers,
|
||||
envelope.uri().query(),
|
||||
envelope.verified_presigned().is_some(),
|
||||
)?;
|
||||
let helper =
|
||||
OperationHelper::new(&req, EventName::ObjectCreatedCreateMultipartUpload, S3Operation::CreateMultipartUpload)
|
||||
.suppress_event();
|
||||
let replication_authorized = replication_request_authorized(&req);
|
||||
let CreateMultipartUploadInput {
|
||||
let helper = OperationHelper::from_view(
|
||||
&envelope.audit_view(),
|
||||
EventName::ObjectCreatedCreateMultipartUpload,
|
||||
S3Operation::CreateMultipartUpload,
|
||||
)
|
||||
.suppress_event();
|
||||
let replication_authorized = replication_request_authorized(&envelope);
|
||||
let CreateMultipartUploadFields {
|
||||
bucket,
|
||||
key,
|
||||
tagging,
|
||||
version_id,
|
||||
storage_class,
|
||||
server_side_encryption,
|
||||
sse_customer_algorithm,
|
||||
@@ -99,24 +103,21 @@ impl DefaultMultipartUsecase {
|
||||
object_lock_mode,
|
||||
object_lock_retain_until_date,
|
||||
metadata: input_metadata,
|
||||
..
|
||||
} = req.input.clone();
|
||||
} = input;
|
||||
|
||||
let server_side_encryption = server_side_encryption
|
||||
.map(|value| value.as_str().to_owned())
|
||||
.or(extract_server_side_encryption_from_headers(&req.headers)?);
|
||||
let server_side_encryption = server_side_encryption.or(extract_server_side_encryption_from_headers(headers)?);
|
||||
let ssekms_key_id = ssekms_key_id.or_else(|| {
|
||||
req.headers
|
||||
headers
|
||||
.get("x-amz-server-side-encryption-aws-kms-key-id")
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.map(ToOwned::to_owned)
|
||||
});
|
||||
|
||||
// Validate storage class if provided
|
||||
if let Some(ref storage_class) = storage_class
|
||||
&& !is_valid_storage_class(storage_class.as_str())
|
||||
if let Some(storage_class) = storage_class.as_deref()
|
||||
&& !is_valid_storage_class(storage_class)
|
||||
{
|
||||
return Err(s3_error!(InvalidStorageClass));
|
||||
return Err(s3_error!(InvalidStorageClass, "The storage class you specified is not valid."));
|
||||
}
|
||||
|
||||
reject_veeam_sosapi_mutation(|| self.object_store(), &bucket, &key).await?;
|
||||
@@ -128,11 +129,11 @@ impl DefaultMultipartUsecase {
|
||||
|
||||
validate_archive_content_encoding(
|
||||
&key,
|
||||
req.headers.get("content-type").and_then(|value| value.to_str().ok()),
|
||||
req.headers.get("content-encoding").and_then(|value| value.to_str().ok()),
|
||||
headers.get("content-type").and_then(|value| value.to_str().ok()),
|
||||
headers.get("content-encoding").and_then(|value| value.to_str().ok()),
|
||||
)?;
|
||||
|
||||
let mut metadata = create_multipart_upload_metadata(input_metadata, &req.headers, tagging, storage_class.as_ref());
|
||||
let mut metadata = create_multipart_upload_metadata(input_metadata, headers, tagging, storage_class.as_deref());
|
||||
if let Some(limit) = multipart_max_total_object_size {
|
||||
insert_str(&mut metadata, SUFFIX_MAX_TOTAL_OBJECT_SIZE, limit.to_string());
|
||||
}
|
||||
@@ -142,9 +143,9 @@ impl DefaultMultipartUsecase {
|
||||
|
||||
let has_explicit_object_lock_retention = object_lock_mode.is_some()
|
||||
|| object_lock_retain_until_date.is_some()
|
||||
|| has_replication_retention_update(&req.headers, replication_authorized);
|
||||
|| has_replication_retention_update(headers, replication_authorized);
|
||||
let object_lock_config_state = load_bucket_object_lock_config_state(&bucket).await?;
|
||||
if let Some(object_lock_metadata) = build_put_like_object_lock_metadata(
|
||||
if let Some(object_lock_metadata) = build_put_like_object_lock_metadata_native(
|
||||
&bucket,
|
||||
&object_lock_config_state,
|
||||
object_lock_legal_hold_status,
|
||||
@@ -160,20 +161,20 @@ impl DefaultMultipartUsecase {
|
||||
has_explicit_object_lock_retention,
|
||||
)?;
|
||||
let (header_sse_customer_algorithm, header_sse_customer_key, header_sse_customer_key_md5) =
|
||||
extract_ssec_params_from_headers(&req.headers)?;
|
||||
extract_ssec_params_from_headers(headers)?;
|
||||
let sse_customer_algorithm = sse_customer_algorithm.or(header_sse_customer_algorithm);
|
||||
let sse_customer_key = sse_customer_key.or(header_sse_customer_key);
|
||||
let sse_customer_key_md5 = sse_customer_key_md5.or(header_sse_customer_key_md5);
|
||||
|
||||
// The session data key is generated here, so this is where a multipart upload is held
|
||||
// to the KMS key it names. Parts and completion also authorize their own caller.
|
||||
let session_principal = SseKmsPrincipal::from_request(&req);
|
||||
let session_principal = SseKmsPrincipal::from_request(&envelope);
|
||||
let encryption_request = PrepareEncryptionRequest {
|
||||
bucket: &bucket,
|
||||
key: &key,
|
||||
server_side_encryption,
|
||||
ssekms_key_id,
|
||||
ssekms_context: extract_ssekms_context_from_headers(&req.headers)?,
|
||||
ssekms_context: extract_ssekms_context_from_headers(headers)?,
|
||||
sse_customer_algorithm: sse_customer_algorithm.clone(),
|
||||
sse_customer_key,
|
||||
sse_customer_key_md5: sse_customer_key_md5.clone(),
|
||||
@@ -184,8 +185,8 @@ impl DefaultMultipartUsecase {
|
||||
// session DEK is prepared; a session marker tells UploadPart to store
|
||||
// the ciphertext verbatim instead of recovering encryption material.
|
||||
let ciphertext_passthrough = replication_authorized
|
||||
&& get_header(&req.headers, SUFFIX_SOURCE_REPLICATION_REQUEST).as_deref() == Some("true")
|
||||
&& rustfs_utils::http::ssec_transport_to_stored_metadata(&req.headers).is_some();
|
||||
&& get_header(headers, SUFFIX_SOURCE_REPLICATION_REQUEST).as_deref() == Some("true")
|
||||
&& rustfs_utils::http::ssec_transport_to_stored_metadata(headers).is_some();
|
||||
if ciphertext_passthrough && let Some(metadata_sys) = self.bucket_metadata_sys() {
|
||||
let check_result = map_quota_check_outcome(
|
||||
&bucket,
|
||||
@@ -228,7 +229,7 @@ impl DefaultMultipartUsecase {
|
||||
if should_advertise_session_compression(
|
||||
is_multipart_disk_compression_enabled(),
|
||||
ciphertext_passthrough,
|
||||
is_disk_compressible(&req.headers, &key),
|
||||
is_disk_compressible(headers, &key),
|
||||
) {
|
||||
rustfs_utils::http::insert_str(
|
||||
&mut metadata,
|
||||
@@ -239,10 +240,10 @@ impl DefaultMultipartUsecase {
|
||||
|
||||
let mt2 = metadata.clone();
|
||||
let mut opts: ObjectOptions =
|
||||
put_opts_with_replication_authorization(&bucket, &key, version_id, &req.headers, metadata, replication_authorized)
|
||||
put_opts_with_replication_authorization(&bucket, &key, query_version_id, headers, metadata, replication_authorized)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
apply_bucket_generation_guard(&req, &bucket, &mut opts)?;
|
||||
apply_bucket_generation_guard(&envelope, &bucket, &mut opts)?;
|
||||
|
||||
let dsc =
|
||||
must_replicate_object(&bucket, &key, &mt2, "".to_string(), opts.delete_marker_replication_status(), opts.clone())
|
||||
@@ -261,7 +262,7 @@ impl DefaultMultipartUsecase {
|
||||
&& super::super::object::shared::object_lock_checks_required_for_state(&object_lock_config_state)
|
||||
{
|
||||
let current_opts: ObjectOptions = internal_object_info_lookup_opts(
|
||||
get_opts(&bucket, &key, opts.version_id.clone(), None, &req.headers)
|
||||
get_opts(&bucket, &key, opts.version_id.clone(), None, headers)
|
||||
.await
|
||||
.map_err(ApiError::from)?,
|
||||
);
|
||||
@@ -277,7 +278,7 @@ impl DefaultMultipartUsecase {
|
||||
}
|
||||
}
|
||||
|
||||
let checksum_type = rustfs_rio::ChecksumType::from_header(&req.headers);
|
||||
let checksum_type = rustfs_rio::ChecksumType::from_header(headers);
|
||||
if checksum_type.is(rustfs_rio::ChecksumType::INVALID) {
|
||||
return Err(s3_error!(InvalidArgument, "Invalid checksum type"));
|
||||
} else if checksum_type.is_set() && !checksum_type.is(rustfs_rio::ChecksumType::TRAILING) {
|
||||
@@ -299,21 +300,20 @@ impl DefaultMultipartUsecase {
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
let output = CreateMultipartUploadOutput {
|
||||
bucket: Some(bucket),
|
||||
key: Some(key),
|
||||
upload_id: Some(upload_id),
|
||||
server_side_encryption: effective_sse.map(Into::into),
|
||||
let output = CreateMultipartUploadResult {
|
||||
bucket,
|
||||
key,
|
||||
upload_id,
|
||||
server_side_encryption: effective_sse,
|
||||
sse_customer_algorithm,
|
||||
sse_customer_key_md5,
|
||||
ssekms_key_id: effective_kms_key_id,
|
||||
checksum_algorithm: checksum_algo.map(ChecksumAlgorithm::from),
|
||||
checksum_type: checksum_type.map(ChecksumType::from),
|
||||
..Default::default()
|
||||
checksum_algorithm: checksum_algo,
|
||||
checksum_type,
|
||||
};
|
||||
|
||||
let result = Ok(S3Response::new(output));
|
||||
let _ = helper.complete(&result);
|
||||
let result = Ok(output);
|
||||
let _ = helper.complete_owned(&result, http::StatusCode::OK, &HeaderMap::new());
|
||||
result
|
||||
}
|
||||
}
|
||||
@@ -322,6 +322,51 @@ impl DefaultMultipartUsecase {
|
||||
mod tests {
|
||||
use super::super::test_support::*;
|
||||
use super::*;
|
||||
use crate::app::storage_api::s3::{S3ErrorCode, dto::CreateMultipartUploadInput};
|
||||
|
||||
#[tokio::test]
|
||||
async fn create_multipart_baseline_legacy_direct() {
|
||||
use crate::app::storage_api::s3::dto::Timestamp;
|
||||
|
||||
let date = time::Date::from_calendar_date(-1, time::Month::January, 1)
|
||||
.unwrap()
|
||||
.midnight()
|
||||
.assume_utc();
|
||||
let long_key = "k".repeat(1025);
|
||||
let invalid_class = ("InvalidStorageClass", "The storage class you specified is not valid.");
|
||||
for (bucket, key, invalid, date, expected) in [
|
||||
("ab", "object", true, None, invalid_class),
|
||||
("a/b", "object", true, None, invalid_class),
|
||||
("bucket", "", true, None, ("InvalidArgument", "Invalid object key")),
|
||||
(
|
||||
"bucket",
|
||||
"a\0b",
|
||||
true,
|
||||
None,
|
||||
("InvalidArgument", "Object key contains invalid control characters: \"a\\0b\""),
|
||||
),
|
||||
("bucket", long_key.as_str(), true, None, invalid_class),
|
||||
("bucket", "object", false, Some(date), ("InternalError", "Not init")),
|
||||
("bucket", "object", true, Some(date), invalid_class),
|
||||
] {
|
||||
let input = CreateMultipartUploadInput::builder()
|
||||
.bucket(bucket.to_owned())
|
||||
.key(key.to_owned())
|
||||
.storage_class(invalid.then(|| StorageClass::from("unknown-class".to_owned())))
|
||||
.object_lock_retain_until_date(date.map(Timestamp::from))
|
||||
.build()
|
||||
.expect("direct fixture DTO");
|
||||
let error = make_usecase()
|
||||
.execute_create_multipart_upload_from_s3s(build_request(input, Method::POST))
|
||||
.await
|
||||
.expect_err("direct fixture does not create an upload");
|
||||
assert_eq!(
|
||||
(error.code().as_str(), error.message()),
|
||||
(expected.0, Some(expected.1)),
|
||||
"{bucket:?}/{key:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn session_compression_is_advertised_only_for_non_passthrough_compressible_uploads() {
|
||||
@@ -358,18 +403,16 @@ mod tests {
|
||||
.unwrap();
|
||||
let req = build_request(input, Method::POST);
|
||||
|
||||
let err = make_usecase().execute_create_multipart_upload(req).await.unwrap_err();
|
||||
let err = make_usecase()
|
||||
.execute_create_multipart_upload_from_s3s(req)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidStorageClass);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn create_multipart_upload_metadata_persists_dto_storage_class_without_raw_header() {
|
||||
let metadata = create_multipart_upload_metadata(
|
||||
None,
|
||||
&HeaderMap::new(),
|
||||
None,
|
||||
Some(&StorageClass::from_static("REDUCED_REDUNDANCY")),
|
||||
);
|
||||
let metadata = create_multipart_upload_metadata(None, &HeaderMap::new(), None, Some("REDUCED_REDUNDANCY"));
|
||||
|
||||
assert_eq!(metadata.get(AMZ_STORAGE_CLASS), Some(&"REDUCED_REDUNDANCY".to_string()));
|
||||
}
|
||||
@@ -388,7 +431,7 @@ mod tests {
|
||||
Some(input_metadata),
|
||||
&headers,
|
||||
Some("project=rustfs".to_string()),
|
||||
Some(&StorageClass::from_static("REDUCED_REDUNDANCY")),
|
||||
Some("REDUCED_REDUNDANCY"),
|
||||
);
|
||||
|
||||
assert_eq!(metadata.get("content-type"), Some(&"application/octet-stream".to_string()));
|
||||
@@ -401,10 +444,10 @@ mod tests {
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn create_multipart_rejects_ciphertext_replication_before_parts_are_staged() {
|
||||
let (_store, bucket) = crate::app::gating_test_env::durable_quota_test_bucket("ciphertext-multipart-quota", 4096).await;
|
||||
let (store, bucket) = crate::app::gating_test_env::durable_quota_test_bucket("ciphertext-multipart-quota", 4096).await;
|
||||
let usecase = DefaultMultipartUsecase::from_global();
|
||||
let input = CreateMultipartUploadInput::builder()
|
||||
.bucket(bucket)
|
||||
.bucket(bucket.clone())
|
||||
.key("object".to_string())
|
||||
.build()
|
||||
.expect("create multipart request should build");
|
||||
@@ -418,10 +461,200 @@ mod tests {
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
let err = usecase
|
||||
.execute_create_multipart_upload(request)
|
||||
.await
|
||||
.expect_err("quota-enabled ciphertext multipart replication should fail before upload creation");
|
||||
let result = usecase.execute_create_multipart_upload_from_s3s(request).await;
|
||||
assert!(
|
||||
store
|
||||
.list_multipart_uploads(&bucket, "", None, None, None, 100)
|
||||
.await
|
||||
.expect("list durable uploads")
|
||||
.uploads
|
||||
.is_empty()
|
||||
);
|
||||
let err = result.expect_err("quota-enabled ciphertext multipart replication must be refused");
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest);
|
||||
assert_eq!(
|
||||
err.message(),
|
||||
Some("SSE-C ciphertext replication is unavailable for quota-enabled buckets")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn create_multipart_checks_kms_permission_before_creating_a_session() {
|
||||
crate::app::gating_test_env::run_large_stack_test("create-multipart-kms-policy", || async {
|
||||
use crate::app::storage_api::s3::dto::ServerSideEncryption;
|
||||
use crate::app::storage_api::test::{
|
||||
ReqInfo,
|
||||
sse::{RecordingKmsKeyAuthorizer, TestAuthorizationHooks},
|
||||
};
|
||||
use rustfs_policy::policy::action::KmsAction;
|
||||
|
||||
let (store, bucket) = crate::app::gating_test_env::durable_quota_test_bucket("create-multipart-kms", 16_384).await;
|
||||
let context = crate::app::runtime_sources::current_app_context().expect("test app context");
|
||||
let usecase = DefaultMultipartUsecase::with_context(Some(context.clone()));
|
||||
let key_id = "multipart-test-key";
|
||||
let kms = context.kms().handle();
|
||||
kms.reconfigure(rustfs_kms::config::KmsConfig::static_kms(
|
||||
key_id.to_owned(),
|
||||
base64_simd::STANDARD.encode_to_string([1_u8; 32]),
|
||||
))
|
||||
.await
|
||||
.expect("configure test KMS");
|
||||
for allowed in [false, true] {
|
||||
let input = CreateMultipartUploadInput {
|
||||
bucket: bucket.clone(),
|
||||
key: "object".to_owned(),
|
||||
server_side_encryption: Some(ServerSideEncryption::from_static("aws:kms")),
|
||||
ssekms_key_id: Some(key_id.to_owned()),
|
||||
..Default::default()
|
||||
};
|
||||
let authorizer = RecordingKmsKeyAuthorizer::new(allowed);
|
||||
let mut req = build_request(input, Method::POST);
|
||||
req.extensions.insert(ReqInfo {
|
||||
cred: Some(rustfs_credentials::Credentials {
|
||||
access_key: "test-kms-writer".to_owned(),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
});
|
||||
req.extensions.insert(TestAuthorizationHooks {
|
||||
enforced: true,
|
||||
authorizer: authorizer.clone(),
|
||||
});
|
||||
let result = usecase.execute_create_multipart_upload_from_s3s(req).await;
|
||||
let uploads = store
|
||||
.list_multipart_uploads(&bucket, "", None, None, None, 100)
|
||||
.await
|
||||
.expect("list uploads after KMS decision")
|
||||
.uploads;
|
||||
assert_eq!(uploads.len(), usize::from(allowed));
|
||||
assert_eq!(authorizer.calls(), [(KmsAction::GenerateDataKeyAction, key_id.to_owned())]);
|
||||
if allowed {
|
||||
let output = result.expect("KMS authorizes the selected key").output;
|
||||
assert_eq!(output.upload_id.as_deref(), Some(uploads[0].upload_id.as_str()));
|
||||
} else {
|
||||
let error = result.expect_err("KMS denied the selected key");
|
||||
assert_eq!(error.code(), &S3ErrorCode::AccessDenied);
|
||||
}
|
||||
}
|
||||
kms.stop().await.expect("stop test KMS");
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn create_multipart_preserves_legacy_date_precision() {
|
||||
use time::{Date, Month, Time};
|
||||
|
||||
let date = Date::from_calendar_date(-1, Month::January, 1)
|
||||
.unwrap()
|
||||
.with_time(Time::from_hms_nano(0, 0, 0, 123_456_789).unwrap())
|
||||
.assume_utc();
|
||||
let input = CreateMultipartUploadInput {
|
||||
bucket: "sthree-multipart".to_owned(),
|
||||
key: "object".to_owned(),
|
||||
object_lock_retain_until_date: Some(date.into()),
|
||||
..Default::default()
|
||||
};
|
||||
let fields = CreateMultipartUploadFields::from_legacy(input);
|
||||
assert_eq!(fields.object_lock_retain_until_date, Some(date));
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn create_multipart_generation_fences_the_durable_upload() {
|
||||
crate::app::gating_test_env::run_large_stack_test("create-multipart-generation", || async {
|
||||
use crate::app::storage_api::multipart_usecase::access::{AuthorizationRequest, load_bucket_generation_from_store};
|
||||
use crate::app::storage_api::test::contract::bucket::{
|
||||
BucketOperations as _, DeleteBucketOptions, MakeBucketOptions,
|
||||
};
|
||||
use crate::runtime_sources::ServerContextSlot;
|
||||
|
||||
let context = crate::app::gating_test_env::shared_gating_ambient().await;
|
||||
let store = context.object_store();
|
||||
let usecase = DefaultMultipartUsecase::with_context(Some(context.clone()));
|
||||
let slot = ServerContextSlot::new();
|
||||
assert!(slot.install(context));
|
||||
let bucket = format!("create-multipart-{}", Uuid::new_v4().simple());
|
||||
let other = format!("create-multipart-{}", Uuid::new_v4().simple());
|
||||
let request = |bucket: &str| {
|
||||
let input = CreateMultipartUploadInput {
|
||||
bucket: bucket.to_owned(),
|
||||
key: "object".to_owned(),
|
||||
..Default::default()
|
||||
};
|
||||
let mut request = build_request(input, Method::POST);
|
||||
request.uri = format!("/{bucket}/object?uploads").parse().expect("fixture URI");
|
||||
request.extensions.insert(slot.clone());
|
||||
request
|
||||
};
|
||||
for name in [&bucket, &other] {
|
||||
store
|
||||
.make_bucket(name, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
}
|
||||
|
||||
let mut stale = request(&bucket);
|
||||
let guard = load_bucket_generation_from_store(&store, &stale, &bucket)
|
||||
.await
|
||||
.expect("capture original incarnation");
|
||||
stale.install_bucket_generation(guard);
|
||||
let original = store.bucket_incarnation_id(&bucket).await.expect("old bucket generation");
|
||||
|
||||
store
|
||||
.delete_bucket(&bucket, &DeleteBucketOptions::default())
|
||||
.await
|
||||
.expect("delete original bucket");
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("recreate bucket");
|
||||
let current = store.bucket_incarnation_id(&bucket).await.expect("new bucket generation");
|
||||
assert_ne!(original, current);
|
||||
|
||||
let missing = request(&bucket);
|
||||
let mut wrong = request(&bucket);
|
||||
let guard = load_bucket_generation_from_store(&store, &wrong, &other)
|
||||
.await
|
||||
.expect("capture the other bucket");
|
||||
wrong.install_bucket_generation(guard);
|
||||
for (request, code) in [
|
||||
(stale, S3ErrorCode::NoSuchBucket),
|
||||
(missing, S3ErrorCode::InternalError),
|
||||
(wrong, S3ErrorCode::InternalError),
|
||||
] {
|
||||
let result = usecase.execute_create_multipart_upload_from_s3s(request).await;
|
||||
assert!(
|
||||
store
|
||||
.list_multipart_uploads(&bucket, "", None, None, None, 100)
|
||||
.await
|
||||
.expect("list uploads in the recreated bucket")
|
||||
.uploads
|
||||
.is_empty()
|
||||
);
|
||||
let error = result.expect_err("creation requires the target incarnation");
|
||||
assert_eq!(error.code(), &code);
|
||||
}
|
||||
|
||||
let probe = crate::app::storage_api::multipart_usecase::helper::MutationCompletionProbe::install(&bucket);
|
||||
let mut allowed = request(&bucket);
|
||||
let guard = load_bucket_generation_from_store(&store, &allowed, &bucket)
|
||||
.await
|
||||
.expect("capture current incarnation");
|
||||
allowed.install_bucket_generation(guard);
|
||||
let response = usecase
|
||||
.execute_create_multipart_upload_from_s3s(allowed)
|
||||
.await
|
||||
.expect("a current guard allows creation");
|
||||
let uploads = store
|
||||
.list_multipart_uploads(&bucket, "", None, None, None, 100)
|
||||
.await
|
||||
.expect("list durable uploads")
|
||||
.uploads;
|
||||
assert_eq!(uploads.len(), 1);
|
||||
assert_eq!(Some(uploads[0].upload_id.as_str()), response.output.upload_id.as_deref());
|
||||
assert_eq!(probe.audits.lock().unwrap().len(), 1);
|
||||
assert!(probe.events.lock().unwrap().is_empty());
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Values consumed and produced by multipart initiation.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use time::OffsetDateTime;
|
||||
|
||||
/// Keeps the decoded values in their original form until the use case validates them.
|
||||
// The customer key must never appear in diagnostics.
|
||||
pub(crate) struct CreateMultipartUploadFields {
|
||||
pub(crate) bucket: String,
|
||||
pub(crate) key: String,
|
||||
pub(crate) metadata: Option<HashMap<String, String>>,
|
||||
pub(crate) tagging: Option<String>,
|
||||
pub(crate) storage_class: Option<String>,
|
||||
pub(crate) server_side_encryption: Option<String>,
|
||||
pub(crate) sse_customer_algorithm: Option<String>,
|
||||
pub(crate) sse_customer_key: Option<String>,
|
||||
pub(crate) sse_customer_key_md5: Option<String>,
|
||||
pub(crate) ssekms_key_id: Option<String>,
|
||||
pub(crate) object_lock_legal_hold_status: Option<String>,
|
||||
pub(crate) object_lock_mode: Option<String>,
|
||||
pub(crate) object_lock_retain_until_date: Option<OffsetDateTime>,
|
||||
}
|
||||
|
||||
pub(crate) struct CreateMultipartUploadResult {
|
||||
pub(crate) bucket: String,
|
||||
pub(crate) key: String,
|
||||
pub(crate) upload_id: String,
|
||||
pub(crate) server_side_encryption: Option<String>,
|
||||
pub(crate) sse_customer_algorithm: Option<String>,
|
||||
pub(crate) sse_customer_key_md5: Option<String>,
|
||||
pub(crate) ssekms_key_id: Option<String>,
|
||||
pub(crate) checksum_algorithm: Option<String>,
|
||||
pub(crate) checksum_type: Option<String>,
|
||||
}
|
||||
@@ -18,9 +18,82 @@ use super::*;
|
||||
use crate::app::RequestEnvelope;
|
||||
use crate::app::dto_values;
|
||||
use crate::app::storage_api::multipart_usecase::contract::multipart::ListMultipartsInfo;
|
||||
use crate::app::storage_api::s3::dto::{ListPartsInput, ListPartsOutput};
|
||||
use crate::app::storage_api::s3::{S3Error, S3ErrorCode, S3Request, S3Response, S3Result};
|
||||
pub(crate) use crate::app::storage_api::multipart_usecase::contract::multipart::ListPartsInfo;
|
||||
#[cfg(any(feature = "gateway-stack", test))]
|
||||
use crate::app::storage_api::multipart_usecase::contract::multipart::ListPartsOwner;
|
||||
use rustfs_gateway_types::dto as gateway_dto;
|
||||
#[cfg(any(feature = "gateway-stack", test))]
|
||||
use rustfs_gateway_types::{BucketName, ObjectKey, RecordedUpload, UploadIdClaim, resolve_upload_async};
|
||||
use rustfs_s3_types::S3Result;
|
||||
#[cfg(any(feature = "gateway-stack", test))]
|
||||
use rustfs_s3_types::{S3Error, S3ErrorCode};
|
||||
|
||||
pub(crate) enum ListPartsTarget {
|
||||
Legacy {
|
||||
bucket: String,
|
||||
key: String,
|
||||
upload_id: String,
|
||||
},
|
||||
#[cfg(any(feature = "gateway-stack", test))]
|
||||
Gateway {
|
||||
bucket: BucketName,
|
||||
key: ObjectKey,
|
||||
upload_id: UploadIdClaim,
|
||||
},
|
||||
}
|
||||
|
||||
impl ListPartsTarget {
|
||||
fn resource(&self) -> (&str, &str) {
|
||||
match self {
|
||||
Self::Legacy { bucket, key, .. } => (bucket, key),
|
||||
#[cfg(any(feature = "gateway-stack", test))]
|
||||
Self::Gateway { bucket, key, .. } => (bucket.as_str(), key.as_str()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) struct ListPartsFields {
|
||||
pub(crate) target: ListPartsTarget,
|
||||
pub(crate) part_number_marker: Option<String>,
|
||||
pub(crate) max_parts: Option<i32>,
|
||||
}
|
||||
|
||||
#[cfg(any(feature = "gateway-stack", test))]
|
||||
struct RecordedListParts {
|
||||
owner: ListPartsOwner,
|
||||
page: ListPartsInfo,
|
||||
}
|
||||
|
||||
#[cfg(any(feature = "gateway-stack", test))]
|
||||
impl RecordedUpload for RecordedListParts {
|
||||
fn bucket(&self) -> &str {
|
||||
&self.owner.bucket
|
||||
}
|
||||
|
||||
fn key(&self) -> &str {
|
||||
&self.owner.object
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(any(feature = "gateway-stack", test))]
|
||||
async fn resolve_list_parts<'a, F>(
|
||||
upload_id: &'a UploadIdClaim,
|
||||
bucket: &BucketName,
|
||||
key: &ObjectKey,
|
||||
lookup: impl FnOnce(&'a str) -> F,
|
||||
) -> S3Result<ListPartsInfo>
|
||||
where
|
||||
F: Future<Output = Result<ListPartsInfo, StorageError>>,
|
||||
{
|
||||
let (_, record) = resolve_upload_async(upload_id, bucket, key, |id| async move {
|
||||
let mut page = lookup(id).await?;
|
||||
Ok::<_, StorageError>(page.recorded_owner.take().map(|owner| RecordedListParts { owner, page }))
|
||||
})
|
||||
.await
|
||||
.map_err(|error| S3Error::from(multipart_upload_api_error(error)))?
|
||||
.map_err(|refusal| S3Error::with_message(S3ErrorCode::NoSuchUpload, refusal.reason()))?;
|
||||
Ok(record.page)
|
||||
}
|
||||
|
||||
impl DefaultMultipartUsecase {
|
||||
/// One page of ListMultipartUploads: the uploads of the bucket's current incarnation.
|
||||
@@ -101,41 +174,58 @@ impl DefaultMultipartUsecase {
|
||||
list_multipart_uploads_output(bucket, prefix, result, input.encoding_type)
|
||||
}
|
||||
|
||||
pub async fn execute_list_parts(&self, req: S3Request<ListPartsInput>) -> S3Result<S3Response<ListPartsOutput>> {
|
||||
pub(crate) async fn execute_list_parts(&self, envelope: RequestEnvelope, fields: ListPartsFields) -> S3Result<ListPartsInfo> {
|
||||
reject_presigned_multipart_max_total_object_size_for_other_operation(
|
||||
&req.headers,
|
||||
req.uri.query(),
|
||||
req.extensions.get::<VerifiedSigV4Request>().is_some(),
|
||||
envelope.headers(),
|
||||
envelope.uri().query(),
|
||||
envelope.verified_sigv4().is_some(),
|
||||
)?;
|
||||
reject_presigned_put_max_content_length_for_other_operation(
|
||||
&req.headers,
|
||||
req.uri.query(),
|
||||
req.extensions.get::<VerifiedPresignedRequest>().is_some(),
|
||||
envelope.headers(),
|
||||
envelope.uri().query(),
|
||||
envelope.verified_presigned().is_some(),
|
||||
)?;
|
||||
let (bucket, key) = fields.target.resource();
|
||||
let mut opts = ObjectOptions::default();
|
||||
apply_bucket_generation_guard(&req, &req.input.bucket, &mut opts)?;
|
||||
let ListPartsInput {
|
||||
bucket,
|
||||
key,
|
||||
upload_id,
|
||||
part_number_marker,
|
||||
max_parts,
|
||||
..
|
||||
} = req.input;
|
||||
apply_bucket_generation_guard(&envelope, bucket, &mut opts)?;
|
||||
reject_veeam_sosapi_mutation(|| self.object_store(), bucket, key).await?;
|
||||
|
||||
reject_veeam_sosapi_mutation(|| self.object_store(), &bucket, &key).await?;
|
||||
let params = parse_list_parts_params(part_number_marker, max_parts)?;
|
||||
let marker = fields
|
||||
.part_number_marker
|
||||
.as_deref()
|
||||
.map(|marker| {
|
||||
marker
|
||||
.parse::<i32>()
|
||||
.map_err(|_| rustfs_s3_types::s3_error!(InvalidArgument, "invalid query: part-number-marker: {marker}"))
|
||||
})
|
||||
.transpose()?;
|
||||
let params = parse_list_parts_params(marker, fields.max_parts)?;
|
||||
|
||||
let Some(store) = self.object_store() else {
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
|
||||
return Err(rustfs_s3_types::s3_error!(InternalError, "Not init"));
|
||||
};
|
||||
|
||||
let res = store
|
||||
.list_object_parts(&bucket, &key, &upload_id, params.part_number_marker, params.max_parts, &opts)
|
||||
.await
|
||||
.map_err(multipart_upload_api_error)?;
|
||||
|
||||
Ok(S3Response::new(build_list_parts_output(res)))
|
||||
match &fields.target {
|
||||
ListPartsTarget::Legacy { upload_id, .. } => store
|
||||
.list_object_parts(bucket, key, upload_id, params.part_number_marker, params.max_parts, &opts)
|
||||
.await
|
||||
.map_err(|error| multipart_upload_api_error(error).into()),
|
||||
#[cfg(any(feature = "gateway-stack", test))]
|
||||
ListPartsTarget::Gateway {
|
||||
bucket: bucket_name,
|
||||
key: object_key,
|
||||
upload_id,
|
||||
} => {
|
||||
let store = &store;
|
||||
let opts = &opts;
|
||||
resolve_list_parts(upload_id, bucket_name, object_key, |id| async move {
|
||||
store
|
||||
.list_object_parts(bucket, key, id, params.part_number_marker, params.max_parts, opts)
|
||||
.await
|
||||
})
|
||||
.await
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -192,6 +282,9 @@ fn list_multipart_uploads_output(
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod parts_tests;
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::super::test_support::*;
|
||||
@@ -301,48 +394,38 @@ mod tests {
|
||||
assert!(output.common_prefixes.is_empty());
|
||||
}
|
||||
|
||||
async fn list_parts_refusal(marker: Option<&str>, max_parts: Option<i32>) -> S3Error {
|
||||
let fields = ListPartsFields {
|
||||
target: ListPartsTarget::Legacy {
|
||||
bucket: "bucket".to_owned(),
|
||||
key: "object".to_owned(),
|
||||
upload_id: "upload-id".to_owned(),
|
||||
},
|
||||
part_number_marker: marker.map(str::to_owned),
|
||||
max_parts,
|
||||
};
|
||||
make_usecase()
|
||||
.execute_list_parts(RequestEnvelope::from_s3s(&build_request((), Method::GET)), fields)
|
||||
.await
|
||||
.expect_err("refused")
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_list_parts_returns_internal_error_when_store_uninitialized() {
|
||||
let input = ListPartsInput::builder()
|
||||
.bucket("bucket".to_string())
|
||||
.key("object".to_string())
|
||||
.upload_id("upload-id".to_string())
|
||||
.build()
|
||||
.unwrap();
|
||||
let req = build_request(input, Method::GET);
|
||||
|
||||
let err = make_usecase().execute_list_parts(req).await.unwrap_err();
|
||||
let err = list_parts_refusal(None, None).await;
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_list_parts_rejects_negative_part_number_marker_before_store_lookup() {
|
||||
let input = ListPartsInput::builder()
|
||||
.bucket("bucket".to_string())
|
||||
.key("object".to_string())
|
||||
.upload_id("upload-id".to_string())
|
||||
.part_number_marker(Some(-1))
|
||||
.build()
|
||||
.unwrap();
|
||||
let req = build_request(input, Method::GET);
|
||||
|
||||
let err = make_usecase().execute_list_parts(req).await.unwrap_err();
|
||||
let err = list_parts_refusal(Some("-1"), Some(0)).await;
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
|
||||
assert_eq!(err.message(), Some("part-number-marker must be non-negative"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_list_parts_rejects_invalid_max_parts_before_store_lookup() {
|
||||
let input = ListPartsInput::builder()
|
||||
.bucket("bucket".to_string())
|
||||
.key("object".to_string())
|
||||
.upload_id("upload-id".to_string())
|
||||
.max_parts(Some(1001))
|
||||
.build()
|
||||
.unwrap();
|
||||
let req = build_request(input, Method::GET);
|
||||
|
||||
let err = make_usecase().execute_list_parts(req).await.unwrap_err();
|
||||
let err = list_parts_refusal(None, Some(1001)).await;
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
|
||||
assert_eq!(err.message(), Some("max-parts must be between 1 and 1000"));
|
||||
}
|
||||
|
||||
@@ -0,0 +1,310 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! ListParts execution against real multipart metadata and the native claim exchange.
|
||||
|
||||
use super::super::test_support::{Method, build_request};
|
||||
use super::*;
|
||||
use crate::app::gating_test_env::{run_large_stack_test, shared_gating_ambient, shared_gating_ecstore_and_disk_paths};
|
||||
use crate::app::storage_api::multipart_usecase::access::{AuthorizationRequest, load_bucket_generation_from_store};
|
||||
use crate::app::storage_api::s3::S3Error as LegacyS3Error;
|
||||
use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, DeleteBucketOptions, MakeBucketOptions};
|
||||
use crate::runtime_sources::ServerContextSlot;
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::sync::atomic::{AtomicUsize, Ordering::Relaxed};
|
||||
use tracing::instrument::WithSubscriber as _;
|
||||
use tracing::{Subscriber, span};
|
||||
use tracing_subscriber::{Layer, layer::Context, layer::SubscriberExt as _};
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
struct UploadLookups(Arc<AtomicUsize>);
|
||||
|
||||
impl<S: Subscriber> Layer<S> for UploadLookups {
|
||||
fn on_new_span(&self, attrs: &span::Attributes<'_>, _: &span::Id, _: Context<'_, S>) {
|
||||
let meta = attrs.metadata();
|
||||
if meta
|
||||
.file()
|
||||
.is_some_and(|file| file.ends_with("crates/ecstore/src/store/mod.rs"))
|
||||
&& matches!(meta.name(), "list_object_parts" | "get_multipart_info")
|
||||
{
|
||||
self.0.fetch_add(1, Relaxed);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn gateway_fields(bucket: &str, key: &str, id: &str, marker: Option<&str>, max_parts: Option<i32>) -> ListPartsFields {
|
||||
ListPartsFields {
|
||||
target: ListPartsTarget::Gateway {
|
||||
bucket: BucketName::new(bucket).expect("fixture bucket"),
|
||||
key: ObjectKey::new(key).expect("fixture key"),
|
||||
upload_id: UploadIdClaim::from_wire(id),
|
||||
},
|
||||
part_number_marker: marker.map(str::to_owned),
|
||||
max_parts,
|
||||
}
|
||||
}
|
||||
|
||||
async fn execute_counted(
|
||||
usecase: &DefaultMultipartUsecase,
|
||||
envelope: RequestEnvelope,
|
||||
fields: ListPartsFields,
|
||||
expected: usize,
|
||||
) -> S3Result<ListPartsInfo> {
|
||||
let lookups = UploadLookups::default();
|
||||
let result = usecase
|
||||
.execute_list_parts(envelope, fields)
|
||||
.with_subscriber(tracing_subscriber::registry().with(lookups.clone()))
|
||||
.await;
|
||||
assert_eq!(lookups.0.load(Relaxed), expected, "storage upload lookups");
|
||||
result
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn list_parts_resolves_real_owners_and_retains_storage_errors() {
|
||||
run_large_stack_test("list-parts-ownership", || async {
|
||||
let (paths, store) = shared_gating_ecstore_and_disk_paths().await;
|
||||
let usecase = DefaultMultipartUsecase::with_context(Some(shared_gating_ambient().await));
|
||||
let bucket = format!("list-parts-{}", Uuid::new_v4().simple());
|
||||
let foreign_bucket = format!("list-parts-{}", Uuid::new_v4().simple());
|
||||
let key = "fixture/one";
|
||||
let foreign_key = "fixture/two";
|
||||
let opts = ObjectOptions::default();
|
||||
let envelope = || RequestEnvelope::from_s3s(&build_request((), Method::GET));
|
||||
for name in [&bucket, &foreign_bucket] {
|
||||
store
|
||||
.make_bucket(name, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create a fixture bucket");
|
||||
}
|
||||
let upload = store
|
||||
.new_multipart_upload(&bucket, key, &opts)
|
||||
.await
|
||||
.expect("create an active session");
|
||||
let id = upload.upload_id.as_str();
|
||||
|
||||
for (claim, marker, max, lookups, code) in [
|
||||
(id, None, None, 1, None),
|
||||
("", None, None, 0, Some("NoSuchUpload")),
|
||||
("/impossible", None, None, 0, Some("NoSuchUpload")),
|
||||
("malformed", None, None, 1, Some("NoSuchUpload")),
|
||||
("/impossible", Some("-1"), Some(0), 0, Some("InvalidArgument")),
|
||||
("/impossible", None, Some(0), 0, Some("InvalidArgument")),
|
||||
] {
|
||||
let result = execute_counted(&usecase, envelope(), gateway_fields(&bucket, key, claim, marker, max), lookups).await;
|
||||
assert_eq!(
|
||||
result.as_ref().err().map(|err| err.code().as_str()),
|
||||
code,
|
||||
"marker {marker:?}, max {max:?}"
|
||||
);
|
||||
if let Ok(page) = result {
|
||||
assert!(page.parts.is_empty());
|
||||
assert!(page.upload_id == id);
|
||||
}
|
||||
}
|
||||
|
||||
let typed_bucket = BucketName::new(bucket.clone()).unwrap();
|
||||
let typed_key = ObjectKey::new(key).unwrap();
|
||||
let claim = UploadIdClaim::from_wire(id);
|
||||
for (owner_bucket, owner_key) in [(foreign_bucket.as_str(), key), (bucket.as_str(), foreign_key)] {
|
||||
let other = store
|
||||
.new_multipart_upload(owner_bucket, owner_key, &opts)
|
||||
.await
|
||||
.expect("create an independent active session");
|
||||
let mut actual_page = store
|
||||
.list_object_parts(owner_bucket, owner_key, &other.upload_id, None, 1, &opts)
|
||||
.await
|
||||
.expect("the independent session returns a real page");
|
||||
// Only the durable owner observation is relevant, even if presentation agrees.
|
||||
actual_page.bucket.clone_from(&bucket);
|
||||
actual_page.object = key.to_owned();
|
||||
let refusal = resolve_list_parts(&claim, &typed_bucket, &typed_key, |supplied| {
|
||||
assert!(supplied == id, "the callback receives the original upload ID");
|
||||
std::future::ready(Ok(actual_page))
|
||||
})
|
||||
.await
|
||||
.expect_err("a foreign recorded owner cannot be returned");
|
||||
assert_eq!(refusal.code(), &S3ErrorCode::NoSuchUpload);
|
||||
}
|
||||
|
||||
let mut unproved = store
|
||||
.list_object_parts(&bucket, key, id, None, 1, &opts)
|
||||
.await
|
||||
.expect("live session");
|
||||
unproved.recorded_owner = None;
|
||||
let refusal = resolve_list_parts(&claim, &typed_bucket, &typed_key, |_| std::future::ready(Ok(unproved)))
|
||||
.await
|
||||
.expect_err("a page without owner proof must be refused");
|
||||
assert_eq!(refusal.code(), &S3ErrorCode::NoSuchUpload);
|
||||
|
||||
let staging_name = hex_simd::encode_to_string(Sha256::digest(format!("{bucket}/{key}")), hex_simd::AsciiCase::Lower);
|
||||
for disk in &paths[..3] {
|
||||
let staging = disk.join(".rustfs.sys/multipart").join(&staging_name);
|
||||
let uploads: Vec<_> = std::fs::read_dir(&staging).expect("real upload storage namespace").collect();
|
||||
assert_eq!(uploads.len(), 1, "the test owns this upload namespace");
|
||||
let path = uploads.into_iter().next().unwrap().unwrap().path().join("xl.meta");
|
||||
assert!(path.is_file(), "the upload has persisted metadata");
|
||||
std::fs::write(path, b"unreadable test metadata").expect("damage the fixture metadata only");
|
||||
}
|
||||
let storage_error = store
|
||||
.list_object_parts(&bucket, key, id, None, 1000, &opts)
|
||||
.await
|
||||
.expect_err("loss of three upload metadata replicas fails the read");
|
||||
assert!(!matches!(
|
||||
storage_error,
|
||||
StorageError::InvalidUploadID(..) | StorageError::MalformedUploadID(..)
|
||||
));
|
||||
let expected = multipart_upload_api_error(storage_error);
|
||||
for target in [
|
||||
gateway_fields(&bucket, key, id, None, None),
|
||||
ListPartsFields {
|
||||
target: ListPartsTarget::Legacy {
|
||||
bucket: bucket.clone(),
|
||||
key: key.to_owned(),
|
||||
upload_id: id.to_owned(),
|
||||
},
|
||||
part_number_marker: None,
|
||||
max_parts: None,
|
||||
},
|
||||
] {
|
||||
let refusal = execute_counted(&usecase, envelope(), target, 1)
|
||||
.await
|
||||
.expect_err("surface the outage");
|
||||
let legacy: LegacyS3Error = refusal.into();
|
||||
assert_eq!(legacy.code(), expected.code());
|
||||
assert_eq!(legacy.status_code(), expected.status_code());
|
||||
assert_eq!(legacy.message(), expected.message());
|
||||
assert!(
|
||||
legacy
|
||||
.source()
|
||||
.and_then(|cause| cause.downcast_ref::<StorageError>())
|
||||
.is_some()
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn list_parts_applies_the_authorized_generation_before_lookup() {
|
||||
run_large_stack_test("list-parts-generation", || async {
|
||||
let context = shared_gating_ambient().await;
|
||||
let store = context.object_store();
|
||||
let usecase = DefaultMultipartUsecase::with_context(Some(context.clone()));
|
||||
let slot = ServerContextSlot::new();
|
||||
assert!(slot.install(context));
|
||||
let bucket = format!("list-parts-{}", Uuid::new_v4().simple());
|
||||
let other = format!("list-parts-{}", Uuid::new_v4().simple());
|
||||
let key = "fixture";
|
||||
let opts = ObjectOptions::default();
|
||||
for name in [&bucket, &other] {
|
||||
store
|
||||
.make_bucket(name, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create a fixture bucket");
|
||||
}
|
||||
let req = || {
|
||||
let mut req = build_request((), Method::GET);
|
||||
req.extensions.insert(slot.clone());
|
||||
req
|
||||
};
|
||||
let capability = |query: &str| {
|
||||
let mut request = req();
|
||||
request.uri = format!("/fixture?{query}").parse().expect("fixture URI");
|
||||
RequestEnvelope::from_s3s(&request)
|
||||
};
|
||||
for (query, first) in [
|
||||
(
|
||||
"x-rustfs-max-total-object-size=1&x-rustfs-max-content-length=1",
|
||||
"x-rustfs-max-total-object-size",
|
||||
),
|
||||
("x-rustfs-max-content-length=1", "x-rustfs-max-content-length"),
|
||||
] {
|
||||
let err = execute_counted(
|
||||
&usecase,
|
||||
capability(query),
|
||||
gateway_fields(&bucket, key, "/impossible", Some("-1"), None),
|
||||
0,
|
||||
)
|
||||
.await
|
||||
.expect_err("capability validation precedes generation, pagination and claim resolution");
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest);
|
||||
assert!(err.message().is_some_and(|message| message.starts_with(first)));
|
||||
}
|
||||
|
||||
let old = load_bucket_generation_from_store(&store, &req(), &bucket)
|
||||
.await
|
||||
.expect("old generation");
|
||||
let other_guard = load_bucket_generation_from_store(&store, &req(), &other)
|
||||
.await
|
||||
.expect("other generation");
|
||||
store
|
||||
.delete_bucket(&bucket, &DeleteBucketOptions::default())
|
||||
.await
|
||||
.expect("remove the original bucket");
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("recreate the bucket");
|
||||
let upload = store
|
||||
.new_multipart_upload(&bucket, key, &opts)
|
||||
.await
|
||||
.expect("session from the recreated bucket");
|
||||
let current = load_bucket_generation_from_store(&store, &req(), &bucket)
|
||||
.await
|
||||
.expect("current generation");
|
||||
let mut veeam_request = req();
|
||||
veeam_request.install_bucket_generation(current.clone());
|
||||
temp_env::async_with_vars([(rustfs_config::ENV_VEEAM_SOSAPI_ENABLE, Some("true"))], async {
|
||||
let err = execute_counted(
|
||||
&usecase,
|
||||
RequestEnvelope::from_s3s(&veeam_request),
|
||||
gateway_fields(
|
||||
&bucket,
|
||||
".system-d26a9498-cb7c-4a87-a44a-8ae204f5ba6c/system.xml",
|
||||
"/impossible",
|
||||
Some("-1"),
|
||||
None,
|
||||
),
|
||||
0,
|
||||
)
|
||||
.await
|
||||
.expect_err("reserved objects are checked before pagination and upload claims");
|
||||
assert_eq!(err.code(), &S3ErrorCode::MethodNotAllowed);
|
||||
})
|
||||
.await;
|
||||
for (guard, marker, lookups, code) in [
|
||||
(Some(old), None, 1, Some("NoSuchBucket")),
|
||||
(Some(other_guard), Some("-1"), 0, Some("InternalError")),
|
||||
(None, Some("-1"), 0, Some("InternalError")),
|
||||
(Some(current), None, 1, None),
|
||||
] {
|
||||
let mut request = req();
|
||||
if let Some(guard) = guard {
|
||||
request.install_bucket_generation(guard);
|
||||
}
|
||||
let result = execute_counted(
|
||||
&usecase,
|
||||
RequestEnvelope::from_s3s(&request),
|
||||
gateway_fields(&bucket, key, &upload.upload_id, marker, None),
|
||||
lookups,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(result.as_ref().err().map(|err| err.code().as_str()), code);
|
||||
if let Ok(page) = result {
|
||||
assert!(page.upload_id == upload.upload_id);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -15,17 +15,43 @@
|
||||
//! UploadPart.
|
||||
|
||||
use super::*;
|
||||
use crate::app::storage_api::s3::dto::{
|
||||
ChecksumAlgorithm, ServerSideEncryption, StreamingBlob, UploadPartInput, UploadPartOutput,
|
||||
};
|
||||
use crate::app::storage_api::s3::stream::ByteStream;
|
||||
use crate::app::storage_api::s3::{S3Error, S3ErrorCode, S3Request, S3Response, S3Result, s3_error};
|
||||
use crate::app::RequestEnvelope;
|
||||
use crate::app::object::BodyChunks;
|
||||
use crate::app::object::request_body::{DemandReader, UploadChunks, guard_native_upload_progress};
|
||||
use crate::app::storage_api::multipart_usecase::access::AuthorizationRequest;
|
||||
use crate::app::storage_api::multipart_usecase::contract::multipart::MultipartInfo;
|
||||
use http_body::Body as _;
|
||||
use rustfs_gateway_stream::Body;
|
||||
#[cfg(any(feature = "gateway-stack", test))]
|
||||
use rustfs_gateway_types::{RecordedUpload, resolve_upload_async};
|
||||
use rustfs_s3_types::{S3Error, S3ErrorCode, S3Result, s3_error};
|
||||
|
||||
fn resolve_upload_part_size(content_length: Option<i64>, body: Option<&StreamingBlob>) -> rustfs_s3_types::S3Result<Option<i64>> {
|
||||
#[cfg(any(feature = "gateway-stack", test))]
|
||||
struct PartUpload(MultipartInfo);
|
||||
|
||||
#[cfg(any(feature = "gateway-stack", test))]
|
||||
impl RecordedUpload for PartUpload {
|
||||
fn bucket(&self) -> &str {
|
||||
&self.0.bucket
|
||||
}
|
||||
|
||||
fn key(&self) -> &str {
|
||||
&self.0.object
|
||||
}
|
||||
}
|
||||
|
||||
async fn lookup_part_upload(store: &ECStore, bucket: &str, key: &str, id: &str, opts: &ObjectOptions) -> S3Result<MultipartInfo> {
|
||||
let result = store.get_multipart_info(bucket, key, id, opts).await;
|
||||
#[cfg(test)]
|
||||
let result = tests::observe_lookup(result);
|
||||
result.map_err(|error| multipart_upload_api_error(error).into())
|
||||
}
|
||||
|
||||
fn resolve_upload_part_size(content_length: Option<i64>, body: Option<&Body>) -> rustfs_s3_types::S3Result<Option<i64>> {
|
||||
if let Some(length) = content_length {
|
||||
return Ok(Some(length));
|
||||
}
|
||||
body.and_then(|body| body.remaining_length().exact())
|
||||
body.and_then(|body| body.size_hint().exact())
|
||||
.map(i64::try_from)
|
||||
.transpose()
|
||||
.map_err(|_| rustfs_s3_types::s3_error!(UnexpectedContent))
|
||||
@@ -59,28 +85,29 @@ fn passthrough_part_actual_size(headers: &HeaderMap) -> Option<i64> {
|
||||
|
||||
// Hoisted values are signed query parameters, not HTTP headers. Normalize only
|
||||
// after access control has verified the presigned request.
|
||||
fn normalize_presigned_part_checksums(req: &mut S3Request<UploadPartInput>) -> S3Result<()> {
|
||||
if req.extensions.get::<VerifiedPresignedRequest>().is_none() {
|
||||
fn normalize_presigned_part_checksums(req: &mut RequestEnvelope, input: &mut UploadPartWriteInput) -> S3Result<()> {
|
||||
if req.verified_presigned().is_none() {
|
||||
return Ok(());
|
||||
}
|
||||
let Some(query) = req.uri.query() else {
|
||||
let uri = req.uri().clone();
|
||||
let Some(query) = uri.query() else {
|
||||
return Ok(());
|
||||
};
|
||||
let mut seen = std::collections::HashSet::new();
|
||||
for (name, value) in url::form_urlencoded::parse(query.as_bytes()) {
|
||||
let field = match name.as_ref() {
|
||||
"x-amz-checksum-crc32" => Some(&mut req.input.checksum_crc32),
|
||||
"x-amz-checksum-crc32c" => Some(&mut req.input.checksum_crc32c),
|
||||
"x-amz-checksum-crc64nvme" => Some(&mut req.input.checksum_crc64nvme),
|
||||
"x-amz-checksum-sha1" => Some(&mut req.input.checksum_sha1),
|
||||
"x-amz-checksum-sha256" => Some(&mut req.input.checksum_sha256),
|
||||
"x-amz-checksum-crc32" => Some(&mut input.checksum_crc32),
|
||||
"x-amz-checksum-crc32c" => Some(&mut input.checksum_crc32c),
|
||||
"x-amz-checksum-crc64nvme" => Some(&mut input.checksum_crc64nvme),
|
||||
"x-amz-checksum-sha1" => Some(&mut input.checksum_sha1),
|
||||
"x-amz-checksum-sha256" => Some(&mut input.checksum_sha256),
|
||||
"x-amz-sdk-checksum-algorithm" => None,
|
||||
_ => continue,
|
||||
};
|
||||
if !seen.insert(name.clone()) {
|
||||
return Err(S3Error::with_message(S3ErrorCode::InvalidRequest, "Duplicate checksum query parameter"));
|
||||
}
|
||||
if let Some(header) = req.headers.get(name.as_ref())
|
||||
if let Some(header) = req.headers().get(name.as_ref())
|
||||
&& header.as_bytes() != value.as_bytes()
|
||||
{
|
||||
return Err(S3Error::with_message(
|
||||
@@ -90,7 +117,7 @@ fn normalize_presigned_part_checksums(req: &mut S3Request<UploadPartInput>) -> S
|
||||
}
|
||||
let header = http::HeaderValue::from_str(&value)
|
||||
.map_err(|_| S3Error::with_message(S3ErrorCode::InvalidArgument, "Invalid checksum query parameter"))?;
|
||||
req.headers.insert(
|
||||
req.headers_mut().insert(
|
||||
http::header::HeaderName::from_bytes(name.as_bytes())
|
||||
.map_err(|_| S3Error::with_message(S3ErrorCode::InvalidArgument, "Invalid checksum query parameter"))?,
|
||||
header,
|
||||
@@ -98,59 +125,62 @@ fn normalize_presigned_part_checksums(req: &mut S3Request<UploadPartInput>) -> S
|
||||
if let Some(field) = field {
|
||||
*field = Some(value.into_owned());
|
||||
} else {
|
||||
req.input.checksum_algorithm = Some(ChecksumAlgorithm::from(value.into_owned()));
|
||||
input.checksum_algorithm = Some(value.into_owned());
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
impl DefaultMultipartUsecase {
|
||||
#[instrument(level = "debug", skip(self, req))]
|
||||
#[instrument(name = "execute_upload_part", level = "debug", skip(self, req, input))]
|
||||
#[hotpath::measure(impl_type = "MultipartUsecase")]
|
||||
pub async fn execute_upload_part(&self, mut req: S3Request<UploadPartInput>) -> S3Result<S3Response<UploadPartOutput>> {
|
||||
pub(crate) async fn execute_upload_part_owned(
|
||||
&self,
|
||||
mut req: RequestEnvelope,
|
||||
mut input: UploadPartWriteInput,
|
||||
) -> S3Result<UploadPartWriteOutput> {
|
||||
reject_presigned_multipart_max_total_object_size_for_other_operation(
|
||||
&req.headers,
|
||||
req.uri.query(),
|
||||
req.extensions.get::<VerifiedSigV4Request>().is_some(),
|
||||
req.headers(),
|
||||
req.uri().query(),
|
||||
req.verified_sigv4().is_some(),
|
||||
)?;
|
||||
reject_presigned_put_max_content_length_for_other_operation(
|
||||
&req.headers,
|
||||
req.uri.query(),
|
||||
req.extensions.get::<VerifiedPresignedRequest>().is_some(),
|
||||
req.headers(),
|
||||
req.uri().query(),
|
||||
req.verified_presigned().is_some(),
|
||||
)?;
|
||||
normalize_presigned_part_checksums(&mut req)?;
|
||||
normalize_presigned_part_checksums(&mut req, &mut input)?;
|
||||
let (bucket, key) = input.target.resource();
|
||||
let mut opts = ObjectOptions::default();
|
||||
apply_bucket_generation_guard(&req, &req.input.bucket, &mut opts)?;
|
||||
apply_bucket_generation_guard(&req, bucket, &mut opts)?;
|
||||
let part_principal = SseKmsPrincipal::from_request(&req);
|
||||
let input = req.input;
|
||||
let UploadPartInput {
|
||||
let UploadPartWriteInput {
|
||||
body,
|
||||
bucket,
|
||||
key,
|
||||
upload_id,
|
||||
frame_trailers,
|
||||
trailing_headers,
|
||||
part_number,
|
||||
content_length,
|
||||
sse_customer_algorithm,
|
||||
sse_customer_key,
|
||||
sse_customer_key_md5,
|
||||
// content_md5,
|
||||
..
|
||||
} = input;
|
||||
|
||||
let part_id = parse_upload_part_number(part_number)?;
|
||||
|
||||
reject_veeam_sosapi_mutation(|| self.object_store(), &bucket, &key).await?;
|
||||
validate_table_catalog_object_mutation(&bucket, &key).await?;
|
||||
reject_veeam_sosapi_mutation(|| self.object_store(), bucket, key).await?;
|
||||
validate_table_catalog_object_mutation(bucket, key).await?;
|
||||
|
||||
// cuObject S3-over-RDMA UploadPart: an empty HTTP body plus
|
||||
// `x-amz-rdma-token` means the part bytes are read from the client's
|
||||
// registered buffer (see `object::cuobject::rdma_put_body`).
|
||||
#[cfg(all(any(feature = "cuobject", feature = "nixl"), target_os = "linux"))]
|
||||
let (body, content_length, rdma_put) = match crate::app::object::cuobject::rdma_token_for_request(&req.headers) {
|
||||
let (body, content_length, rdma_put) = match crate::app::object::cuobject::rdma_token_for_request(req.headers()) {
|
||||
Some(token) if content_length.unwrap_or(0) == 0 => {
|
||||
let (body, total) = crate::app::object::cuobject::rdma_put_body(&bucket, &key, &token, true)?;
|
||||
let (body, total) =
|
||||
crate::app::object::cuobject::rdma_put_body(bucket, key, &token, true).map_err(S3Error::from)?;
|
||||
let total = i64::try_from(total).map_err(|_| S3Error::new(S3ErrorCode::EntityTooLarge))?;
|
||||
(Some(body), Some(total), Some(total))
|
||||
(Some(crate::app::object::legacy_put_body(body)), Some(total), Some(total))
|
||||
}
|
||||
_ => (body, content_length, None),
|
||||
};
|
||||
@@ -165,10 +195,29 @@ impl DefaultMultipartUsecase {
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
|
||||
};
|
||||
let multipart_info_started = rustfs_io_metrics::put_stage_timer();
|
||||
let fi = store
|
||||
.get_multipart_info(&bucket, &key, &upload_id, &opts)
|
||||
.await
|
||||
.map_err(multipart_upload_api_error)?;
|
||||
let (upload_id, fi) = match &input.target {
|
||||
UploadPartTarget::Legacy { upload_id, .. } => {
|
||||
let info = lookup_part_upload(&store, bucket, key, upload_id, &opts).await?;
|
||||
(upload_id.clone(), info)
|
||||
}
|
||||
#[cfg(any(feature = "gateway-stack", test))]
|
||||
UploadPartTarget::Gateway {
|
||||
bucket: bucket_name,
|
||||
key: object_key,
|
||||
upload_id,
|
||||
} => {
|
||||
let lookup_store = &store;
|
||||
let lookup_opts = &opts;
|
||||
let (id, upload) = resolve_upload_async(upload_id, bucket_name, object_key, |id| async move {
|
||||
lookup_part_upload(lookup_store, bucket, key, id, lookup_opts)
|
||||
.await
|
||||
.map(|info| Some(PartUpload(info)))
|
||||
})
|
||||
.await?
|
||||
.map_err(|rejection| S3Error::with_message(S3ErrorCode::NoSuchUpload, rejection.reason()))?;
|
||||
(id.id().to_owned(), upload.0)
|
||||
}
|
||||
};
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from("multipart_get_info", multipart_info_started);
|
||||
authorize_sse_kms_multipart_write(part_principal.as_ref(), &fi.user_defined).await?;
|
||||
let max_total_object_size = multipart_max_total_object_size(&fi.user_defined)?;
|
||||
@@ -197,26 +246,16 @@ impl DefaultMultipartUsecase {
|
||||
));
|
||||
}
|
||||
};
|
||||
let request_id = req
|
||||
.extensions
|
||||
.get::<super::super::storage_api::multipart_usecase::request_context::RequestContext>()
|
||||
.map(|ctx| ctx.request_id.as_str())
|
||||
.unwrap_or_default();
|
||||
let (chunks, _) = BodyChunks::with_trailers(body_stream, frame_trailers);
|
||||
let body_stream: UploadChunks = Box::new(chunks);
|
||||
let request_id = req.request_context().map(|ctx| ctx.request_id.as_str()).unwrap_or_default();
|
||||
let timeout = upload_part_body_read_timeout(put_object_body_read_timeout(), max_total_object_size.is_some());
|
||||
let (body_stream, progress_control) =
|
||||
if let Some(control) = req.extensions.get::<super::super::object::request_body::BodyReadControl>() {
|
||||
control.activate_upload(timeout, &bucket, &key, request_id, size.max(0) as u64);
|
||||
(body_stream, control.clone())
|
||||
} else {
|
||||
super::super::object::request_body::guard_upload_progress(
|
||||
body_stream,
|
||||
timeout,
|
||||
&bucket,
|
||||
&key,
|
||||
request_id,
|
||||
size.max(0) as u64,
|
||||
)
|
||||
};
|
||||
let (body_stream, progress_control) = if let Some(control) = req.body_read_control() {
|
||||
control.activate_upload(timeout, bucket, key, request_id, size.max(0) as u64);
|
||||
(body_stream, control.clone())
|
||||
} else {
|
||||
guard_native_upload_progress(body_stream, timeout, bucket, key, request_id, size.max(0) as u64)
|
||||
};
|
||||
|
||||
let ingress_stage_start = rustfs_io_metrics::put_stage_metrics_enabled().then(std::time::Instant::now);
|
||||
|
||||
@@ -238,7 +277,7 @@ impl DefaultMultipartUsecase {
|
||||
&& rustfs_utils::http::contains_key_str(&fi.user_defined, rustfs_utils::http::SUFFIX_COMPRESSION);
|
||||
|
||||
let actual_size = if preserve_ciphertext {
|
||||
passthrough_part_actual_size(&req.headers).unwrap_or(size)
|
||||
passthrough_part_actual_size(req.headers()).unwrap_or(size)
|
||||
} else {
|
||||
size
|
||||
};
|
||||
@@ -252,7 +291,7 @@ impl DefaultMultipartUsecase {
|
||||
None
|
||||
};
|
||||
|
||||
let mut sha256hex = get_content_sha256_with_query(&req.headers, req.uri.query());
|
||||
let mut sha256hex = get_content_sha256_with_query(req.headers(), req.uri().query());
|
||||
|
||||
let mut write_plan = WritePlan::new();
|
||||
let etag_algorithm = multipart_etag_algorithm(&fi.user_defined);
|
||||
@@ -269,7 +308,7 @@ impl DefaultMultipartUsecase {
|
||||
)
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
if let Err(err) = hrd.add_checksum(&req.headers, trailer_source(req.trailing_headers.clone()), false) {
|
||||
if let Err(err) = hrd.add_checksum(req.headers(), trailing_headers.clone(), false) {
|
||||
return Err(ApiError::from(err).into());
|
||||
}
|
||||
|
||||
@@ -281,7 +320,7 @@ impl DefaultMultipartUsecase {
|
||||
.map_err(ApiError::from)?
|
||||
};
|
||||
|
||||
if let Err(err) = reader.add_checksum(&req.headers, trailer_source(req.trailing_headers.clone()), size < 0) {
|
||||
if let Err(err) = reader.add_checksum(req.headers(), trailing_headers.clone(), size < 0) {
|
||||
return Err(ApiError::from(err).into());
|
||||
}
|
||||
opts.want_checksum = reader.checksum();
|
||||
@@ -297,7 +336,7 @@ impl DefaultMultipartUsecase {
|
||||
} else {
|
||||
let sse = fi.user_defined.get("x-amz-server-side-encryption").cloned();
|
||||
let key_id = match sse.as_ref() {
|
||||
Some(sse) if sse.as_str() == ServerSideEncryption::AWS_KMS => fi
|
||||
Some(sse) if sse.as_str() == "aws:kms" => fi
|
||||
.user_defined
|
||||
.get("x-amz-server-side-encryption-aws-kms-key-id")
|
||||
.map(|s| s.to_string()),
|
||||
@@ -307,8 +346,8 @@ impl DefaultMultipartUsecase {
|
||||
};
|
||||
if !preserve_ciphertext {
|
||||
EncryptionRequest {
|
||||
bucket: &bucket,
|
||||
key: &key,
|
||||
bucket,
|
||||
key,
|
||||
server_side_encryption: server_side_encryption.clone(),
|
||||
ssekms_key_id: ssekms_key_id.clone(),
|
||||
ssekms_context: None,
|
||||
@@ -322,8 +361,8 @@ impl DefaultMultipartUsecase {
|
||||
}
|
||||
let (requested_sse, requested_kms_key_id) = if has_ssec {
|
||||
let ssec_material = sse_decryption(DecryptionRequest {
|
||||
bucket: &bucket,
|
||||
key: &key,
|
||||
bucket,
|
||||
key,
|
||||
metadata: &fi.user_defined,
|
||||
sse_customer_key: sse_customer_key.as_ref(),
|
||||
sse_customer_key_md5: sse_customer_key_md5.as_ref(),
|
||||
@@ -343,8 +382,8 @@ impl DefaultMultipartUsecase {
|
||||
// Reuse the stored envelope after authorizing this caller against its
|
||||
// pinned destination key above.
|
||||
let managed_material = sse_decryption(DecryptionRequest {
|
||||
bucket: &bucket,
|
||||
key: &key,
|
||||
bucket,
|
||||
key,
|
||||
metadata: &fi.user_defined,
|
||||
sse_customer_key: None,
|
||||
sse_customer_key_md5: None,
|
||||
@@ -369,8 +408,7 @@ impl DefaultMultipartUsecase {
|
||||
{
|
||||
use rustfs_rio::HashReaderMut;
|
||||
let inner = reader.take_inner();
|
||||
reader.inner =
|
||||
rustfs_rio::boxed_reader(super::super::object::request_body::DemandReader::new(inner, progress_control));
|
||||
reader.inner = rustfs_rio::boxed_reader(DemandReader::new(inner, progress_control));
|
||||
}
|
||||
|
||||
let mut reader = PutObjReader::new(reader);
|
||||
@@ -385,7 +423,7 @@ impl DefaultMultipartUsecase {
|
||||
let _upload_part_admission = upload_part_admission;
|
||||
let store_write_started = rustfs_io_metrics::put_stage_timer();
|
||||
let result = store
|
||||
.put_object_part(&bucket, &key, &upload_id, part_id, &mut reader, &opts)
|
||||
.put_object_part(bucket, key, &upload_id, part_id, &mut reader, &opts)
|
||||
.await;
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from("multipart_store_write", store_write_started);
|
||||
drop(_upload_part_admission);
|
||||
@@ -397,30 +435,21 @@ impl DefaultMultipartUsecase {
|
||||
let mut checksum_sha256 = input.checksum_sha256;
|
||||
let mut checksum_crc64nvme = input.checksum_crc64nvme;
|
||||
|
||||
if let Some(alg) = &input.checksum_algorithm
|
||||
&& let Some(Some(checksum_str)) = req.trailing_headers.as_ref().map(|trailer| {
|
||||
let key = match alg.as_str() {
|
||||
ChecksumAlgorithm::CRC32 => rustfs_rio::ChecksumType::CRC32.key(),
|
||||
ChecksumAlgorithm::CRC32C => rustfs_rio::ChecksumType::CRC32C.key(),
|
||||
ChecksumAlgorithm::SHA1 => rustfs_rio::ChecksumType::SHA1.key(),
|
||||
ChecksumAlgorithm::SHA256 => rustfs_rio::ChecksumType::SHA256.key(),
|
||||
ChecksumAlgorithm::CRC64NVME => rustfs_rio::ChecksumType::CRC64_NVME.key(),
|
||||
_ => return None,
|
||||
};
|
||||
trailer.read(|headers| {
|
||||
headers
|
||||
.get(key.unwrap_or_default())
|
||||
.and_then(|value| value.to_str().ok().map(|s| s.to_string()))
|
||||
})
|
||||
})
|
||||
{
|
||||
match alg.as_str() {
|
||||
ChecksumAlgorithm::CRC32 => checksum_crc32 = checksum_str,
|
||||
ChecksumAlgorithm::CRC32C => checksum_crc32c = checksum_str,
|
||||
ChecksumAlgorithm::SHA1 => checksum_sha1 = checksum_str,
|
||||
ChecksumAlgorithm::SHA256 => checksum_sha256 = checksum_str,
|
||||
ChecksumAlgorithm::CRC64NVME => checksum_crc64nvme = checksum_str,
|
||||
_ => (),
|
||||
if let (Some(algorithm), Some(trailers)) = (input.checksum_algorithm.as_deref(), trailing_headers) {
|
||||
let echo = match algorithm {
|
||||
"CRC32" => Some(("x-amz-checksum-crc32", &mut checksum_crc32)),
|
||||
"CRC32C" => Some(("x-amz-checksum-crc32c", &mut checksum_crc32c)),
|
||||
"SHA1" => Some(("x-amz-checksum-sha1", &mut checksum_sha1)),
|
||||
"SHA256" => Some(("x-amz-checksum-sha256", &mut checksum_sha256)),
|
||||
"CRC64NVME" => Some(("x-amz-checksum-crc64nvme", &mut checksum_crc64nvme)),
|
||||
_ => None,
|
||||
};
|
||||
if let Some((name, field)) = echo {
|
||||
match trailers.lookup(name) {
|
||||
rustfs_rio::TrailerValue::Present(value) => *field = Some(value),
|
||||
rustfs_rio::TrailerValue::Missing => *field = None,
|
||||
rustfs_rio::TrailerValue::Pending => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -441,8 +470,10 @@ impl DefaultMultipartUsecase {
|
||||
.push((crate::app::object::cuobject::RDMA_BYTES_TRANSFERRED_HEADER, moved.to_string()));
|
||||
}
|
||||
|
||||
let output = UploadPartOutput {
|
||||
server_side_encryption: requested_sse.map(ServerSideEncryption::from),
|
||||
let mut headers = HeaderMap::new();
|
||||
crate::app::object_usecase::inject_additional_checksum_headers(&mut headers, &upload_part_extra_checksum_headers);
|
||||
Ok(UploadPartWriteOutput {
|
||||
server_side_encryption: requested_sse,
|
||||
ssekms_key_id: requested_kms_key_id,
|
||||
sse_customer_algorithm,
|
||||
sse_customer_key_md5,
|
||||
@@ -451,16 +482,9 @@ impl DefaultMultipartUsecase {
|
||||
checksum_sha1,
|
||||
checksum_sha256,
|
||||
checksum_crc64nvme,
|
||||
e_tag: info.etag.map(|etag| to_s3s_etag(&etag)),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let mut response = S3Response::new(output);
|
||||
crate::app::object_usecase::inject_additional_checksum_headers(
|
||||
&mut response.headers,
|
||||
&upload_part_extra_checksum_headers,
|
||||
);
|
||||
Ok(response)
|
||||
e_tag: info.etag,
|
||||
headers,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -468,12 +492,15 @@ impl DefaultMultipartUsecase {
|
||||
mod tests {
|
||||
use super::super::test_support::*;
|
||||
use super::*;
|
||||
use crate::app::object::legacy_put_body;
|
||||
use crate::app::storage_api::s3::S3ErrorCode as LegacyErrorCode;
|
||||
use crate::app::storage_api::s3::dto::{StreamingBlob, UploadPartInput};
|
||||
|
||||
fn presigned_checksum_request(query: &str) -> S3Request<UploadPartInput> {
|
||||
fn presigned_checksum_request(query: &str) -> (RequestEnvelope, UploadPartWriteInput) {
|
||||
let mut req = build_request(UploadPartInput::default(), Method::PUT);
|
||||
req.uri = format!("/bucket/object?{query}").parse().unwrap();
|
||||
req.extensions.insert(VerifiedPresignedRequest);
|
||||
req
|
||||
(RequestEnvelope::from_s3s(&req), UploadPartWriteInput::from_legacy(req.input, None))
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -483,15 +510,12 @@ mod tests {
|
||||
let query =
|
||||
"x-amz-checksum-sha256=ungWv48Bz%2BpBQUDeXa4iI7ADYaOWF3qctBD%2FYfIAFa0%3D&x-amz-sdk-checksum-algorithm=SHA256";
|
||||
for payload in [b"abc", b"abd"] {
|
||||
let mut req = presigned_checksum_request(query);
|
||||
normalize_presigned_part_checksums(&mut req).unwrap();
|
||||
assert_eq!(req.input.checksum_sha256.as_deref(), Some(checksum));
|
||||
assert_eq!(
|
||||
req.input.checksum_algorithm.as_ref().map(|algorithm| algorithm.as_str()),
|
||||
Some(ChecksumAlgorithm::SHA256)
|
||||
);
|
||||
let (mut req, mut input) = presigned_checksum_request(query);
|
||||
normalize_presigned_part_checksums(&mut req, &mut input).unwrap();
|
||||
assert_eq!(input.checksum_sha256.as_deref(), Some(checksum));
|
||||
assert_eq!(input.checksum_algorithm.as_deref(), Some("SHA256"));
|
||||
let mut reader = HashReader::from_stream(Cursor::new(payload), 3, 3, None, None, false).unwrap();
|
||||
reader.add_checksum(&req.headers, None, false).unwrap();
|
||||
reader.add_checksum(req.headers(), None, false).unwrap();
|
||||
assert_eq!(reader.content_crc_type(), Some(rustfs_rio::ChecksumType::SHA256));
|
||||
let mut bytes = Vec::new();
|
||||
let result = reader.read_to_end(&mut bytes).await;
|
||||
@@ -508,11 +532,12 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn normalize_presigned_part_checksums_accepts_sdk_crc32_default() {
|
||||
let mut req = presigned_checksum_request("x-amz-checksum-crc32=y%2FQ5Jg%3D%3D&x-amz-sdk-checksum-algorithm=CRC32");
|
||||
normalize_presigned_part_checksums(&mut req).unwrap();
|
||||
assert_eq!(req.input.checksum_crc32.as_deref(), Some("y/Q5Jg=="));
|
||||
let (mut req, mut input) =
|
||||
presigned_checksum_request("x-amz-checksum-crc32=y%2FQ5Jg%3D%3D&x-amz-sdk-checksum-algorithm=CRC32");
|
||||
normalize_presigned_part_checksums(&mut req, &mut input).unwrap();
|
||||
assert_eq!(input.checksum_crc32.as_deref(), Some("y/Q5Jg=="));
|
||||
let mut reader = HashReader::from_stream(Cursor::new(b"123456789"), 9, 9, None, None, false).unwrap();
|
||||
reader.add_checksum(&req.headers, None, false).unwrap();
|
||||
reader.add_checksum(req.headers(), None, false).unwrap();
|
||||
let mut bytes = Vec::new();
|
||||
reader.read_to_end(&mut bytes).await.unwrap();
|
||||
assert_eq!(bytes, b"123456789");
|
||||
@@ -525,44 +550,47 @@ mod tests {
|
||||
"x-amz-checksum-sha256=one&x-amz-checksum-sha256=two",
|
||||
"x-amz-sdk-checksum-algorithm=SHA256&x-amz-sdk-checksum-algorithm=CRC32",
|
||||
] {
|
||||
let mut req = presigned_checksum_request(query);
|
||||
let (mut req, mut input) = presigned_checksum_request(query);
|
||||
assert_eq!(
|
||||
*normalize_presigned_part_checksums(&mut req).unwrap_err().code(),
|
||||
*normalize_presigned_part_checksums(&mut req, &mut input).unwrap_err().code(),
|
||||
S3ErrorCode::InvalidRequest
|
||||
);
|
||||
}
|
||||
let mut req = presigned_checksum_request("x-amz-checksum-sha256=one");
|
||||
req.headers.insert("x-amz-checksum-sha256", HeaderValue::from_static("two"));
|
||||
let (mut req, mut input) = presigned_checksum_request("x-amz-checksum-sha256=one");
|
||||
req.headers_mut()
|
||||
.insert("x-amz-checksum-sha256", HeaderValue::from_static("two"));
|
||||
assert_eq!(
|
||||
*normalize_presigned_part_checksums(&mut req).unwrap_err().code(),
|
||||
*normalize_presigned_part_checksums(&mut req, &mut input).unwrap_err().code(),
|
||||
S3ErrorCode::InvalidRequest
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn normalize_presigned_part_checksums_preserves_headers_and_rejects_invalid_values() {
|
||||
let mut req = presigned_checksum_request("partNumber=1&uploadId=test");
|
||||
req.input.checksum_sha256 = Some("existing".to_owned());
|
||||
req.headers
|
||||
let (mut req, mut input) = presigned_checksum_request("partNumber=1&uploadId=test");
|
||||
input.checksum_sha256 = Some("existing".to_owned());
|
||||
req.headers_mut()
|
||||
.insert("x-amz-checksum-sha256", HeaderValue::from_static("existing"));
|
||||
normalize_presigned_part_checksums(&mut req).unwrap();
|
||||
assert_eq!(req.input.checksum_sha256.as_deref(), Some("existing"));
|
||||
assert_eq!(req.headers["x-amz-checksum-sha256"], "existing");
|
||||
normalize_presigned_part_checksums(&mut req, &mut input).unwrap();
|
||||
assert_eq!(input.checksum_sha256.as_deref(), Some("existing"));
|
||||
assert_eq!(req.headers()["x-amz-checksum-sha256"], "existing");
|
||||
|
||||
let mut req = presigned_checksum_request("x-amz-checksum-sha256=%0D%0A");
|
||||
let (mut req, mut input) = presigned_checksum_request("x-amz-checksum-sha256=%0D%0A");
|
||||
assert_eq!(
|
||||
*normalize_presigned_part_checksums(&mut req).unwrap_err().code(),
|
||||
*normalize_presigned_part_checksums(&mut req, &mut input).unwrap_err().code(),
|
||||
S3ErrorCode::InvalidArgument
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn normalize_presigned_part_checksums_requires_verified_presign() {
|
||||
let mut req = presigned_checksum_request("x-amz-checksum-sha256=one");
|
||||
req.extensions.remove::<VerifiedPresignedRequest>();
|
||||
normalize_presigned_part_checksums(&mut req).unwrap();
|
||||
assert!(req.input.checksum_sha256.is_none());
|
||||
assert!(req.headers.is_empty());
|
||||
let mut legacy = build_request(UploadPartInput::default(), Method::PUT);
|
||||
legacy.uri = "/bucket/object?x-amz-checksum-sha256=one".parse().unwrap();
|
||||
let mut req = RequestEnvelope::from_s3s(&legacy);
|
||||
let mut input = UploadPartWriteInput::from_legacy(legacy.input, None);
|
||||
normalize_presigned_part_checksums(&mut req, &mut input).unwrap();
|
||||
assert!(input.checksum_sha256.is_none());
|
||||
assert!(req.headers().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -570,12 +598,15 @@ mod tests {
|
||||
let result: rustfs_s3_types::S3Result<Option<i64>> = resolve_upload_part_size(Some(5242880), None);
|
||||
assert_eq!(result.expect("DTO length"), Some(5242880));
|
||||
assert_eq!(resolve_upload_part_size(None, None).expect("unknown length"), None);
|
||||
let body = StreamingBlob::from(Bytes::from_static(b"abc"));
|
||||
let body = Body::from(Bytes::from_static(b"abc"));
|
||||
assert_eq!(resolve_upload_part_size(None, Some(&body)).expect("exact bytes"), Some(3));
|
||||
assert_eq!(resolve_upload_part_size(Some(0), Some(&body)).expect("explicit length wins"), Some(0));
|
||||
let body = StreamingBlob::wrap(futures::stream::iter([Ok::<_, std::io::Error>(Bytes::from_static(b"abc"))]));
|
||||
assert_eq!(futures::Stream::size_hint(&body), (1, Some(1)), "the stream knows its item count");
|
||||
assert_eq!(resolve_upload_part_size(None, Some(&body)).expect("unknown byte length"), None);
|
||||
assert_eq!(
|
||||
resolve_upload_part_size(None, Some(&legacy_put_body(body))).expect("unknown byte length"),
|
||||
None
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -621,7 +652,7 @@ mod tests {
|
||||
let req = build_request(input, Method::PUT);
|
||||
|
||||
let err = make_usecase().execute_upload_part(req).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::IncompleteBody);
|
||||
assert_eq!(err.code(), &LegacyErrorCode::IncompleteBody);
|
||||
}
|
||||
|
||||
/// issue #7596: a part whose declared length exceeds the 5 GiB
|
||||
@@ -646,10 +677,10 @@ mod tests {
|
||||
|
||||
let err = make_usecase().execute_upload_part(req).await.unwrap_err();
|
||||
if expect_too_large {
|
||||
assert_eq!(err.code(), &S3ErrorCode::EntityTooLarge, "declared {declared}");
|
||||
assert_eq!(err.code(), &LegacyErrorCode::EntityTooLarge, "declared {declared}");
|
||||
assert_eq!(polls.load(std::sync::atomic::Ordering::SeqCst), 0, "body must not be polled");
|
||||
} else {
|
||||
assert_ne!(err.code(), &S3ErrorCode::EntityTooLarge, "declared {declared} must pass admission");
|
||||
assert_ne!(err.code(), &LegacyErrorCode::EntityTooLarge, "declared {declared} must pass admission");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -667,7 +698,7 @@ mod tests {
|
||||
let req = build_request(input, Method::PUT);
|
||||
|
||||
let err = make_usecase().execute_upload_part(req).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
|
||||
assert_eq!(err.code(), &LegacyErrorCode::InvalidArgument);
|
||||
assert_eq!(err.message(), Some("partNumber must be between 1 and 10000"));
|
||||
}
|
||||
}
|
||||
@@ -734,9 +765,9 @@ mod tests {
|
||||
assert_eq!(
|
||||
error.code(),
|
||||
&if capped || content_length.is_some() {
|
||||
S3ErrorCode::UnexpectedContent
|
||||
LegacyErrorCode::UnexpectedContent
|
||||
} else {
|
||||
S3ErrorCode::MissingContentLength
|
||||
LegacyErrorCode::MissingContentLength
|
||||
}
|
||||
);
|
||||
assert_eq!(polls.load(std::sync::atomic::Ordering::Relaxed), 0);
|
||||
@@ -796,7 +827,7 @@ mod tests {
|
||||
));
|
||||
control.advance_wait_for_test(Duration::from_secs(60));
|
||||
sender.send(Ok(Frame::data(Bytes::from_static(b"x")))).unwrap();
|
||||
assert_eq!(owner.await.unwrap().unwrap_err().code(), &S3ErrorCode::RequestTimeout);
|
||||
assert_eq!(owner.await.unwrap().unwrap_err().code(), &LegacyErrorCode::RequestTimeout);
|
||||
assert!(matches!(
|
||||
manager.admit_put_object(65536).await.unwrap(),
|
||||
ForegroundWriteAdmission::Admitted(_)
|
||||
@@ -852,9 +883,11 @@ mod tests {
|
||||
.execute_upload_part(build_request(input, Method::PUT))
|
||||
.await
|
||||
.expect_err("full foreground write admission should reject before body ingest");
|
||||
assert_eq!(err.code(), &S3ErrorCode::SlowDown);
|
||||
assert_eq!(err.code(), &LegacyErrorCode::SlowDown);
|
||||
drop(held);
|
||||
}
|
||||
|
||||
mod body_read_tests;
|
||||
mod claim_tests;
|
||||
pub(super) use claim_tests::observe_lookup;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,127 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Upload target ownership and refusal order against real multipart state.
|
||||
|
||||
use super::*;
|
||||
use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions};
|
||||
use rustfs_gateway_types::{BucketName, ObjectKey, UploadIdClaim};
|
||||
use std::cell::Cell;
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
enum LookupResult {
|
||||
Real,
|
||||
ForeignBucket,
|
||||
ForeignKey,
|
||||
Outage,
|
||||
}
|
||||
|
||||
tokio::task_local! {
|
||||
static LOOKUP: (Cell<usize>, LookupResult);
|
||||
}
|
||||
|
||||
pub(in crate::app::multipart_usecase::upload_part) fn observe_lookup(
|
||||
result: Result<MultipartInfo, StorageError>,
|
||||
) -> Result<MultipartInfo, StorageError> {
|
||||
let Ok(mode) = LOOKUP.try_with(|(calls, mode)| {
|
||||
calls.set(calls.get() + 1);
|
||||
*mode
|
||||
}) else {
|
||||
return result;
|
||||
};
|
||||
let mut info = result?;
|
||||
match mode {
|
||||
LookupResult::Real => {}
|
||||
LookupResult::ForeignBucket => info.bucket = "foreign-bucket".to_owned(),
|
||||
LookupResult::ForeignKey => info.object = "foreign-key".to_owned(),
|
||||
LookupResult::Outage => return Err(StorageError::ErasureReadQuorum),
|
||||
}
|
||||
Ok(info)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn native_claim_checks_order_ownership_and_one_active_lookup() {
|
||||
let store = crate::app::gating_test_env::shared_gating_ecstore().await;
|
||||
let context = crate::app::gating_test_env::shared_gating_ambient().await;
|
||||
let usecase = DefaultMultipartUsecase::with_context(Some(context));
|
||||
let bucket = format!("part-claim-{}", Uuid::new_v4().simple());
|
||||
let key = "encoded + key/";
|
||||
let opts = ObjectOptions::default();
|
||||
store.make_bucket(&bucket, &MakeBucketOptions::default()).await.unwrap();
|
||||
let active = store.new_multipart_upload(&bucket, key, &opts).await.unwrap().upload_id;
|
||||
|
||||
for (number, id, length, body, mode, calls, expected) in [
|
||||
(1, active.as_str(), Some(3), true, LookupResult::Real, 1, None),
|
||||
(0, "/impossible", Some(3), true, LookupResult::Real, 0, Some("InvalidArgument")),
|
||||
(-1, "/impossible", Some(3), true, LookupResult::Real, 0, Some("InvalidArgument")),
|
||||
(10001, "/impossible", Some(3), true, LookupResult::Real, 0, Some("InvalidArgument")),
|
||||
(2, "/impossible", Some(3), false, LookupResult::Real, 0, Some("IncompleteBody")),
|
||||
(2, active.as_str(), Some(3), false, LookupResult::Real, 0, Some("IncompleteBody")),
|
||||
(2, "/impossible", Some(i64::MAX), true, LookupResult::Real, 0, Some("EntityTooLarge")),
|
||||
(2, "", Some(3), true, LookupResult::Real, 0, Some("NoSuchUpload")),
|
||||
(2, "/impossible", Some(-1), true, LookupResult::Real, 0, Some("NoSuchUpload")),
|
||||
(2, "malformed", None, true, LookupResult::Real, 1, Some("NoSuchUpload")),
|
||||
(2, "malformed", Some(-1), true, LookupResult::Real, 1, Some("NoSuchUpload")),
|
||||
(2, active.as_str(), Some(-1), true, LookupResult::Real, 1, Some("UnexpectedContent")),
|
||||
(2, active.as_str(), None, true, LookupResult::Real, 1, Some("MissingContentLength")),
|
||||
(2, active.as_str(), Some(3), true, LookupResult::ForeignBucket, 1, Some("NoSuchUpload")),
|
||||
(2, active.as_str(), Some(3), true, LookupResult::ForeignKey, 1, Some("NoSuchUpload")),
|
||||
(2, active.as_str(), Some(3), true, LookupResult::Outage, 1, Some("ServiceUnavailable")),
|
||||
(10000, active.as_str(), Some(3), true, LookupResult::Real, 1, None),
|
||||
] {
|
||||
let mut input = UploadPartWriteInput::from_legacy(UploadPartInput::default(), None);
|
||||
input.target = UploadPartTarget::Gateway {
|
||||
bucket: BucketName::new(bucket.clone()).unwrap(),
|
||||
key: ObjectKey::new(key).unwrap(),
|
||||
upload_id: UploadIdClaim::from_wire(id),
|
||||
};
|
||||
input.part_number = number;
|
||||
input.content_length = length;
|
||||
input.body = body.then(|| {
|
||||
if length.is_none() {
|
||||
legacy_put_body(StreamingBlob::wrap(futures::stream::pending::<Result<Bytes, std::io::Error>>()))
|
||||
} else {
|
||||
Body::from(Bytes::from_static(b"abc"))
|
||||
}
|
||||
});
|
||||
let envelope = RequestEnvelope::from_s3s(&build_request((), Method::PUT));
|
||||
LOOKUP
|
||||
.scope((Cell::new(0), mode), async {
|
||||
let result = usecase.execute_upload_part_owned(envelope, input).await;
|
||||
assert_eq!(
|
||||
result.as_ref().err().map(|err| err.code().as_str()),
|
||||
expected,
|
||||
"part {number}, lookup {mode:?}"
|
||||
);
|
||||
assert_eq!(LOOKUP.with(|(observed, _)| observed.get()), calls, "part {number}, lookup {mode:?}");
|
||||
if let Ok(output) = result {
|
||||
assert_eq!(output.e_tag.as_deref(), Some("900150983cd24fb0d6963f7d28e17f72"));
|
||||
}
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
let parts = store
|
||||
.list_object_parts(&bucket, key, &active, None, 20, &opts)
|
||||
.await
|
||||
.unwrap()
|
||||
.parts;
|
||||
let parts: Vec<_> = parts
|
||||
.iter()
|
||||
.map(|part| (part.part_num, part.size, part.etag.as_deref()))
|
||||
.collect();
|
||||
let etag = Some("900150983cd24fb0d6963f7d28e17f72");
|
||||
assert_eq!(parts, [(1, 3, etag), (10000, 3, etag)]);
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Owned UploadPart values projected by each S3 edge.
|
||||
|
||||
use crate::app::object::FrameTrailers;
|
||||
use rustfs_gateway_stream::Body;
|
||||
#[cfg(any(feature = "gateway-stack", test))]
|
||||
use rustfs_gateway_types::{BucketName, ObjectKey, UploadIdClaim};
|
||||
use rustfs_rio::SharedTrailerSource;
|
||||
use std::sync::Arc;
|
||||
|
||||
pub(crate) enum UploadPartTarget {
|
||||
Legacy {
|
||||
bucket: String,
|
||||
key: String,
|
||||
upload_id: String,
|
||||
},
|
||||
#[cfg(any(feature = "gateway-stack", test))]
|
||||
Gateway {
|
||||
bucket: BucketName,
|
||||
key: ObjectKey,
|
||||
upload_id: UploadIdClaim,
|
||||
},
|
||||
}
|
||||
|
||||
impl UploadPartTarget {
|
||||
pub(super) fn resource(&self) -> (&str, &str) {
|
||||
match self {
|
||||
Self::Legacy { bucket, key, .. } => (bucket, key),
|
||||
#[cfg(any(feature = "gateway-stack", test))]
|
||||
Self::Gateway { bucket, key, .. } => (bucket.as_str(), key.as_str()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) struct UploadPartWriteInput {
|
||||
pub(crate) target: UploadPartTarget,
|
||||
pub(crate) part_number: i32,
|
||||
pub(crate) content_length: Option<i64>,
|
||||
pub(crate) body: Option<Body>,
|
||||
pub(crate) trailing_headers: Option<SharedTrailerSource>,
|
||||
pub(crate) frame_trailers: Option<Arc<FrameTrailers>>,
|
||||
pub(crate) content_md5: Option<String>,
|
||||
pub(crate) checksum_algorithm: Option<String>,
|
||||
pub(crate) checksum_crc32: Option<String>,
|
||||
pub(crate) checksum_crc32c: Option<String>,
|
||||
pub(crate) checksum_crc64nvme: Option<String>,
|
||||
pub(crate) checksum_sha1: Option<String>,
|
||||
pub(crate) checksum_sha256: Option<String>,
|
||||
pub(crate) sse_customer_algorithm: Option<String>,
|
||||
pub(crate) sse_customer_key: Option<String>,
|
||||
pub(crate) sse_customer_key_md5: Option<String>,
|
||||
}
|
||||
|
||||
pub(crate) struct UploadPartWriteOutput {
|
||||
pub(crate) e_tag: Option<String>,
|
||||
pub(crate) checksum_crc32: Option<String>,
|
||||
pub(crate) checksum_crc32c: Option<String>,
|
||||
pub(crate) checksum_crc64nvme: Option<String>,
|
||||
pub(crate) checksum_sha1: Option<String>,
|
||||
pub(crate) checksum_sha256: Option<String>,
|
||||
pub(crate) server_side_encryption: Option<String>,
|
||||
pub(crate) ssekms_key_id: Option<String>,
|
||||
pub(crate) sse_customer_algorithm: Option<String>,
|
||||
pub(crate) sse_customer_key_md5: Option<String>,
|
||||
pub(crate) headers: http::HeaderMap,
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user