mirror of
https://github.com/rustfs/rustfs.git
synced 2026-07-30 01:58:59 +00:00
Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 overtrue
|
94f9c5b718 |
@@ -42,7 +42,7 @@ e2e-reliability = { max-threads = 1 }
|
||||
|
||||
# --- default profile (local): serialize the flaky groups, never retry --------
|
||||
[[profile.default.overrides]]
|
||||
filter = 'package(rustfs-ecstore) & (test(concurrent_resend_same_part_commits_one_generation) | test(/^store::bucket::tests::bucket_delete_(mark_delete|purge_removes|default_s3_delete)/))'
|
||||
filter = 'package(rustfs-ecstore) & (test(concurrent_resend_same_part_commits_one_generation) | test(/^store::bucket::tests::bucket_delete_(mark_delete_marks|purge_removes|default_s3_delete)/))'
|
||||
test-group = 'ecstore-serial-flaky'
|
||||
|
||||
# Serialize the multipart crash-consistency scenarios (dist-2, backlog#1150):
|
||||
@@ -99,7 +99,7 @@ retries = 2
|
||||
# QUARANTINE: OPEN backlog#937 — store::bucket::tests::bucket_delete_* race
|
||||
# make_bucket into InsufficientWriteQuorum via shared global state under load.
|
||||
[[profile.ci.overrides]]
|
||||
filter = 'package(rustfs-ecstore) & test(/^store::bucket::tests::bucket_delete_(mark_delete|purge_removes|default_s3_delete)/)'
|
||||
filter = 'package(rustfs-ecstore) & test(/^store::bucket::tests::bucket_delete_(mark_delete_marks|purge_removes|default_s3_delete)/)'
|
||||
test-group = 'ecstore-serial-flaky'
|
||||
retries = 2
|
||||
|
||||
@@ -192,7 +192,7 @@ test-group = 'ecstore-serial-flaky'
|
||||
[profile.e2e-smoke]
|
||||
default-filter = """
|
||||
package(e2e_test) & (
|
||||
test(/^(delete_marker_migration_semantics|version_id_regression|list_objects_v2_pagination|list_object_versions_regression|list_objects_duplicates|list_buckets_double_slash|leading_slash_key|special_chars|create_bucket_region|delete_objects_versioning|head_object_consistency|head_object_range|copy_object_metadata|copy_source_invalid_date|content_encoding|anonymous_access|bucket_policy_check|presigned_negative|negative_sigv4|admin_auth|notification_webhook|tls_hot_reload|console_smoke|admin_iam_crud|admin_pools)_test::|^fake_s3_target::/)
|
||||
test(/^(delete_marker_migration_semantics|version_id_regression|list_objects_v2_pagination|list_object_versions_regression|list_objects_duplicates|list_buckets_double_slash|leading_slash_key|special_chars|create_bucket_region|delete_objects_versioning|head_object_consistency|head_object_range|copy_object_metadata|copy_source_invalid_date|content_encoding|anonymous_access|bucket_policy_check|presigned_negative|negative_sigv4|admin_auth|notification_webhook|tls_hot_reload|console_smoke|admin_iam_crud)_test::|^fake_s3_target::/)
|
||||
| test(/^replication_extension_test::(test_replication_check_succeeds_with_remote_target|test_replication_check_rejects_target_without_object_lock|test_set_remote_target_rejects_unversioned_source_bucket|test_replication_check_rejects_unversioned_source_bucket|test_replication_check_rejects_missing_replication_config|test_replication_check_rejects_invalid_bucket|test_set_remote_target_rejects_same_bucket_on_same_deployment|test_set_remote_target_rejects_unversioned_target_bucket|test_set_remote_target_update_requires_arn|test_set_remote_target_update_rejects_missing_target|test_set_remote_target_rejects_invalid_target_url|test_set_remote_target_rejects_self_signed_https_target_without_skip_tls_verify|test_set_remote_target_rejects_private_ca_https_target_without_ca_cert_pem|test_list_remote_targets_rejects_empty_bucket|test_list_remote_targets_rejects_invalid_bucket|test_remove_remote_target_rejects_missing_target|test_remove_remote_target_rejects_missing_arn|test_remove_remote_target_rejects_invalid_bucket|test_remove_remote_target_rejects_target_used_by_replication|test_delete_bucket_replication_removes_remote_target)$/)
|
||||
| test(/^reliant::lifecycle::/)
|
||||
| test(/^reliant::tiering::/)
|
||||
|
||||
@@ -1744,7 +1744,7 @@
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "max by (job, bucket) (rustfs_cluster_usage_buckets_objects_count{job=~\"$job\", bucket=~\"$bucket\"})",
|
||||
"expr": "sum by (bucket) (rustfs_bucket_api_objects_total{job=~\"$job\", bucket=~\"$bucket\"})",
|
||||
"legendFormat": "{{bucket}}",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
@@ -1844,7 +1844,7 @@
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "max by (job, bucket) (rustfs_cluster_usage_buckets_total_bytes{job=~\"$job\", bucket=~\"$bucket\"})",
|
||||
"expr": "sum by (bucket) (rustfs_bucket_api_usage_bytes{job=~\"$job\", bucket=~\"$bucket\"})",
|
||||
"legendFormat": "{{bucket}}",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
@@ -11583,7 +11583,7 @@
|
||||
"text": "All",
|
||||
"value": "$__all"
|
||||
},
|
||||
"definition": "label_values(rustfs_cluster_usage_buckets_objects_count,bucket)",
|
||||
"definition": "label_values(rustfs_bucket_api_objects_total,bucket)",
|
||||
"includeAll": true,
|
||||
"label": "Bucket",
|
||||
"multi": true,
|
||||
@@ -11591,7 +11591,7 @@
|
||||
"options": [],
|
||||
"query": {
|
||||
"qryType": 1,
|
||||
"query": "label_values(rustfs_cluster_usage_buckets_objects_count,bucket)",
|
||||
"query": "label_values(rustfs_bucket_api_objects_total,bucket)",
|
||||
"refId": "PrometheusVariableQueryEditor-VariableQuery"
|
||||
},
|
||||
"refresh": 2,
|
||||
|
||||
@@ -18,7 +18,6 @@ set -eu
|
||||
ACCESS_KEY="${RUSTFS_SITE_REPL_ACCESS_KEY:-rustfsadmin}"
|
||||
SECRET_KEY="${RUSTFS_SITE_REPL_SECRET_KEY:-rustfsadmin}"
|
||||
BUCKET="${RUSTFS_SITE_REPL_FLOW_BUCKET:-site-repl-flow-check}"
|
||||
DELETE_BUCKET="${RUSTFS_SITE_REPL_DELETE_BUCKET:-site-repl-delete-$(date +%Y%m%d-%H%M%S)-$$}"
|
||||
PREFIX="${RUSTFS_SITE_REPL_FLOW_PREFIX:-flow-$(date +%Y%m%d-%H%M%S)}"
|
||||
WAIT_ATTEMPTS="${RUSTFS_SITE_REPL_WAIT_ATTEMPTS:-90}"
|
||||
WAIT_SLEEP_SECONDS="${RUSTFS_SITE_REPL_WAIT_SLEEP_SECONDS:-2}"
|
||||
@@ -86,39 +85,17 @@ wait_for_object() {
|
||||
|
||||
wait_for_bucket() {
|
||||
site="$1"
|
||||
bucket="${2:-$BUCKET}"
|
||||
attempt=1
|
||||
|
||||
while [ "$attempt" -le "$WAIT_ATTEMPTS" ]; do
|
||||
if mc stat "$site/$bucket" >/dev/null 2>&1; then
|
||||
if mc stat "$site/$BUCKET" >/dev/null 2>&1; then
|
||||
return 0
|
||||
fi
|
||||
sleep "$WAIT_SLEEP_SECONDS"
|
||||
attempt=$((attempt + 1))
|
||||
done
|
||||
|
||||
echo "bucket was not replicated in time: $site/$bucket" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
wait_for_bucket_delete() {
|
||||
site="$1"
|
||||
bucket="$2"
|
||||
attempt=1
|
||||
|
||||
while [ "$attempt" -le "$WAIT_ATTEMPTS" ]; do
|
||||
if result="$(mc stat --json "$site/$bucket" 2>&1)"; then
|
||||
:
|
||||
else
|
||||
case "$result" in
|
||||
*NoSuchBucket*) return 0 ;;
|
||||
esac
|
||||
fi
|
||||
sleep "$WAIT_SLEEP_SECONDS"
|
||||
attempt=$((attempt + 1))
|
||||
done
|
||||
|
||||
echo "bucket deletion was not replicated in time: $site/$bucket" >&2
|
||||
echo "bucket was not replicated in time: $site/$BUCKET" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
@@ -209,20 +186,6 @@ EOF
|
||||
echo "verified replicated downloads for $object_name"
|
||||
done
|
||||
|
||||
echo "creating empty bucket for replicated delete check: $DELETE_BUCKET"
|
||||
mc mb "site1/$DELETE_BUCKET" >/dev/null
|
||||
|
||||
for site in site1 site2 site3; do
|
||||
wait_for_bucket "$site" "$DELETE_BUCKET"
|
||||
done
|
||||
|
||||
echo "deleting empty bucket on site1: $DELETE_BUCKET"
|
||||
mc rb "site1/$DELETE_BUCKET" >/dev/null
|
||||
|
||||
for site in site1 site2 site3; do
|
||||
wait_for_bucket_delete "$site" "$DELETE_BUCKET"
|
||||
done
|
||||
|
||||
echo "site replication object flow check passed"
|
||||
echo "bucket: $BUCKET"
|
||||
echo "prefix: $PREFIX"
|
||||
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 105 KiB |
Generated
+82
-101
@@ -694,9 +694,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "async-nats"
|
||||
version = "0.50.0"
|
||||
version = "0.49.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d83a251fa1a4c9d0fe6e816b7acd60549e473e08d14f27a1d992c2675abff05f"
|
||||
checksum = "fad3cd6df81292728e2a8cb1f1dcb4d7e7a1ab59b80c14fbbcba2baf9d5cf86a"
|
||||
dependencies = [
|
||||
"base64 0.22.1",
|
||||
"bytes",
|
||||
@@ -769,7 +769,7 @@ checksum = "ae36dc4177970ef04fde5178d3e2429882def40e57a451f919c098f72baa6cec"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.2",
|
||||
"syn 3.0.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1519,7 +1519,7 @@ version = "0.10.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
|
||||
dependencies = [
|
||||
"generic-array 0.14.7",
|
||||
"generic-array 0.14.9",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1538,7 +1538,7 @@ version = "0.3.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93"
|
||||
dependencies = [
|
||||
"generic-array 0.14.7",
|
||||
"generic-array 0.14.9",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1861,7 +1861,7 @@ version = "0.4.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad"
|
||||
dependencies = [
|
||||
"crypto-common 0.1.7",
|
||||
"crypto-common 0.1.6",
|
||||
"inout 0.1.4",
|
||||
]
|
||||
|
||||
@@ -1879,9 +1879,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "clap"
|
||||
version = "4.6.3"
|
||||
version = "4.6.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0fb99565819980999fb7b4a1796046a5c949e6d4ff132cf5fadf5a641e20d776"
|
||||
checksum = "dd059f9da4f5c36b3787f65d38ccaab1cc315f07b01f89abc8359ee6a8205011"
|
||||
dependencies = [
|
||||
"clap_builder",
|
||||
"clap_derive",
|
||||
@@ -1901,9 +1901,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "clap_derive"
|
||||
version = "4.6.3"
|
||||
version = "4.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "32f2392eae7f16557a3d727ef3a12e57b2b2ca6f98566a5f4fb41ffe305df077"
|
||||
checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9"
|
||||
dependencies = [
|
||||
"heck",
|
||||
"proc-macro2",
|
||||
@@ -2319,7 +2319,7 @@ version = "0.5.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76"
|
||||
dependencies = [
|
||||
"generic-array 0.14.7",
|
||||
"generic-array 0.14.9",
|
||||
"rand_core 0.6.4",
|
||||
"subtle",
|
||||
"zeroize",
|
||||
@@ -2344,11 +2344,11 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "crypto-common"
|
||||
version = "0.1.7"
|
||||
version = "0.1.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a"
|
||||
checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3"
|
||||
dependencies = [
|
||||
"generic-array 0.14.7",
|
||||
"generic-array 0.14.9",
|
||||
"typenum",
|
||||
]
|
||||
|
||||
@@ -2437,16 +2437,15 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "curve25519-dalek"
|
||||
version = "5.0.0"
|
||||
version = "5.0.0-rc.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b5eed333089e2e1c1ac8c6c0398e5e2497b4c9926ca6d0365ed1e099afa5bc23"
|
||||
checksum = "c906a87e53a36ff795d72e06e8162a83c5436e3ea89e942a9cb9fc083f0a384f"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"cpufeatures 0.3.0",
|
||||
"curve25519-dalek-derive",
|
||||
"digest 0.11.3",
|
||||
"fiat-crypto 0.3.0",
|
||||
"rand_core 0.10.1",
|
||||
"rustc_version",
|
||||
"subtle",
|
||||
"zeroize",
|
||||
@@ -3484,9 +3483,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "dial9-tokio-telemetry"
|
||||
version = "0.3.15"
|
||||
version = "0.3.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9b511dfd54f5191f7eb86856fe19d8e3c8f71673bd256e1bfa1c8128fbbc0cdb"
|
||||
checksum = "4ae338262e40cf9036474cc3c1f0dc67c09421674ced255b7c643060829a3f5d"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"bon",
|
||||
@@ -3545,7 +3544,7 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
|
||||
dependencies = [
|
||||
"block-buffer 0.10.4",
|
||||
"const-oid 0.9.6",
|
||||
"crypto-common 0.1.7",
|
||||
"crypto-common 0.1.6",
|
||||
"subtle",
|
||||
]
|
||||
|
||||
@@ -3768,11 +3767,11 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "ed25519-dalek"
|
||||
version = "3.0.0"
|
||||
version = "3.0.0-rc.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6ebaa1a2bf1290ab3bfe5a7b771d050ebffab2711c19a81691c683a5144a25de"
|
||||
checksum = "1685663e23882cd8517dcbcb1c23a6ebff4433c22dfb681d760219b62cd1b849"
|
||||
dependencies = [
|
||||
"curve25519-dalek 5.0.0",
|
||||
"curve25519-dalek 5.0.0-rc.1",
|
||||
"ed25519 3.0.0",
|
||||
"rand_core 0.10.1",
|
||||
"serde",
|
||||
@@ -3798,7 +3797,7 @@ dependencies = [
|
||||
"crypto-bigint 0.5.5",
|
||||
"digest 0.10.7",
|
||||
"ff 0.13.1",
|
||||
"generic-array 0.14.7",
|
||||
"generic-array 0.14.9",
|
||||
"group 0.13.0",
|
||||
"hkdf 0.12.4",
|
||||
"pem-rfc7468 0.7.0",
|
||||
@@ -3866,9 +3865,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "enumset"
|
||||
version = "1.1.14"
|
||||
version = "1.1.13"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ccc5801fd11762e24d1e420d01d2ac518f2a2ca4329d4fbb6639f2412b6204e0"
|
||||
checksum = "839c4174b41e75c8f7306110b2c51996a293b8d1d850edd529011841d9fede7d"
|
||||
dependencies = [
|
||||
"enumset_derive",
|
||||
]
|
||||
@@ -4244,9 +4243,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "generic-array"
|
||||
version = "0.14.7"
|
||||
version = "0.14.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
|
||||
checksum = "4bb6743198531e02858aeaea5398fcc883e71851fcbcb5a2f773e2fb6cb1edf2"
|
||||
dependencies = [
|
||||
"typenum",
|
||||
"version_check",
|
||||
@@ -4259,7 +4258,7 @@ version = "1.4.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ab4e5aa225bc56696909483320f0ff9b600f1a971b52e07a17d70f3d9b43254b"
|
||||
dependencies = [
|
||||
"generic-array 0.14.7",
|
||||
"generic-array 0.14.9",
|
||||
"rustversion",
|
||||
"typenum",
|
||||
]
|
||||
@@ -5021,9 +5020,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "hyper"
|
||||
version = "1.11.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d22053281f852e11534f5198498373cbb59295120a20771d90f7ed1897490a72"
|
||||
version = "1.10.1"
|
||||
source = "git+https://github.com/hyperium/hyper.git?rev=ccc1e850dc0cda3e71b0acd11f60ca3d48d09034#ccc1e850dc0cda3e71b0acd11f60ca3d48d09034"
|
||||
dependencies = [
|
||||
"atomic-waker",
|
||||
"bytes",
|
||||
@@ -5259,7 +5257,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01"
|
||||
dependencies = [
|
||||
"block-padding 0.3.3",
|
||||
"generic-array 0.14.7",
|
||||
"generic-array 0.14.9",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -5710,9 +5708,9 @@ checksum = "34b357333733e8260735ba5894eb928c02ecc69c78715f01a8019e7fa7f2db4c"
|
||||
|
||||
[[package]]
|
||||
name = "libc"
|
||||
version = "0.2.187"
|
||||
version = "0.2.186"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a7743783ea728ef5c31194c6590797eed286449b4a4e87d626d8a51f0a94e732"
|
||||
checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66"
|
||||
|
||||
[[package]]
|
||||
name = "libflate"
|
||||
@@ -6256,7 +6254,7 @@ dependencies = [
|
||||
"module-lattice",
|
||||
"pkcs8 0.11.0",
|
||||
"rand_core 0.10.1",
|
||||
"sha3 0.11.0",
|
||||
"sha3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -6292,9 +6290,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "mqttbytes-core-next"
|
||||
version = "0.33.3"
|
||||
version = "0.33.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3ff7ae19c74aba9e0ed6e4071cd52aa364e020076fa3cc6ef17e43662f756f3c"
|
||||
checksum = "6d9e8f1593f6a8affb39a2df042cc28df033ba18907858cde71993e5780d29a3"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"thiserror 2.0.19",
|
||||
@@ -6674,7 +6672,7 @@ version = "5.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "51e219e79014df21a225b1860a479e2dcd7cbd9130f4defd4bd0e191ea31d67d"
|
||||
dependencies = [
|
||||
"base64 0.22.1",
|
||||
"base64 0.21.7",
|
||||
"chrono",
|
||||
"getrandom 0.2.17",
|
||||
"http 1.4.2",
|
||||
@@ -7037,9 +7035,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "p256"
|
||||
version = "0.14.0"
|
||||
version = "0.14.0-rc.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d2c9239b2dbc807adbbe147e8cf72ea7450c3a0aabe62cb8e75ff4ec22e1f72a"
|
||||
checksum = "b6bb40a5099e2c38a09dd29321a7a7f045f165a54317679c7cdfb0cbaf8f6b1e"
|
||||
dependencies = [
|
||||
"ecdsa 0.17.0",
|
||||
"elliptic-curve 0.14.1",
|
||||
@@ -7062,9 +7060,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "p384"
|
||||
version = "0.14.0"
|
||||
version = "0.14.0-rc.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d17b851e6b3e378ab4ecb07fa2ed23f4d15f075735f8fec9fa1e7bdce5f8301f"
|
||||
checksum = "492f329d7eb11d22dadc988626b9ea1f503b4ab043a8b1e4e2cc4ae45dabdd70"
|
||||
dependencies = [
|
||||
"ecdsa 0.17.0",
|
||||
"elliptic-curve 0.14.1",
|
||||
@@ -7076,9 +7074,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "p521"
|
||||
version = "0.14.0"
|
||||
version = "0.14.0-rc.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4ad64cc32c2dc466317c12ee5853e61f159f9eab1fe7efade0395dc2e7b43449"
|
||||
checksum = "ff42e4ace5424e3b6d7cb82514be89866b85af87015f80341e37dcc21a66ce6e"
|
||||
dependencies = [
|
||||
"base16ct 1.0.0",
|
||||
"ecdsa 0.17.0",
|
||||
@@ -7796,7 +7794,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf"
|
||||
dependencies = [
|
||||
"heck",
|
||||
"itertools 0.14.0",
|
||||
"itertools 0.10.5",
|
||||
"log",
|
||||
"multimap",
|
||||
"once_cell",
|
||||
@@ -7816,7 +7814,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042"
|
||||
dependencies = [
|
||||
"heck",
|
||||
"itertools 0.14.0",
|
||||
"itertools 0.10.5",
|
||||
"log",
|
||||
"multimap",
|
||||
"petgraph 0.8.3",
|
||||
@@ -7837,7 +7835,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"itertools 0.14.0",
|
||||
"itertools 0.10.5",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.119",
|
||||
@@ -7850,7 +7848,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"itertools 0.14.0",
|
||||
"itertools 0.10.5",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.119",
|
||||
@@ -8394,7 +8392,7 @@ checksum = "2c9283685feec7d69af75fb0e858d5e7378f33fe4fc699383b2916ab9273e03c"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.2",
|
||||
"syn 3.0.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -8596,9 +8594,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rumqttc-core-next"
|
||||
version = "0.33.3"
|
||||
version = "0.33.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7d7d9205738dd41a2546e82d27a634d07d8b303dcf7558565ff70caf3ceb0f9c"
|
||||
checksum = "eaa975ffc1af5aac70a0e139491a1fc553ab48f378f909dad5d43fd4b2ad28db"
|
||||
dependencies = [
|
||||
"async-tungstenite",
|
||||
"futures-io",
|
||||
@@ -8614,18 +8612,18 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rumqttc-next"
|
||||
version = "0.33.3"
|
||||
version = "0.33.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ed1bad2180ff539da671da9a996152a921bc5316eb6d8a9cc3bd441653138b08"
|
||||
checksum = "df63fbd71ba5830b5bd0952ae37761e3af64aeae8f6213eb6d31d6e72a16d71e"
|
||||
dependencies = [
|
||||
"rumqttc-v5-next",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rumqttc-v5-next"
|
||||
version = "0.33.3"
|
||||
version = "0.33.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "229576cbedfa9089f90c17c9454e9429ac1e89cdd223bac5cb39d837593f79bc"
|
||||
checksum = "6417b354aa28ad72b271e6a5cea7fdb1594d18cb040ebed35fa87bb683cf364f"
|
||||
dependencies = [
|
||||
"async-tungstenite",
|
||||
"bytes",
|
||||
@@ -8648,9 +8646,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "russh"
|
||||
version = "0.62.3"
|
||||
version = "0.62.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "059dd24c0fe20721639f7acad7b82cd51ec3dd3254ed8cf7a0b7df6c20eaff1c"
|
||||
checksum = "ca6c3c1dd0a9ad3a9915a2f6e907d158f9a7e9ac32ddc772b003faafc027d9a8"
|
||||
dependencies = [
|
||||
"aes 0.9.1",
|
||||
"aws-lc-rs",
|
||||
@@ -8662,13 +8660,13 @@ dependencies = [
|
||||
"cipher 0.5.2",
|
||||
"crypto-bigint 0.7.5",
|
||||
"ctr",
|
||||
"curve25519-dalek 5.0.0",
|
||||
"curve25519-dalek 5.0.0-rc.1",
|
||||
"data-encoding",
|
||||
"delegate",
|
||||
"der 0.8.1",
|
||||
"digest 0.11.3",
|
||||
"ecdsa 0.17.0",
|
||||
"ed25519-dalek 3.0.0",
|
||||
"ed25519-dalek 3.0.0-rc.1",
|
||||
"elliptic-curve 0.14.1",
|
||||
"enum_dispatch",
|
||||
"flate2",
|
||||
@@ -8686,8 +8684,8 @@ dependencies = [
|
||||
"ml-kem",
|
||||
"module-lattice",
|
||||
"num-bigint",
|
||||
"p256 0.14.0",
|
||||
"p384 0.14.0",
|
||||
"p256 0.14.0-rc.15",
|
||||
"p384 0.14.0-rc.15",
|
||||
"p521",
|
||||
"pageant",
|
||||
"pbkdf2 0.13.0",
|
||||
@@ -8705,7 +8703,7 @@ dependencies = [
|
||||
"sec1 0.8.1",
|
||||
"sha1 0.11.0",
|
||||
"sha2 0.11.0",
|
||||
"sha3 0.12.0",
|
||||
"sha3",
|
||||
"signature 3.0.0",
|
||||
"spki 0.8.0",
|
||||
"ssh-encoding",
|
||||
@@ -10455,7 +10453,7 @@ checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc"
|
||||
dependencies = [
|
||||
"base16ct 0.2.0",
|
||||
"der 0.7.10",
|
||||
"generic-array 0.14.7",
|
||||
"generic-array 0.14.9",
|
||||
"pkcs8 0.10.2",
|
||||
"subtle",
|
||||
"zeroize",
|
||||
@@ -10561,14 +10559,14 @@ checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.2",
|
||||
"syn 3.0.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_json"
|
||||
version = "1.0.151"
|
||||
version = "1.0.150"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14"
|
||||
checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9"
|
||||
dependencies = [
|
||||
"indexmap 2.14.0",
|
||||
"itoa",
|
||||
@@ -10615,7 +10613,7 @@ checksum = "8d3b1629de253c70a0508c3899572da79ca359fdab27c7920ff00406df418906"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.2",
|
||||
"syn 3.0.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -10757,17 +10755,6 @@ dependencies = [
|
||||
"keccak",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sha3"
|
||||
version = "0.12.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bc9bad02c26382724b2d2692c6f179285e4b54eeecd7968f52a50059c3c11759"
|
||||
dependencies = [
|
||||
"digest 0.11.3",
|
||||
"keccak",
|
||||
"sponge-cursor",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "shadow-rs"
|
||||
version = "2.0.0"
|
||||
@@ -11040,12 +11027,6 @@ dependencies = [
|
||||
"der 0.8.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sponge-cursor"
|
||||
version = "0.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3a0219bd7d979d58245a4f41f695e1ac9f8befdffadd7f61f1bae9e39abc6620"
|
||||
|
||||
[[package]]
|
||||
name = "sqlparser"
|
||||
version = "0.62.0"
|
||||
@@ -11110,11 +11091,11 @@ dependencies = [
|
||||
"argon2",
|
||||
"bcrypt-pbkdf",
|
||||
"ctutils",
|
||||
"ed25519-dalek 3.0.0",
|
||||
"ed25519-dalek 3.0.0-rc.1",
|
||||
"hex",
|
||||
"hmac 0.13.0",
|
||||
"p256 0.14.0",
|
||||
"p384 0.14.0",
|
||||
"p256 0.14.0-rc.15",
|
||||
"p384 0.14.0-rc.15",
|
||||
"p521",
|
||||
"rand_core 0.10.1",
|
||||
"rsa 0.10.0-rc.18",
|
||||
@@ -11320,9 +11301,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "3.0.2"
|
||||
version = "3.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a207d6d6a2b7fc470b80443726053f18a2481b7e1eee970597051596567987a3"
|
||||
checksum = "5edbec4ed188954a10c12c038215f8ce7606b2d5c973cd8dc43e8795065c5f2f"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -11541,7 +11522,7 @@ checksum = "43cbfe0cf76104d42a574802844187e84a305e531ed54455f11fbde0f10541cd"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.2",
|
||||
"syn 3.0.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -11555,9 +11536,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "time"
|
||||
version = "0.3.54"
|
||||
version = "0.3.53"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3e1d5e639ff6bab73cb6885cc7e7b1de96c3f32c68ec55f3952614bec1092244"
|
||||
checksum = "18dfaaeddcb932337b5e7866ee7d0ce9b76d2fd092997146f187ec09b4558a50"
|
||||
dependencies = [
|
||||
"deranged",
|
||||
"js-sys",
|
||||
@@ -11578,9 +11559,9 @@ checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109"
|
||||
|
||||
[[package]]
|
||||
name = "time-macros"
|
||||
version = "0.2.32"
|
||||
version = "0.2.31"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7e689342a48d2ea927c87ea50cabf8594854bf940e9310208848d680d668ed85"
|
||||
checksum = "c431b87111666e491a90baa837f914fb45cd5dc3c268591b0220ff5057f2085f"
|
||||
dependencies = [
|
||||
"num-conv",
|
||||
"time-core",
|
||||
@@ -11665,9 +11646,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "tokio"
|
||||
version = "1.53.1"
|
||||
version = "1.53.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed"
|
||||
checksum = "d988bcd52dbe076d3d46903332f58c912b87a2c49b1428419a5845154762ffee"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"libc",
|
||||
@@ -12921,18 +12902,18 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "zerocopy"
|
||||
version = "0.8.55"
|
||||
version = "0.8.54"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb"
|
||||
checksum = "b7cbbc0a705a0fd05cc3676525980d2bf5a9bc4adac6d6475209a7887cf59d19"
|
||||
dependencies = [
|
||||
"zerocopy-derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zerocopy-derive"
|
||||
version = "0.8.55"
|
||||
version = "0.8.54"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb"
|
||||
checksum = "e2e817b7b52d0c7358d3246da9d69935ebb18116b2b102b4230dac079b4862f5"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
|
||||
+29
-11
@@ -140,7 +140,7 @@ mysql_async = { default-features = false, version = "0.37" }
|
||||
async-compression = { version = "0.4.42" }
|
||||
async-recursion = "1.1.1"
|
||||
async-trait = "0.1.91"
|
||||
async-nats = "0.50.0"
|
||||
async-nats = "0.49.1"
|
||||
axum = "0.8.9"
|
||||
futures = "0.3.33"
|
||||
futures-core = "0.3.33"
|
||||
@@ -149,17 +149,17 @@ futures-util = "0.3.33"
|
||||
pollster = "1.0.1"
|
||||
pulsar = { default-features = false, version = "6.8.0" }
|
||||
lapin = { default-features = false, version = "4.10.0" }
|
||||
hyper = { version = "1.11.0" }
|
||||
hyper = { version = "1.10.1" }
|
||||
hyper-rustls = { default-features = false, version = "0.27.9" }
|
||||
hyper-util = { version = "0.1.20" }
|
||||
http = "1.4.2"
|
||||
http-body = "1.1.0"
|
||||
http-body-util = "0.1.4"
|
||||
minlz = "1.2.3"
|
||||
reqwest = "0.13.4"
|
||||
reqwest = { default-features = false, version = "0.13.4" }
|
||||
rustfs-kafka-async = { version = "1.2.0" }
|
||||
socket2 = { version = "0.6.5" }
|
||||
tokio = { version = "1.53.1" }
|
||||
tokio = { version = "1.53.0" }
|
||||
tokio-rustls = { default-features = false, version = "0.26.4" }
|
||||
tokio-stream = { version = "0.1.18" }
|
||||
tokio-test = "0.4.5"
|
||||
@@ -182,7 +182,7 @@ quick-xml = "0.41.0"
|
||||
rmp = { version = "0.8.15" }
|
||||
rmp-serde = { version = "1.3.1" }
|
||||
serde = { version = "1.0.229" }
|
||||
serde_json = { version = "1.0.151" }
|
||||
serde_json = { version = "1.0.150" }
|
||||
serde_urlencoded = "0.7.1"
|
||||
|
||||
# Cryptography and Security
|
||||
@@ -212,7 +212,7 @@ zeroize = { version = "1.9.0" }
|
||||
chrono = { version = "0.4.45" }
|
||||
humantime = "2.4.0"
|
||||
jiff = { version = "0.2.34" }
|
||||
time = { version = "0.3.54" }
|
||||
time = { version = "0.3.53" }
|
||||
|
||||
# Database
|
||||
deadpool-postgres = { version = "0.14" }
|
||||
@@ -234,7 +234,7 @@ aws-smithy-types = { version = "1.6.1" }
|
||||
base64 = "0.22.1"
|
||||
base64-simd = "0.8.0"
|
||||
brotli = "8.0.4"
|
||||
clap = { version = "4.6.3" }
|
||||
clap = { version = "4.6.2" }
|
||||
const-str = { version = "1.1.0" }
|
||||
convert_case = "0.11.0"
|
||||
criterion = { version = "0.8" }
|
||||
@@ -244,7 +244,7 @@ crossbeam-deque = "0.8.7"
|
||||
crossbeam-utils = "0.8.22"
|
||||
datafusion = { default-features = false, git = "https://github.com/apache/datafusion.git", rev = "dae03ee062b2abf986de8df12ea82fb1578a2d99" }
|
||||
derive_builder = "0.20.2"
|
||||
enumset = "1.1.14"
|
||||
enumset = "1.1.13"
|
||||
faster-hex = "0.10.0"
|
||||
flate2 = "1.1.9"
|
||||
glob = "0.3.3"
|
||||
@@ -256,7 +256,7 @@ hex-simd = "0.8.0"
|
||||
highway = { version = "1.3.0" }
|
||||
ipnetwork = { version = "0.21.1" }
|
||||
lazy_static = "1.5.0"
|
||||
libc = "0.2.187"
|
||||
libc = "0.2.186"
|
||||
libsystemd = "0.7.2"
|
||||
local-ip-address = "0.6.13"
|
||||
memmap2 = "0.9.11"
|
||||
@@ -281,7 +281,7 @@ rayon = "1.12.0"
|
||||
reed-solomon-erasure = { package = "rustfs-erasure-codec", version = "8.0.0" }
|
||||
reed-solomon-simd = "3.1.0"
|
||||
regex = { version = "1.13.1" }
|
||||
rumqttc = { package = "rumqttc-next", version = "0.33.3" }
|
||||
rumqttc = { package = "rumqttc-next", version = "0.33.2" }
|
||||
redis = { version = "1.4.1" }
|
||||
rustix = { version = "1.1.4" }
|
||||
rust-embed = { version = "8.12.0" }
|
||||
@@ -333,7 +333,7 @@ libunftp = { version = "0.23.0" }
|
||||
unftp-core = "0.1.0"
|
||||
suppaftp = { version = "10.0.1" }
|
||||
rcgen = "0.14.8"
|
||||
russh = { version = "0.62.3" }
|
||||
russh = { version = "0.62.2" }
|
||||
russh-sftp = "2.3.0"
|
||||
|
||||
# WebDAV
|
||||
@@ -370,3 +370,21 @@ inherits = "release"
|
||||
inherits = "release"
|
||||
debug = true
|
||||
strip = "none"
|
||||
|
||||
# Pin hyper to a revision that carries the HTTP/1 "flush buffered data before
|
||||
# shutdown" fix (hyperium/hyper#4018, commit 72046cc7). This lands as a
|
||||
# `[patch.crates-io]` entry — not on the `hyper` workspace dependency — so that
|
||||
# every consumer in the tree, including the transitive `hyper-util` server path
|
||||
# (`conn::auto` / `GracefulShutdown`) that actually drives our connections,
|
||||
# resolves to the fixed hyper rather than the buggy crates.io copy.
|
||||
#
|
||||
# hyper <= 1.10.1 can call `poll_shutdown()` on the socket while response bytes
|
||||
# are still buffered (a prior `poll_flush()` returned `Poll::Pending` and the
|
||||
# result was discarded). A backpressured / slow-reading peer then receives a
|
||||
# graceful FIN before the full Content-Length body is flushed, which standard S3
|
||||
# clients (minio-go / warp) report as `unexpected EOF` on large-object GET under
|
||||
# load. The fix is not in any crates.io release yet as of hyper 1.10.1; drop
|
||||
# this patch once a released version (> 1.10.1) contains commit 72046cc7.
|
||||
# See rustfs/backlog#1232.
|
||||
[patch.crates-io]
|
||||
hyper = { git = "https://github.com/hyperium/hyper.git", rev = "ccc1e850dc0cda3e71b0acd11f60ca3d48d09034" }
|
||||
|
||||
@@ -39,11 +39,6 @@ pub const DEFAULT_DRIVE_WALKDIR_TIMEOUT_SECS: u64 = 5;
|
||||
pub const ENV_DRIVE_WALKDIR_STALL_TIMEOUT_SECS: &str = "RUSTFS_DRIVE_WALKDIR_STALL_TIMEOUT_SECS";
|
||||
pub const DEFAULT_DRIVE_WALKDIR_STALL_TIMEOUT_SECS: u64 = 5;
|
||||
|
||||
/// Maximum time the metacache merge consumer waits for the next visible
|
||||
/// `walk_dir()` entry from a reader before detaching it from the merge.
|
||||
pub const ENV_DRIVE_WALKDIR_PEEK_TIMEOUT_SECS: &str = "RUSTFS_DRIVE_WALKDIR_PEEK_TIMEOUT_SECS";
|
||||
pub const DEFAULT_DRIVE_WALKDIR_PEEK_TIMEOUT_SECS: u64 = 10;
|
||||
|
||||
/// Interval in seconds between active health probes for local and remote drives.
|
||||
pub const ENV_DRIVE_ACTIVE_CHECK_INTERVAL_SECS: &str = "RUSTFS_DRIVE_ACTIVE_CHECK_INTERVAL_SECS";
|
||||
pub const DEFAULT_DRIVE_ACTIVE_CHECK_INTERVAL_SECS: u64 = 15;
|
||||
|
||||
@@ -57,7 +57,7 @@ http.workspace = true
|
||||
http-body-util.workspace = true
|
||||
hyper = { workspace = true, features = ["http2", "http1", "server"] }
|
||||
hyper-util = { workspace = true, features = ["tokio", "server-auto", "server-graceful", "tracing"] }
|
||||
reqwest = { workspace = true, features = ["json", "multipart", "stream"] }
|
||||
reqwest = { workspace = true, default-features = false, features = ["rustls", "charset", "http2", "system-proxy", "stream", "json", "multipart"] }
|
||||
rustfs-signer.workspace = true
|
||||
tracing = { workspace = true }
|
||||
tracing-subscriber = { workspace = true, features = ["env-filter", "time"] }
|
||||
|
||||
@@ -1,79 +0,0 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
|
||||
use http::header::HOST;
|
||||
use reqwest::StatusCode;
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
use rustfs_signer::sign_v4;
|
||||
use s3s::Body;
|
||||
use serde::Deserialize;
|
||||
use std::error::Error;
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct PoolListItem {
|
||||
id: usize,
|
||||
cmdline: String,
|
||||
status: String,
|
||||
}
|
||||
|
||||
async fn signed_admin_get(env: &RustFSTestEnvironment, path: &str) -> Result<reqwest::Response, Box<dyn Error + Send + Sync>> {
|
||||
let url = format!("{}{path}", env.url);
|
||||
let uri = url.parse::<http::Uri>()?;
|
||||
let authority = uri.authority().ok_or("admin URL missing authority")?.to_string();
|
||||
let request = http::Request::builder()
|
||||
.method(http::Method::GET)
|
||||
.uri(uri)
|
||||
.header(HOST, authority)
|
||||
.header("x-amz-content-sha256", UNSIGNED_PAYLOAD)
|
||||
.body(Body::empty())?;
|
||||
let signed = sign_v4(request, 0, &env.access_key, &env.secret_key, "", "us-east-1");
|
||||
|
||||
let mut request = local_http_client().get(&url);
|
||||
for (name, value) in signed.headers() {
|
||||
request = request.header(name, value);
|
||||
}
|
||||
Ok(request.send().await?)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn single_drive_pools_list_succeeds_without_enabling_decommission_status() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
|
||||
let response = signed_admin_get(&env, "/rustfs/admin/v3/pools/list").await?;
|
||||
let status = response.status();
|
||||
let body = response.bytes().await?;
|
||||
|
||||
assert_eq!(status, StatusCode::OK, "pools list failed: {}", String::from_utf8_lossy(&body));
|
||||
let pools: Vec<PoolListItem> = serde_json::from_slice(&body)?;
|
||||
assert_eq!(pools.len(), 1);
|
||||
assert_eq!(pools[0].id, 0);
|
||||
assert_eq!(pools[0].cmdline, env.temp_dir);
|
||||
assert_eq!(pools[0].status, "active");
|
||||
|
||||
let response = signed_admin_get(&env, "/rustfs/admin/v3/decommission/status").await?;
|
||||
let status = response.status();
|
||||
let body = response.text().await?;
|
||||
assert_eq!(
|
||||
status,
|
||||
StatusCode::NOT_IMPLEMENTED,
|
||||
"decommission status changed for a single pool: {body}"
|
||||
);
|
||||
assert!(body.contains("NotImplemented"), "unexpected decommission error body: {body}");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -242,9 +242,6 @@ mod console_smoke_test;
|
||||
#[cfg(test)]
|
||||
mod admin_iam_crud_test;
|
||||
|
||||
#[cfg(test)]
|
||||
mod admin_pools_test;
|
||||
|
||||
// Replication extension end-to-end regression tests
|
||||
#[cfg(test)]
|
||||
mod replication_extension_test;
|
||||
|
||||
@@ -43,12 +43,6 @@ use s3s::Body;
|
||||
use serde_json::Value;
|
||||
use serial_test::serial;
|
||||
use std::error::Error;
|
||||
use std::path::Path;
|
||||
use std::sync::{
|
||||
Arc, Once,
|
||||
atomic::{AtomicBool, Ordering},
|
||||
};
|
||||
use std::thread;
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
use tokio::net::TcpListener;
|
||||
use tokio::sync::mpsc;
|
||||
@@ -156,83 +150,6 @@ async fn spawn_event_collector() -> Result<(String, mpsc::UnboundedReceiver<Valu
|
||||
Ok((format!("http://{endpoint_ip}.nip.io:{port}/events"), rx, handle))
|
||||
}
|
||||
|
||||
fn spawn_https_event_collector(ca_path: &Path) -> Result<(String, Arc<AtomicBool>, thread::JoinHandle<()>), BoxError> {
|
||||
use rustls::{
|
||||
ServerConfig,
|
||||
pki_types::{PrivateKeyDer, PrivatePkcs8KeyDer},
|
||||
};
|
||||
use std::io::ErrorKind;
|
||||
use std::net::TcpListener as StdTcpListener;
|
||||
|
||||
static INSTALL_CRYPTO_PROVIDER: Once = Once::new();
|
||||
INSTALL_CRYPTO_PROVIDER.call_once(|| {
|
||||
let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();
|
||||
});
|
||||
|
||||
let listener = StdTcpListener::bind("0.0.0.0:0")?;
|
||||
listener.set_nonblocking(true)?;
|
||||
let addr = listener.local_addr()?;
|
||||
let endpoint_ip = local_ip()?;
|
||||
let endpoint_host = format!("{endpoint_ip}.nip.io");
|
||||
|
||||
let rcgen::CertifiedKey { cert, signing_key } = rcgen::generate_simple_self_signed(vec![endpoint_host.clone()])?;
|
||||
std::fs::write(ca_path, cert.pem())?;
|
||||
|
||||
let cert_chain = vec![cert.der().clone()];
|
||||
let key_der = PrivateKeyDer::Pkcs8(PrivatePkcs8KeyDer::from(signing_key.serialize_der()));
|
||||
let server_config = Arc::new(
|
||||
ServerConfig::builder()
|
||||
.with_no_client_auth()
|
||||
.with_single_cert(cert_chain, key_der)?,
|
||||
);
|
||||
|
||||
let running = Arc::new(AtomicBool::new(true));
|
||||
let server_running = Arc::clone(&running);
|
||||
let handle = thread::spawn(move || {
|
||||
while server_running.load(Ordering::Relaxed) {
|
||||
match listener.accept() {
|
||||
Ok((stream, _)) => {
|
||||
let config = Arc::clone(&server_config);
|
||||
handle_https_probe(stream, config);
|
||||
}
|
||||
Err(err) if err.kind() == ErrorKind::WouldBlock => thread::sleep(Duration::from_millis(20)),
|
||||
Err(_) => break,
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
Ok((format!("https://{endpoint_host}:{}/events", addr.port()), running, handle))
|
||||
}
|
||||
|
||||
fn handle_https_probe(stream: std::net::TcpStream, server_config: Arc<rustls::ServerConfig>) {
|
||||
use std::io::{Read, Write};
|
||||
|
||||
let _ = stream.set_read_timeout(Some(Duration::from_secs(5)));
|
||||
let _ = stream.set_write_timeout(Some(Duration::from_secs(5)));
|
||||
let Ok(connection) = rustls::ServerConnection::new(server_config) else {
|
||||
return;
|
||||
};
|
||||
let mut tls_stream = rustls::StreamOwned::new(connection, stream);
|
||||
let mut buf = [0u8; 1024];
|
||||
if tls_stream.read(&mut buf).is_err() {
|
||||
return;
|
||||
}
|
||||
let response = "HTTP/1.1 200 OK\r\ncontent-length: 0\r\nconnection: close\r\n\r\n";
|
||||
let _ = tls_stream.write_all(response.as_bytes());
|
||||
let _ = tls_stream.flush();
|
||||
}
|
||||
|
||||
fn stop_https_event_collector(endpoint: &str, running: Arc<AtomicBool>, handle: thread::JoinHandle<()>) -> TestResult {
|
||||
running.store(false, Ordering::Relaxed);
|
||||
if let Ok(parsed) = endpoint.parse::<reqwest::Url>()
|
||||
&& let Some(port) = parsed.port()
|
||||
{
|
||||
let _ = std::net::TcpStream::connect(("127.0.0.1", port));
|
||||
}
|
||||
handle.join().map_err(|_| "https event collector thread panicked")?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Decoded object key of the first record in an event envelope.
|
||||
fn event_key(envelope: &Value) -> Option<String> {
|
||||
let raw = envelope["Records"][0]["s3"]["object"]["key"].as_str()?;
|
||||
@@ -367,24 +284,13 @@ async fn enable_notify_module(env: &RustFSTestEnvironment) -> TestResult {
|
||||
/// Registers a webhook notification target with a persistent queue directory, so
|
||||
/// delivery goes through the durable store-and-forward path.
|
||||
async fn configure_webhook_target(env: &RustFSTestEnvironment, target_name: &str, endpoint: &str) -> TestResult {
|
||||
configure_webhook_target_with_key_values(env, target_name, vec![("endpoint", endpoint.to_string())]).await
|
||||
}
|
||||
|
||||
async fn configure_webhook_target_with_key_values(
|
||||
env: &RustFSTestEnvironment,
|
||||
target_name: &str,
|
||||
mut key_values: Vec<(&str, String)>,
|
||||
) -> TestResult {
|
||||
let queue_dir = format!("{}/notify-queue-{target_name}", env.temp_dir);
|
||||
tokio::fs::create_dir_all(&queue_dir).await?;
|
||||
if !key_values.iter().any(|(key, _)| *key == "queue_dir") {
|
||||
key_values.push(("queue_dir", queue_dir));
|
||||
}
|
||||
let payload = serde_json::json!({
|
||||
"key_values": key_values
|
||||
.into_iter()
|
||||
.map(|(key, value)| serde_json::json!({ "key": key, "value": value }))
|
||||
.collect::<Vec<_>>(),
|
||||
"key_values": [
|
||||
{ "key": "endpoint", "value": endpoint },
|
||||
{ "key": "queue_dir", "value": queue_dir },
|
||||
]
|
||||
});
|
||||
let url = format!("{}/rustfs/admin/v3/target/notify_webhook/{target_name}", env.url);
|
||||
let response = signed_admin_request(env, http::Method::PUT, &url, Some(payload.to_string().into_bytes())).await?;
|
||||
@@ -415,28 +321,6 @@ async fn wait_for_target_registered(env: &RustFSTestEnvironment, target_name: &s
|
||||
Err(format!("target {target_name} was not registered in admin ARNs").into())
|
||||
}
|
||||
|
||||
async fn wait_for_target_listed(env: &RustFSTestEnvironment, target_name: &str) -> TestResult {
|
||||
let url = format!("{}/rustfs/admin/v3/target/list", env.url);
|
||||
for _ in 0..40 {
|
||||
let response = signed_admin_request(env, http::Method::GET, &url, None).await?;
|
||||
if response.status() == StatusCode::OK {
|
||||
let body: Value = serde_json::from_slice(&response.bytes().await?)?;
|
||||
let listed = body["notification_endpoints"].as_array().is_some_and(|endpoints| {
|
||||
endpoints.iter().any(|endpoint| {
|
||||
endpoint["account_id"].as_str() == Some(target_name)
|
||||
&& endpoint["service"].as_str() == Some("webhook")
|
||||
&& endpoint["status"].as_str().is_some()
|
||||
})
|
||||
});
|
||||
if listed {
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(250)).await;
|
||||
}
|
||||
Err(format!("target {target_name} was not listed in admin targets").into())
|
||||
}
|
||||
|
||||
/// Binds a bucket to a webhook target for ObjectCreated:*/ObjectRemoved:* events,
|
||||
/// filtered to `prefix` + `suffix`.
|
||||
async fn put_notification_config(client: &Client, bucket: &str, target_name: &str, prefix: &str, suffix: &str) -> TestResult {
|
||||
@@ -483,38 +367,6 @@ fn trimmed_etag(value: Option<&str>) -> Option<String> {
|
||||
// Tests
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Regression for rustfs#5052: with the notify module enabled through
|
||||
/// RUSTFS_NOTIFY_ENABLE, an HTTPS webhook using a configured CA must be accepted
|
||||
/// and remain visible in the admin target list.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_https_webhook_target_lists_with_notify_env_enabled() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server_with_env(vec![], &[("RUSTFS_NOTIFY_ENABLE", "true")])
|
||||
.await?;
|
||||
|
||||
let ca_path = Path::new(&env.temp_dir).join("https-webhook-ca.pem");
|
||||
let (endpoint, running, handle) = spawn_https_event_collector(&ca_path)?;
|
||||
let target = "peri1https";
|
||||
|
||||
configure_webhook_target_with_key_values(
|
||||
&env,
|
||||
target,
|
||||
vec![
|
||||
("endpoint", endpoint.clone()),
|
||||
("client_ca", ca_path.to_string_lossy().into_owned()),
|
||||
],
|
||||
)
|
||||
.await?;
|
||||
wait_for_target_listed(&env, target).await?;
|
||||
|
||||
env.stop_server();
|
||||
stop_https_event_collector(&endpoint, running, handle)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// PUT / multipart-complete / DELETE each deliver one event with correct fields,
|
||||
/// and the prefix/suffix filter drops non-matching keys.
|
||||
#[tokio::test]
|
||||
|
||||
@@ -1971,10 +1971,7 @@ async fn wait_for_site_replication_enabled(
|
||||
) -> Result<SiteReplicationInfo, Box<dyn Error + Send + Sync>> {
|
||||
for _ in 0..40 {
|
||||
let info = site_replication_info(env).await?;
|
||||
if info.enabled
|
||||
&& info.sites.len() == expected_sites
|
||||
&& info.sites.iter().all(|peer| peer.sync_state == SyncStatus::Enable)
|
||||
{
|
||||
if info.enabled && info.sites.len() == expected_sites {
|
||||
return Ok(info);
|
||||
}
|
||||
sleep(Duration::from_millis(250)).await;
|
||||
|
||||
@@ -123,7 +123,7 @@ libc.workspace = true
|
||||
# (backlog#1178); "fs" comes from the workspace default.
|
||||
rustix = { workspace = true, features = ["process", "fs"] }
|
||||
rustfs-madmin.workspace = true
|
||||
reqwest = { workspace = true }
|
||||
reqwest = { workspace = true, default-features = false, features = ["rustls", "http2", "system-proxy"] }
|
||||
aes-gcm = { workspace = true, features = ["rand_core"] }
|
||||
chacha20poly1305.workspace = true
|
||||
aws-sdk-s3 = { workspace = true, default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
|
||||
|
||||
@@ -67,8 +67,6 @@ pub mod bucket {
|
||||
}
|
||||
|
||||
pub mod tier_delete_journal {
|
||||
#[cfg(feature = "test-util")]
|
||||
pub use crate::bucket::lifecycle::tier_delete_journal::recover_tier_delete_journal_entries;
|
||||
pub use crate::bucket::lifecycle::tier_delete_journal::{
|
||||
persist_tier_delete_journal_entry, record_tier_delete_journal_backend_identity,
|
||||
};
|
||||
@@ -426,7 +424,7 @@ pub mod tier {
|
||||
pub mod tier_config {
|
||||
pub use crate::services::tier::tier_config::{
|
||||
ServicePrincipalAuth, TierAliyun, TierAzure, TierConfig, TierGCS, TierHuaweicloud, TierMinIO, TierR2, TierRustFS,
|
||||
TierS3, TierTencent, TierType, TierWasabi,
|
||||
TierS3, TierTencent, TierType,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -437,13 +435,6 @@ pub mod tier {
|
||||
};
|
||||
}
|
||||
|
||||
pub mod tier_mutation_peer {
|
||||
pub use crate::services::tier::tier_mutation_peer::{
|
||||
MAX_TIER_MUTATION_PEER_COMMIT_ETAG_SIZE, TierMutationPeerError, TierMutationPeerOutcome, TierMutationPeerResult,
|
||||
TierMutationPeerState, handle_tier_mutation_peer_request,
|
||||
};
|
||||
}
|
||||
|
||||
pub mod warm_backend {
|
||||
pub use crate::services::tier::warm_backend::{
|
||||
WarmBackend, WarmBackendGetOpts, WarmBackendImpl, build_transition_put_options, check_warm_backend, new_warm_backend,
|
||||
@@ -453,9 +444,9 @@ pub mod tier {
|
||||
#[cfg(feature = "test-util")]
|
||||
pub mod test_util {
|
||||
pub use crate::services::tier::test_util::{
|
||||
FaultConfig, MockStoredObject, MockWarmBackend, MockWarmOp, TransitionCleanupStoreBarrier, TransitionMeta,
|
||||
assert_transition_meta_consistent, free_version_count, read_transition_meta, register_mock_tier,
|
||||
register_mock_tier_backend, wait_for_free_version_absence,
|
||||
FaultConfig, MockStoredObject, MockWarmBackend, MockWarmOp, TransitionMeta, assert_transition_meta_consistent,
|
||||
free_version_count, read_transition_meta, register_mock_tier, register_mock_tier_backend,
|
||||
wait_for_free_version_absence,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -28,4 +28,3 @@ pub mod tier_delete_journal;
|
||||
pub mod tier_free_version_recovery;
|
||||
pub mod tier_last_day_stats;
|
||||
pub mod tier_sweeper;
|
||||
pub mod transition_transaction;
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::{future::Future, sync::Arc, time::Duration};
|
||||
use std::sync::Arc;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sha2::{Digest, Sha256};
|
||||
@@ -38,11 +38,8 @@ const LOG_SUBSYSTEM_LIFECYCLE: &str = "lifecycle";
|
||||
const EVENT_LIFECYCLE_TIER_DELETE_JOURNAL: &str = "lifecycle_tier_delete_journal";
|
||||
|
||||
pub const DEFAULT_TIER_DELETE_JOURNAL_RECOVERY_LIMIT: usize = 1_000;
|
||||
const TIER_DELETE_JOURNAL_RECOVERY_INTERVAL: Duration = Duration::from_secs(60);
|
||||
const TIER_DELETE_JOURNAL_RECOVERY_TIMEOUT: Duration = Duration::from_secs(300);
|
||||
const TIER_DELETE_JOURNAL_VERSION: u8 = 2;
|
||||
const TIER_DELETE_JOURNAL_EXACT_VERSION: u8 = 3;
|
||||
pub(crate) const TIER_DELETE_JOURNAL_PREFIX: &str = "ilm/tier-delete-journal/";
|
||||
const TIER_DELETE_JOURNAL_PREFIX: &str = "ilm/tier-delete-journal/";
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
@@ -53,16 +50,12 @@ struct PersistedTierDeleteJournalEntry {
|
||||
tier_name: String,
|
||||
#[serde(default)]
|
||||
backend_identity: Option<[u8; 32]>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
version_id_exact: Option<bool>,
|
||||
}
|
||||
|
||||
impl PersistedTierDeleteJournalEntry {
|
||||
fn from_jentry(je: &Jentry) -> Self {
|
||||
Self {
|
||||
version: if je.version_id_exact {
|
||||
TIER_DELETE_JOURNAL_EXACT_VERSION
|
||||
} else if je.backend_identity.is_some() {
|
||||
version: if je.backend_identity.is_some() {
|
||||
TIER_DELETE_JOURNAL_VERSION
|
||||
} else {
|
||||
1
|
||||
@@ -71,7 +64,6 @@ impl PersistedTierDeleteJournalEntry {
|
||||
version_id: je.version_id.clone(),
|
||||
tier_name: je.tier_name.clone(),
|
||||
backend_identity: je.backend_identity,
|
||||
version_id_exact: je.version_id_exact.then_some(true),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -84,32 +76,12 @@ impl PersistedTierDeleteJournalEntry {
|
||||
if self.obj_name.is_empty() || self.tier_name.is_empty() {
|
||||
return Err(Error::other("tier delete journal entry is incomplete"));
|
||||
}
|
||||
if self.version != TIER_DELETE_JOURNAL_EXACT_VERSION && self.version_id_exact.unwrap_or(false) {
|
||||
return Err(Error::other(
|
||||
"legacy tier delete journal entry has an unsupported exact version constraint",
|
||||
));
|
||||
}
|
||||
let (backend_identity, version_id_exact) = match self.version {
|
||||
1 => (None, false),
|
||||
TIER_DELETE_JOURNAL_VERSION => (
|
||||
Some(
|
||||
self.backend_identity
|
||||
.ok_or_else(|| Error::other("tier delete journal v2 entry is missing its backend identity"))?,
|
||||
),
|
||||
false,
|
||||
let backend_identity = match self.version {
|
||||
1 => None,
|
||||
TIER_DELETE_JOURNAL_VERSION => Some(
|
||||
self.backend_identity
|
||||
.ok_or_else(|| Error::other("tier delete journal v2 entry is missing its backend identity"))?,
|
||||
),
|
||||
TIER_DELETE_JOURNAL_EXACT_VERSION => {
|
||||
if self.version_id.is_empty() || self.version_id_exact != Some(true) {
|
||||
return Err(Error::other("tier delete journal v3 entry is missing its exact version constraint"));
|
||||
}
|
||||
(
|
||||
Some(
|
||||
self.backend_identity
|
||||
.ok_or_else(|| Error::other("tier delete journal v3 entry is missing its backend identity"))?,
|
||||
),
|
||||
true,
|
||||
)
|
||||
}
|
||||
version => return Err(Error::other(format!("unsupported tier delete journal version {version}"))),
|
||||
};
|
||||
Ok(Jentry {
|
||||
@@ -117,7 +89,6 @@ impl PersistedTierDeleteJournalEntry {
|
||||
version_id: self.version_id,
|
||||
tier_name: self.tier_name,
|
||||
backend_identity,
|
||||
version_id_exact,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -142,10 +113,6 @@ pub(crate) fn tier_delete_journal_object_name(je: &Jentry) -> String {
|
||||
hasher.update([0]);
|
||||
hasher.update(backend_identity);
|
||||
}
|
||||
if je.version_id_exact {
|
||||
hasher.update([0]);
|
||||
hasher.update(b"exact-version-id");
|
||||
}
|
||||
format!(
|
||||
"{TIER_DELETE_JOURNAL_PREFIX}{}.json",
|
||||
rustfs_utils::crypto::hex(hasher.finalize().as_slice())
|
||||
@@ -218,7 +185,6 @@ pub async fn process_tier_delete_journal_entry(api: Arc<ECStore>, je: &Jentry) -
|
||||
&je.tier_name,
|
||||
backend_identity,
|
||||
&api.tier_config_mgr(),
|
||||
je.version_id_exact,
|
||||
)
|
||||
.await?;
|
||||
remove_tier_delete_journal_entry(api, je).await
|
||||
@@ -328,33 +294,16 @@ pub async fn recover_tier_delete_journal_entries(
|
||||
}
|
||||
|
||||
pub async fn run_tier_delete_journal_recovery_loop(api: Arc<ECStore>, cancel_token: CancellationToken) {
|
||||
let mut interval = tokio::time::interval(TIER_DELETE_JOURNAL_RECOVERY_INTERVAL);
|
||||
interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
|
||||
let mut interval = tokio::time::interval(std::time::Duration::from_secs(60));
|
||||
let mut marker: Option<String> = None;
|
||||
|
||||
loop {
|
||||
#[cfg(test)]
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = cancel_token.cancelled() => return,
|
||||
_ = interval.tick() => {},
|
||||
_ = api.ctx.wait_for_tier_delete_journal_recovery() => {},
|
||||
}
|
||||
#[cfg(not(test))]
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = cancel_token.cancelled() => return,
|
||||
_ = interval.tick() => {},
|
||||
_ = interval.tick() => {}
|
||||
}
|
||||
|
||||
let recovery =
|
||||
recover_tier_delete_journal_entries(api.clone(), DEFAULT_TIER_DELETE_JOURNAL_RECOVERY_LIMIT, marker.clone());
|
||||
let Some(result) =
|
||||
await_tier_delete_journal_recovery(&cancel_token, TIER_DELETE_JOURNAL_RECOVERY_TIMEOUT, recovery).await
|
||||
else {
|
||||
return;
|
||||
};
|
||||
match result {
|
||||
match recover_tier_delete_journal_entries(api.clone(), DEFAULT_TIER_DELETE_JOURNAL_RECOVERY_LIMIT, marker.clone()).await {
|
||||
Ok(stats) => {
|
||||
marker = stats.next_marker;
|
||||
debug!(
|
||||
@@ -383,36 +332,13 @@ pub async fn run_tier_delete_journal_recovery_loop(api: Arc<ECStore>, cancel_tok
|
||||
}
|
||||
}
|
||||
|
||||
async fn await_tier_delete_journal_recovery<T, F>(
|
||||
cancel_token: &CancellationToken,
|
||||
timeout: Duration,
|
||||
recovery: F,
|
||||
) -> Option<Result<T>>
|
||||
where
|
||||
F: Future<Output = Result<T>>,
|
||||
{
|
||||
tokio::select! {
|
||||
_ = cancel_token.cancelled() => None,
|
||||
result = tokio::time::timeout(timeout, recovery) => Some(match result {
|
||||
Ok(result) => result,
|
||||
Err(_) => Err(Error::other(format!(
|
||||
"tier delete journal recovery timed out after {} seconds",
|
||||
timeout.as_secs()
|
||||
))),
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{
|
||||
TIER_DELETE_JOURNAL_EXACT_VERSION, await_tier_delete_journal_recovery, decode_tier_delete_journal_entry,
|
||||
encode_tier_delete_journal_entry, record_tier_delete_journal_backend_identity, tier_delete_journal_object_name,
|
||||
decode_tier_delete_journal_entry, encode_tier_delete_journal_entry, record_tier_delete_journal_backend_identity,
|
||||
tier_delete_journal_object_name,
|
||||
};
|
||||
use crate::bucket::lifecycle::tier_sweeper::Jentry;
|
||||
use crate::error::Result;
|
||||
use std::time::Duration;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
fn journal_entry() -> Jentry {
|
||||
Jentry {
|
||||
@@ -420,7 +346,6 @@ mod tests {
|
||||
version_id: "remote-version".to_string(),
|
||||
tier_name: "WARM".to_string(),
|
||||
backend_identity: Some([7; 32]),
|
||||
version_id_exact: false,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -435,82 +360,6 @@ mod tests {
|
||||
assert_eq!(decoded.version_id, je.version_id);
|
||||
assert_eq!(decoded.tier_name, je.tier_name);
|
||||
assert_eq!(decoded.backend_identity, je.backend_identity);
|
||||
assert_eq!(decoded.version_id_exact, je.version_id_exact);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier_delete_journal_roundtrips_exact_put_response_constraint() {
|
||||
let mut exact = journal_entry();
|
||||
exact.version_id = uuid::Uuid::nil().to_string();
|
||||
exact.version_id_exact = true;
|
||||
let mut normalized = exact.clone();
|
||||
normalized.version_id_exact = false;
|
||||
|
||||
let encoded = encode_tier_delete_journal_entry(&exact).expect("exact journal entry should encode");
|
||||
let persisted: serde_json::Value = serde_json::from_slice(&encoded).expect("exact journal JSON should decode");
|
||||
let decoded = decode_tier_delete_journal_entry(&encoded).expect("exact journal entry should decode");
|
||||
|
||||
assert_eq!(persisted["version"], TIER_DELETE_JOURNAL_EXACT_VERSION);
|
||||
assert_eq!(persisted["version_id_exact"], true);
|
||||
assert!(decoded.version_id_exact);
|
||||
assert_ne!(tier_delete_journal_object_name(&exact), tier_delete_journal_object_name(&normalized));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier_delete_journal_rejects_invalid_exact_version_constraints() {
|
||||
let identity = vec![7_u8; 32];
|
||||
let invalid = [
|
||||
serde_json::json!({
|
||||
"version": 1,
|
||||
"obj_name": "remote/object",
|
||||
"version_id": "exact-version",
|
||||
"tier_name": "WARM",
|
||||
"version_id_exact": true,
|
||||
}),
|
||||
serde_json::json!({
|
||||
"version": 2,
|
||||
"obj_name": "remote/object",
|
||||
"version_id": "exact-version",
|
||||
"tier_name": "WARM",
|
||||
"backend_identity": identity,
|
||||
"version_id_exact": true,
|
||||
}),
|
||||
serde_json::json!({
|
||||
"version": TIER_DELETE_JOURNAL_EXACT_VERSION,
|
||||
"obj_name": "remote/object",
|
||||
"version_id": "",
|
||||
"tier_name": "WARM",
|
||||
"backend_identity": identity,
|
||||
"version_id_exact": true,
|
||||
}),
|
||||
serde_json::json!({
|
||||
"version": TIER_DELETE_JOURNAL_EXACT_VERSION,
|
||||
"obj_name": "remote/object",
|
||||
"version_id": "exact-version",
|
||||
"tier_name": "WARM",
|
||||
"backend_identity": identity,
|
||||
}),
|
||||
serde_json::json!({
|
||||
"version": TIER_DELETE_JOURNAL_EXACT_VERSION,
|
||||
"obj_name": "remote/object",
|
||||
"version_id": "exact-version",
|
||||
"tier_name": "WARM",
|
||||
"backend_identity": identity,
|
||||
"version_id_exact": false,
|
||||
}),
|
||||
serde_json::json!({
|
||||
"version": TIER_DELETE_JOURNAL_EXACT_VERSION,
|
||||
"obj_name": "remote/object",
|
||||
"version_id": "exact-version",
|
||||
"tier_name": "WARM",
|
||||
"version_id_exact": true,
|
||||
}),
|
||||
];
|
||||
|
||||
for persisted in invalid {
|
||||
let encoded = serde_json::to_vec(&persisted).expect("invalid journal fixture should encode");
|
||||
decode_tier_delete_journal_entry(&encoded).expect_err("invalid exact journal constraint must fail closed");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -627,29 +476,4 @@ mod tests {
|
||||
|
||||
assert!(err.to_string().contains("decode tier delete journal failed"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn tier_delete_journal_recovery_has_a_hard_outer_timeout() {
|
||||
let result = await_tier_delete_journal_recovery(
|
||||
&CancellationToken::new(),
|
||||
Duration::from_millis(10),
|
||||
std::future::pending::<Result<()>>(),
|
||||
)
|
||||
.await
|
||||
.expect("an elapsed timeout should return a recovery error")
|
||||
.expect_err("a permanently pending recovery must time out");
|
||||
|
||||
assert!(result.to_string().contains("recovery timed out"), "{result}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn tier_delete_journal_recovery_drops_in_flight_work_on_shutdown() {
|
||||
let cancel = CancellationToken::new();
|
||||
cancel.cancel();
|
||||
|
||||
let result =
|
||||
await_tier_delete_journal_recovery(&cancel, Duration::from_secs(30), std::future::pending::<Result<()>>()).await;
|
||||
|
||||
assert!(result.is_none(), "shutdown must cancel the in-flight recovery future");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,14 +13,11 @@
|
||||
// limitations under the License.
|
||||
|
||||
use std::sync::Arc;
|
||||
#[cfg(test)]
|
||||
use std::sync::Mutex;
|
||||
use std::time::Duration;
|
||||
|
||||
use tokio::sync::mpsc;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
use crate::bucket::lifecycle::bucket_lifecycle_ops::enqueue_recovered_free_version;
|
||||
use crate::disk::RUSTFS_META_BUCKET;
|
||||
use crate::error::Result;
|
||||
use crate::object_api::ObjectInfo;
|
||||
@@ -34,78 +31,10 @@ use rustfs_filemeta::FileInfo;
|
||||
pub const DEFAULT_FREE_VERSION_RECOVERY_LIMIT: usize = 1_000;
|
||||
const DEFAULT_FREE_VERSION_RECOVERY_SCAN_LIMIT: usize = 10_000;
|
||||
const BACKGROUND_WALKDIR_TIMEOUT: Duration = Duration::from_secs(60);
|
||||
#[cfg(not(test))]
|
||||
const BACKGROUND_WALK_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
#[cfg(test)]
|
||||
const BACKGROUND_WALK_SHUTDOWN_TIMEOUT: Duration = Duration::from_millis(100);
|
||||
|
||||
type ObjectInfoOrErr = StorageObjectInfoOrErr<ObjectInfo, crate::error::Error>;
|
||||
type WalkOptions = StorageWalkOptions<fn(&FileInfo) -> bool>;
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) enum RecoveryWalkTestAction {
|
||||
SendItemsThenError(Vec<ObjectInfo>, crate::error::Error),
|
||||
SendItemsThenHang(Vec<ObjectInfo>, Arc<tokio::sync::Notify>),
|
||||
SendItemsUntilReceiverCloses(Arc<tokio::sync::Notify>),
|
||||
ReturnError(crate::error::Error),
|
||||
WaitForCancellation(Arc<tokio::sync::Notify>),
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
type RecoveryWalkTestHook = Box<dyn Fn(&str) -> Option<RecoveryWalkTestAction> + Send + Sync>;
|
||||
|
||||
#[cfg(test)]
|
||||
static RECOVERY_WALK_TEST_HOOK: Mutex<Option<RecoveryWalkTestHook>> = Mutex::new(None);
|
||||
|
||||
#[cfg(test)]
|
||||
static RECOVERY_BUCKET_LIST_WAIT_HOOK: Mutex<Option<Arc<tokio::sync::Notify>>> = Mutex::new(None);
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) struct RecoveryWalkHookGuard;
|
||||
|
||||
#[cfg(test)]
|
||||
impl Drop for RecoveryWalkHookGuard {
|
||||
fn drop(&mut self) {
|
||||
let mut hook = RECOVERY_WALK_TEST_HOOK
|
||||
.lock()
|
||||
.expect("recovery walk test hook lock should not poison");
|
||||
*hook = None;
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) fn set_recovery_walk_test_hook(
|
||||
hook_fn: impl Fn(&str) -> Option<RecoveryWalkTestAction> + Send + Sync + 'static,
|
||||
) -> RecoveryWalkHookGuard {
|
||||
let mut hook = RECOVERY_WALK_TEST_HOOK
|
||||
.lock()
|
||||
.expect("recovery walk test hook lock should not poison");
|
||||
*hook = Some(Box::new(hook_fn));
|
||||
RecoveryWalkHookGuard
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) struct RecoveryBucketListWaitHookGuard;
|
||||
|
||||
#[cfg(test)]
|
||||
impl Drop for RecoveryBucketListWaitHookGuard {
|
||||
fn drop(&mut self) {
|
||||
let mut hook = RECOVERY_BUCKET_LIST_WAIT_HOOK
|
||||
.lock()
|
||||
.expect("recovery bucket-list test hook lock should not poison");
|
||||
*hook = None;
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) fn set_recovery_bucket_list_wait_hook(started: Arc<tokio::sync::Notify>) -> RecoveryBucketListWaitHookGuard {
|
||||
let mut hook = RECOVERY_BUCKET_LIST_WAIT_HOOK
|
||||
.lock()
|
||||
.expect("recovery bucket-list test hook lock should not poison");
|
||||
*hook = Some(started);
|
||||
RecoveryBucketListWaitHookGuard
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct FreeVersionRecoveryStats {
|
||||
pub scanned: usize,
|
||||
@@ -139,22 +68,12 @@ pub async fn recover_tier_free_versions(
|
||||
limit: usize,
|
||||
bucket_marker: Option<String>,
|
||||
object_marker: Option<String>,
|
||||
) -> Result<FreeVersionRecoveryStats> {
|
||||
recover_tier_free_versions_with_cancel(api, limit, bucket_marker, object_marker, CancellationToken::new()).await
|
||||
}
|
||||
|
||||
pub(super) async fn recover_tier_free_versions_with_cancel(
|
||||
api: Arc<ECStore>,
|
||||
limit: usize,
|
||||
bucket_marker: Option<String>,
|
||||
object_marker: Option<String>,
|
||||
cancel_token: CancellationToken,
|
||||
) -> Result<FreeVersionRecoveryStats> {
|
||||
if limit == 0 {
|
||||
return Err(std::io::Error::other("free-version recovery limit must be greater than zero").into());
|
||||
}
|
||||
|
||||
let page = list_tier_free_versions(api, limit, bucket_marker.clone(), object_marker.clone(), cancel_token.clone()).await?;
|
||||
let page = list_tier_free_versions(api, limit, bucket_marker.clone(), object_marker.clone()).await?;
|
||||
let mut stats = FreeVersionRecoveryStats {
|
||||
scanned: 0,
|
||||
enqueued: 0,
|
||||
@@ -168,11 +87,8 @@ pub(super) async fn recover_tier_free_versions_with_cancel(
|
||||
|
||||
let mut retry_cursor = RetryCursor::new(bucket_marker, object_marker);
|
||||
for oi in page.items {
|
||||
if cancel_token.is_cancelled() {
|
||||
return Err(tier_free_version_recovery_cancelled());
|
||||
}
|
||||
retry_cursor.visit(&oi);
|
||||
if !record_recovered_free_version_enqueue(&mut stats, enqueue_recovered_free_version(oi).await) {
|
||||
if !record_recovered_free_version_enqueue(&mut stats, queue_recovered_free_version(oi).await) {
|
||||
let (bucket_marker, object_marker) = retry_cursor.retry_markers();
|
||||
stats.truncated = true;
|
||||
stats.next_bucket_marker = bucket_marker;
|
||||
@@ -184,18 +100,6 @@ pub(super) async fn recover_tier_free_versions_with_cancel(
|
||||
Ok(stats)
|
||||
}
|
||||
|
||||
fn tier_free_version_recovery_cancelled() -> crate::error::Error {
|
||||
std::io::Error::new(std::io::ErrorKind::Interrupted, "tier free-version recovery cancelled").into()
|
||||
}
|
||||
|
||||
fn tier_free_version_recovery_walk_shutdown_timed_out() -> crate::error::Error {
|
||||
std::io::Error::new(
|
||||
std::io::ErrorKind::TimedOut,
|
||||
"tier free-version recovery walk did not stop after cancellation",
|
||||
)
|
||||
.into()
|
||||
}
|
||||
|
||||
fn record_recovered_free_version_enqueue(stats: &mut FreeVersionRecoveryStats, queued: bool) -> bool {
|
||||
stats.scanned += 1;
|
||||
if queued {
|
||||
@@ -207,6 +111,10 @@ fn record_recovered_free_version_enqueue(stats: &mut FreeVersionRecoveryStats, q
|
||||
}
|
||||
}
|
||||
|
||||
async fn queue_recovered_free_version(oi: ObjectInfo) -> bool {
|
||||
crate::bucket::lifecycle::bucket_lifecycle_ops::enqueue_recovered_free_version(oi).await
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
struct RetryCursor {
|
||||
input_bucket_marker: Option<String>,
|
||||
@@ -252,12 +160,11 @@ impl RetryCursor {
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn list_tier_free_versions(
|
||||
async fn list_tier_free_versions(
|
||||
api: Arc<ECStore>,
|
||||
limit: usize,
|
||||
bucket_marker: Option<String>,
|
||||
object_marker: Option<String>,
|
||||
cancel_token: CancellationToken,
|
||||
) -> Result<FreeVersionRecoveryPage> {
|
||||
let mut page = FreeVersionRecoveryPage {
|
||||
items: Vec::new(),
|
||||
@@ -272,42 +179,21 @@ pub(super) async fn list_tier_free_versions(
|
||||
return Ok(page);
|
||||
}
|
||||
|
||||
let bucket_options = BucketOptions::default();
|
||||
let list_buckets = async {
|
||||
#[cfg(test)]
|
||||
let wait_hook = RECOVERY_BUCKET_LIST_WAIT_HOOK
|
||||
.lock()
|
||||
.expect("recovery bucket-list test hook lock should not poison")
|
||||
.clone();
|
||||
#[cfg(test)]
|
||||
if let Some(started) = wait_hook {
|
||||
started.notify_one();
|
||||
std::future::pending::<()>().await;
|
||||
}
|
||||
api.list_bucket(&bucket_options).await
|
||||
};
|
||||
tokio::pin!(list_buckets);
|
||||
let buckets = tokio::select! {
|
||||
biased;
|
||||
_ = cancel_token.cancelled() => return Err(tier_free_version_recovery_cancelled()),
|
||||
result = &mut list_buckets => result?,
|
||||
};
|
||||
let buckets = api.list_bucket(&BucketOptions::default()).await?;
|
||||
let mut bucket_seen = bucket_marker.is_none();
|
||||
let mut truncated_after: Option<RecoveryCursor> = None;
|
||||
let walk_scan_limit = recovery_walk_scan_limit(limit);
|
||||
|
||||
for bucket in buckets {
|
||||
if cancel_token.is_cancelled() {
|
||||
return Err(tier_free_version_recovery_cancelled());
|
||||
}
|
||||
if bucket.name == RUSTFS_META_BUCKET {
|
||||
continue;
|
||||
}
|
||||
if !bucket_seen {
|
||||
if bucket_marker.as_deref().is_some_and(|marker| bucket.name.as_str() < marker) {
|
||||
if bucket_marker.as_deref() == Some(bucket.name.as_str()) {
|
||||
bucket_seen = true;
|
||||
} else {
|
||||
continue;
|
||||
}
|
||||
bucket_seen = true;
|
||||
}
|
||||
|
||||
page.buckets_scanned += 1;
|
||||
@@ -318,94 +204,16 @@ pub(super) async fn list_tier_free_versions(
|
||||
};
|
||||
|
||||
let (tx, mut rx) = mpsc::channel::<ObjectInfoOrErr>(100);
|
||||
let cancel = cancel_token.child_token();
|
||||
let cancel = CancellationToken::new();
|
||||
let mut draining_after_truncation = false;
|
||||
let mut drain_deadline = None;
|
||||
let mut last_seen_object: Option<String> = None;
|
||||
let mut scanned_objects = 0usize;
|
||||
let mut walk = tokio::spawn({
|
||||
let walk = tokio::spawn({
|
||||
let api = api.clone();
|
||||
let bucket_name = bucket.name.clone();
|
||||
let object_marker = bucket_object_marker.clone();
|
||||
let cancel = cancel.clone();
|
||||
async move {
|
||||
#[cfg(test)]
|
||||
let test_action = {
|
||||
let hook = RECOVERY_WALK_TEST_HOOK
|
||||
.lock()
|
||||
.expect("recovery walk test hook lock should not poison");
|
||||
hook.as_ref().and_then(|hook| hook(&bucket_name))
|
||||
};
|
||||
#[cfg(test)]
|
||||
if let Some(action) = test_action {
|
||||
match action {
|
||||
RecoveryWalkTestAction::SendItemsThenError(items, err) => {
|
||||
for item in items {
|
||||
if tx
|
||||
.send(ObjectInfoOrErr {
|
||||
item: Some(item),
|
||||
err: None,
|
||||
})
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
let _ = tx
|
||||
.send(ObjectInfoOrErr {
|
||||
item: None,
|
||||
err: Some(err),
|
||||
})
|
||||
.await;
|
||||
return Ok(());
|
||||
}
|
||||
RecoveryWalkTestAction::SendItemsThenHang(items, started) => {
|
||||
for item in items {
|
||||
if tx
|
||||
.send(ObjectInfoOrErr {
|
||||
item: Some(item),
|
||||
err: None,
|
||||
})
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
started.notify_one();
|
||||
return std::future::pending().await;
|
||||
}
|
||||
RecoveryWalkTestAction::SendItemsUntilReceiverCloses(started) => {
|
||||
started.notify_one();
|
||||
let mut index = 0usize;
|
||||
loop {
|
||||
if tx
|
||||
.send(ObjectInfoOrErr {
|
||||
item: Some(ObjectInfo {
|
||||
bucket: bucket_name.clone(),
|
||||
name: format!("nonrecoverable-{index:08}"),
|
||||
..Default::default()
|
||||
}),
|
||||
err: None,
|
||||
})
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
index = index.saturating_add(1);
|
||||
}
|
||||
}
|
||||
RecoveryWalkTestAction::ReturnError(err) => return Err(err),
|
||||
RecoveryWalkTestAction::WaitForCancellation(started) => {
|
||||
started.notify_one();
|
||||
cancel.cancelled().await;
|
||||
return Err(tier_free_version_recovery_cancelled());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
api.walk(
|
||||
cancel,
|
||||
&bucket_name,
|
||||
@@ -423,88 +231,47 @@ pub(super) async fn list_tier_free_versions(
|
||||
}
|
||||
});
|
||||
|
||||
let mut receive_error = None;
|
||||
loop {
|
||||
let item = tokio::select! {
|
||||
biased;
|
||||
_ = cancel_token.cancelled() => {
|
||||
cancel.cancel();
|
||||
receive_error = Some(tier_free_version_recovery_cancelled());
|
||||
break;
|
||||
}
|
||||
_ = async {
|
||||
if let Some(deadline) = drain_deadline {
|
||||
tokio::time::sleep_until(deadline).await;
|
||||
} else {
|
||||
std::future::pending::<()>().await;
|
||||
}
|
||||
}, if drain_deadline.is_some() => {
|
||||
receive_error = Some(tier_free_version_recovery_walk_shutdown_timed_out());
|
||||
break;
|
||||
}
|
||||
item = rx.recv() => match item {
|
||||
Some(item) => item,
|
||||
None => break,
|
||||
},
|
||||
};
|
||||
while let Some(item) = rx.recv().await {
|
||||
page.scanned_entries += 1;
|
||||
if let Some(err) = item.err {
|
||||
cancel.cancel();
|
||||
receive_error = Some(err);
|
||||
break;
|
||||
}
|
||||
if draining_after_truncation {
|
||||
continue;
|
||||
}
|
||||
if let Some(err) = item.err {
|
||||
cancel.cancel();
|
||||
walk.await.map_err(|err| std::io::Error::other(err.to_string()))??;
|
||||
return Err(err);
|
||||
}
|
||||
let Some(oi) = item.item else {
|
||||
continue;
|
||||
};
|
||||
record_scanned_object(&mut last_seen_object, &mut scanned_objects, &oi.name);
|
||||
if let Some(cursor) = &truncated_after
|
||||
&& (cursor.bucket.as_str() != bucket.name.as_str() || cursor.object.as_str() != oi.name.as_str())
|
||||
&& cursor.object != oi.name
|
||||
{
|
||||
page.truncated = true;
|
||||
cancel.cancel();
|
||||
draining_after_truncation = true;
|
||||
drain_deadline = Some(tokio::time::Instant::now() + BACKGROUND_WALK_SHUTDOWN_TIMEOUT);
|
||||
continue;
|
||||
}
|
||||
if is_recoverable_tier_free_version(&oi) {
|
||||
let current_cursor = RecoveryCursor {
|
||||
let cursor = RecoveryCursor {
|
||||
bucket: bucket.name.clone(),
|
||||
object: oi.name.clone(),
|
||||
};
|
||||
page.items.push(oi);
|
||||
page.next_bucket_marker = Some(current_cursor.bucket.clone());
|
||||
page.next_object_marker = Some(current_cursor.object.clone());
|
||||
page.next_bucket_marker = Some(cursor.bucket.clone());
|
||||
page.next_object_marker = Some(cursor.object.clone());
|
||||
if page.items.len() >= limit && truncated_after.is_none() {
|
||||
truncated_after = Some(current_cursor);
|
||||
truncated_after = Some(cursor);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
drop(rx);
|
||||
let walk_shutdown_timeout = drain_deadline
|
||||
.map(|deadline| deadline.saturating_duration_since(tokio::time::Instant::now()))
|
||||
.unwrap_or(BACKGROUND_WALK_SHUTDOWN_TIMEOUT);
|
||||
let walk_result = match tokio::time::timeout(walk_shutdown_timeout, &mut walk).await {
|
||||
Ok(result) => result.map_err(|err| std::io::Error::other(err.to_string()))?,
|
||||
Err(_) => {
|
||||
walk.abort();
|
||||
let _ = walk.await;
|
||||
if let Some(err) = receive_error {
|
||||
return Err(err);
|
||||
}
|
||||
return Err(tier_free_version_recovery_walk_shutdown_timed_out());
|
||||
}
|
||||
};
|
||||
if let Some(err) = receive_error {
|
||||
return Err(err);
|
||||
}
|
||||
walk_result?;
|
||||
walk.await.map_err(|err| std::io::Error::other(err.to_string()))??;
|
||||
mark_scan_truncated_if_needed(&mut page, scanned_objects, walk_scan_limit, &bucket.name, last_seen_object.as_deref());
|
||||
|
||||
if page.truncated {
|
||||
page.next_bucket_marker = Some(bucket.name.clone());
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -249,7 +249,6 @@ impl ObjSweeper {
|
||||
version_id: self.transition_version_id.clone(),
|
||||
tier_name: self.transition_tier.clone(),
|
||||
backend_identity: None,
|
||||
version_id_exact: false,
|
||||
});
|
||||
}
|
||||
None
|
||||
@@ -285,7 +284,6 @@ pub struct Jentry {
|
||||
pub(crate) version_id: String,
|
||||
pub(crate) tier_name: String,
|
||||
pub(crate) backend_identity: Option<TierDestinationId>,
|
||||
pub(crate) version_id_exact: bool,
|
||||
}
|
||||
|
||||
impl ExpiryOp for Jentry {
|
||||
@@ -330,14 +328,13 @@ async fn delete_object_from_remote_tier_raw_with_manager(
|
||||
let lease = TierConfigMgr::acquire_operation_lease(&tier_config_mgr, tier_name)
|
||||
.await
|
||||
.map_err(std::io::Error::other)?;
|
||||
delete_object_from_remote_tier_raw_with_lease(obj_name, rv_id, &lease, false).await
|
||||
delete_object_from_remote_tier_raw_with_lease(obj_name, rv_id, &lease).await
|
||||
}
|
||||
|
||||
async fn delete_object_from_remote_tier_raw_with_lease(
|
||||
obj_name: &str,
|
||||
rv_id: &str,
|
||||
lease: &TierOperationLease,
|
||||
version_id_exact: bool,
|
||||
) -> Result<(), std::io::Error> {
|
||||
if remote_delete_breaker_is_open(Instant::now()).await {
|
||||
metrics::counter!(METRIC_DELETE_REMOTE_BREAKER_TOTAL).increment(1);
|
||||
@@ -350,11 +347,7 @@ async fn delete_object_from_remote_tier_raw_with_lease(
|
||||
.map_err(|_| std::io::Error::other(ERR_REMOTE_DELETE_LIMITER_CLOSED))?;
|
||||
let _inflight = RemoteDeleteInflightGuard::new();
|
||||
|
||||
if version_id_exact {
|
||||
lease.remove_exact(obj_name, rv_id).await
|
||||
} else {
|
||||
lease.remove(obj_name, rv_id).await
|
||||
}
|
||||
lease.remove(obj_name, rv_id).await
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -366,30 +359,6 @@ fn run_remote_tier_delete_test_hook(obj_name: &str, rv_id: &str, tier_name: &str
|
||||
.map(|hook| hook(obj_name, rv_id, tier_name))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) struct RemoteTierDeleteHookGuard;
|
||||
|
||||
#[cfg(test)]
|
||||
impl Drop for RemoteTierDeleteHookGuard {
|
||||
fn drop(&mut self) {
|
||||
let mut hook = REMOTE_TIER_DELETE_TEST_HOOK
|
||||
.lock()
|
||||
.expect("remote tier delete test hook lock should not poison");
|
||||
*hook = None;
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) fn set_remote_tier_delete_test_hook(
|
||||
hook_fn: impl Fn(&str, &str, &str) -> std::io::Result<()> + Send + Sync + 'static,
|
||||
) -> RemoteTierDeleteHookGuard {
|
||||
let mut hook = REMOTE_TIER_DELETE_TEST_HOOK
|
||||
.lock()
|
||||
.expect("remote tier delete test hook lock should not poison");
|
||||
*hook = Some(Box::new(hook_fn));
|
||||
RemoteTierDeleteHookGuard
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum RemoteTierDeleteOutcome {
|
||||
Deleted,
|
||||
@@ -419,21 +388,19 @@ pub(crate) async fn delete_object_from_remote_tier_idempotent_with_manager_and_i
|
||||
tier_name: &str,
|
||||
backend_identity: TierDestinationId,
|
||||
tier_config_mgr: &Arc<tokio::sync::RwLock<TierConfigMgr>>,
|
||||
version_id_exact: bool,
|
||||
) -> Result<RemoteTierDeleteOutcome, std::io::Error> {
|
||||
let lease = TierConfigMgr::acquire_operation_lease_for_backend_identity(tier_config_mgr, tier_name, backend_identity)
|
||||
.await
|
||||
.map_err(std::io::Error::other)?;
|
||||
delete_object_from_remote_tier_with_lease_idempotent(obj_name, rv_id, &lease, version_id_exact).await
|
||||
delete_object_from_remote_tier_with_lease_idempotent(obj_name, rv_id, &lease).await
|
||||
}
|
||||
|
||||
pub(crate) async fn delete_object_from_remote_tier_with_lease_idempotent(
|
||||
obj_name: &str,
|
||||
rv_id: &str,
|
||||
lease: &TierOperationLease,
|
||||
version_id_exact: bool,
|
||||
) -> Result<RemoteTierDeleteOutcome, std::io::Error> {
|
||||
match delete_object_from_remote_tier_raw_with_lease(obj_name, rv_id, lease, version_id_exact).await {
|
||||
match delete_object_from_remote_tier_raw_with_lease(obj_name, rv_id, lease).await {
|
||||
Ok(()) => Ok(RemoteTierDeleteOutcome::Deleted),
|
||||
Err(err) if is_remote_tier_not_found_error(&err) => Ok(RemoteTierDeleteOutcome::AlreadyRemoved),
|
||||
Err(err) => {
|
||||
@@ -483,7 +450,6 @@ pub fn transitioned_force_delete_journal_entry(transitioned: &TransitionedObject
|
||||
version_id: transitioned.version_id.clone(),
|
||||
tier_name: transitioned.tier.clone(),
|
||||
backend_identity: None,
|
||||
version_id_exact: false,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -492,14 +458,35 @@ mod test {
|
||||
use crate::client::signer_error::invalid_utf8_header_error;
|
||||
|
||||
use super::{
|
||||
ERR_REMOTE_DELETE_BREAKER_OPEN, ERR_REMOTE_DELETE_LIMITER_CLOSED, RemoteDeleteBreaker, RemoteTierDeleteOutcome,
|
||||
delete_object_from_remote_tier_idempotent, delete_object_from_remote_tier_idempotent_with_manager_and_identity,
|
||||
is_remote_tier_not_found_error, is_signer_header_error, set_remote_tier_delete_test_hook,
|
||||
should_record_remote_delete_failure,
|
||||
ERR_REMOTE_DELETE_BREAKER_OPEN, ERR_REMOTE_DELETE_LIMITER_CLOSED, REMOTE_TIER_DELETE_TEST_HOOK, RemoteDeleteBreaker,
|
||||
RemoteTierDeleteOutcome, delete_object_from_remote_tier_idempotent,
|
||||
delete_object_from_remote_tier_idempotent_with_manager_and_identity, is_remote_tier_not_found_error,
|
||||
is_signer_header_error, should_record_remote_delete_failure,
|
||||
};
|
||||
use std::io::{Error, ErrorKind};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
struct RemoteTierDeleteHookGuard;
|
||||
|
||||
impl Drop for RemoteTierDeleteHookGuard {
|
||||
fn drop(&mut self) {
|
||||
let mut hook = REMOTE_TIER_DELETE_TEST_HOOK
|
||||
.lock()
|
||||
.expect("remote tier delete test hook lock should not poison");
|
||||
*hook = None;
|
||||
}
|
||||
}
|
||||
|
||||
fn set_remote_tier_delete_test_hook(
|
||||
hook_fn: impl Fn(&str, &str, &str) -> std::io::Result<()> + Send + Sync + 'static,
|
||||
) -> RemoteTierDeleteHookGuard {
|
||||
let mut hook = REMOTE_TIER_DELETE_TEST_HOOK
|
||||
.lock()
|
||||
.expect("remote tier delete test hook lock should not poison");
|
||||
*hook = Some(Box::new(hook_fn));
|
||||
RemoteTierDeleteHookGuard
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn signer_header_error_detection_matches_utf8_failures() {
|
||||
let err = Error::new(
|
||||
@@ -587,7 +574,6 @@ mod test {
|
||||
"WARM",
|
||||
mismatched,
|
||||
&manager,
|
||||
false,
|
||||
)
|
||||
.await
|
||||
.expect_err("journal recovery must fail closed when the tier name was rebound");
|
||||
@@ -595,33 +581,6 @@ mod test {
|
||||
assert!(err.to_string().contains("identity no longer matches"));
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
async fn journal_delete_dispatches_an_exact_version_constraint() {
|
||||
let manager = crate::services::tier::tier::TierConfigMgr::new();
|
||||
let backend = crate::services::tier::test_util::register_mock_tier(&manager, "WARM").await;
|
||||
let lease = crate::services::tier::tier::TierConfigMgr::acquire_operation_lease(&manager, "WARM")
|
||||
.await
|
||||
.expect("test tier lease should be available");
|
||||
let identity = lease.backend_identity();
|
||||
drop(lease);
|
||||
|
||||
let outcome = delete_object_from_remote_tier_idempotent_with_manager_and_identity(
|
||||
"remote/object",
|
||||
"exact-version",
|
||||
"WARM",
|
||||
identity,
|
||||
&manager,
|
||||
true,
|
||||
)
|
||||
.await
|
||||
.expect("an exact journal delete should reach the backend");
|
||||
|
||||
assert_eq!(outcome, RemoteTierDeleteOutcome::Deleted);
|
||||
assert_eq!(backend.exact_remove_count(), 1);
|
||||
assert_eq!(backend.remove_count().await, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn breaker_opens_at_threshold_and_recovers_after_window() {
|
||||
let mut breaker = RemoteDeleteBreaker::new(3, Duration::from_secs(30));
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -412,10 +412,6 @@ impl BucketMetadataSys {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn object_store(&self) -> Arc<ECStore> {
|
||||
self.api.clone()
|
||||
}
|
||||
|
||||
pub async fn init(&mut self, buckets: Vec<String>) {
|
||||
let _ = self.init_internal(buckets).await;
|
||||
}
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::disk::disk_store::{get_drive_walkdir_peek_timeout, get_drive_walkdir_stall_timeout};
|
||||
use crate::disk::disk_store::get_drive_walkdir_stall_timeout;
|
||||
use crate::disk::error::DiskError;
|
||||
use crate::disk::{self, DiskAPI, DiskStore, WalkDirOptions};
|
||||
use metrics::counter;
|
||||
@@ -175,10 +175,6 @@ pub(crate) enum TestReaderBehavior {
|
||||
ProducerError(DiskError),
|
||||
PrimaryErrorThenFallback(DiskError),
|
||||
PartialThenTimeout(Vec<MetaCacheEntry>),
|
||||
DelayedEntries {
|
||||
delay: Duration,
|
||||
entries: Vec<MetaCacheEntry>,
|
||||
},
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -343,13 +339,6 @@ async fn list_path_raw_inner(
|
||||
drop(out);
|
||||
Some(err)
|
||||
}
|
||||
TestReaderBehavior::DelayedEntries { delay, entries } => {
|
||||
tokio::time::sleep(delay).await;
|
||||
let mut out = rustfs_filemeta::MetacacheWriter::new(&mut wr);
|
||||
out.write(&entries).await.expect("delayed test entries should be written");
|
||||
out.close().await.expect("delayed test entries should close");
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
} else {
|
||||
None
|
||||
@@ -484,17 +473,6 @@ async fn list_path_raw_inner(
|
||||
record_producer_error(&producer_errs_clone, disk_idx, &err);
|
||||
return Err(err);
|
||||
}
|
||||
TestReaderBehavior::DelayedEntries { delay, entries } => {
|
||||
tokio::time::sleep(delay).await;
|
||||
let mut out = rustfs_filemeta::MetacacheWriter::new(&mut wr);
|
||||
out.write(&entries)
|
||||
.await
|
||||
.expect("delayed test fallback entries should be written");
|
||||
out.close().await.expect("delayed test fallback entries should close");
|
||||
need_fallback = false;
|
||||
last_err = None;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -631,23 +609,47 @@ async fn list_path_raw_inner(
|
||||
// Consumer-side peek timeout, and a caveat worth understanding
|
||||
// (rustfs/backlog#1217).
|
||||
//
|
||||
// This budget must not be stricter than the producer-side walk stall
|
||||
// budget. The two measure different things:
|
||||
// This budget is the SAME SOURCE and SAME VALUE as the producer-side
|
||||
// walk stall budget: both default to `walkdir_stall_timeout` and fall
|
||||
// back to `get_drive_walkdir_stall_timeout()` (5s). But the two measure
|
||||
// different things:
|
||||
// * producer stall: bounds a single drive READ inside the walk (see
|
||||
// `with_walk_stall_deadline` in `disk/local.rs`);
|
||||
// * this consumer peek: bounds the interval between two ADJACENT
|
||||
// entries arriving from a drive's reader (`peek_with_timeout`).
|
||||
//
|
||||
// A drive can spend one stall budget walking a dense non-listable region
|
||||
// before it can publish the next visible entry. Use an independent,
|
||||
// profile-aware consumer budget so the merge does not detach that reader
|
||||
// before the producer itself would have failed.
|
||||
let producer_stall_timeout = opts.walkdir_stall_timeout.unwrap_or_else(get_drive_walkdir_stall_timeout);
|
||||
let configured_peek_timeout = get_drive_walkdir_peek_timeout().max(producer_stall_timeout);
|
||||
#[cfg(not(test))]
|
||||
let peek_timeout = configured_peek_timeout;
|
||||
#[cfg(test)]
|
||||
let peek_timeout = opts.peek_timeout.unwrap_or(configured_peek_timeout);
|
||||
// Because they are coupled to the same value, the consumer cannot wait
|
||||
// meaningfully longer for the next entry than the producer is allowed to
|
||||
// spend producing one. When a drive walks a region dense with
|
||||
// non-listable internal items (many entries the producer filters out
|
||||
// before emitting the next visible one), a HEALTHY drive can take longer
|
||||
// than one budget to hand the consumer its next entry. The consumer then
|
||||
// classifies it as `PeekOutcome::TimedOut` and DETACHES that reader (it
|
||||
// is replaced by a drained duplex below), dropping a good drive from the
|
||||
// merge. That caps the "large prefix always succeeds" guarantee: a wide
|
||||
// enough non-listable stretch can knock healthy drives out of quorum.
|
||||
//
|
||||
// This is left documented, not decoupled. Giving the consumer peek an
|
||||
// independent, strictly-larger budget would reduce these false detaches,
|
||||
// but it also delays detaching a genuinely dead drive by the same amount
|
||||
// and shifts listing tail-latency semantics; that trade-off wants soak
|
||||
// data before it changes the default, so it is deferred to a follow-up.
|
||||
// The constraint for any such change: the consumer peek must be >= the
|
||||
// producer stall (never stricter), so it can never declare a drive
|
||||
// stalled before the producer itself would have failed.
|
||||
let peek_timeout = opts
|
||||
.walkdir_stall_timeout
|
||||
.or({
|
||||
#[cfg(test)]
|
||||
{
|
||||
opts.peek_timeout
|
||||
}
|
||||
#[cfg(not(test))]
|
||||
{
|
||||
None
|
||||
}
|
||||
})
|
||||
.unwrap_or_else(get_drive_walkdir_stall_timeout);
|
||||
let mut errs: Vec<Option<DiskError>> = Vec::with_capacity(readers.len());
|
||||
for _ in 0..readers.len() {
|
||||
errs.push(None);
|
||||
@@ -1244,57 +1246,6 @@ mod tests {
|
||||
assert_eq!(err, DiskError::Timeout);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn list_path_raw_waits_past_producer_stall_for_slow_progressing_reader() {
|
||||
let entry = MetaCacheEntry {
|
||||
name: "bucket/visible-object".to_string(),
|
||||
metadata: vec![1, 2, 3],
|
||||
cached: None,
|
||||
reusable: false,
|
||||
};
|
||||
let seen = Arc::new(Mutex::new(Vec::new()));
|
||||
let seen_clone = seen.clone();
|
||||
|
||||
timeout(
|
||||
Duration::from_secs(1),
|
||||
list_path_raw(
|
||||
CancellationToken::new(),
|
||||
ListPathRawOptions {
|
||||
disks: vec![None, None],
|
||||
min_disks: 2,
|
||||
walkdir_stall_timeout: Some(Duration::from_millis(20)),
|
||||
test_reader_behaviors: vec![
|
||||
TestReaderBehavior::DelayedEntries {
|
||||
delay: Duration::from_millis(60),
|
||||
entries: vec![entry.clone()],
|
||||
},
|
||||
TestReaderBehavior::Entries(vec![entry]),
|
||||
],
|
||||
partial: Some(Box::new(move |entries: MetaCacheEntries, _: &[Option<DiskError>]| {
|
||||
let seen = seen_clone.clone();
|
||||
Box::pin(async move {
|
||||
let mut names = entries.0.iter().flatten().map(|entry| entry.name.as_str());
|
||||
if let (Some(first), Some(second), None) = (names.next(), names.next(), names.next())
|
||||
&& first == second
|
||||
{
|
||||
seen.lock().expect("seen mutex poisoned").push(first.to_owned());
|
||||
}
|
||||
})
|
||||
})),
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("slow-progressing reader should complete inside the consumer peek budget")
|
||||
.expect("slow-progressing reader should not be detached by the producer stall budget");
|
||||
|
||||
assert_eq!(
|
||||
seen.lock().expect("seen mutex poisoned").as_slice(),
|
||||
&["bucket/visible-object".to_string()]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn list_path_raw_prefers_timeout_for_mixed_errors_beyond_quorum_budget() {
|
||||
let err = list_path_raw(
|
||||
|
||||
@@ -103,7 +103,7 @@ pub fn http_resp_to_error_response(
|
||||
object_name: &str,
|
||||
) -> ErrorResponse {
|
||||
let err_body = String::from_utf8_lossy(&b).to_string();
|
||||
if h.is_empty() || !(resp_status.is_client_error() || resp_status.is_server_error()) {
|
||||
if h.is_empty() || resp_status.is_client_error() || resp_status.is_server_error() {
|
||||
return ErrorResponse {
|
||||
status_code: resp_status,
|
||||
code: S3ErrorCode::ResponseInterrupted,
|
||||
@@ -329,21 +329,4 @@ mod tests {
|
||||
assert_eq!(value["RequestId"], Value::String("req-xml-123".to_string()));
|
||||
assert!(value.get("request_id").is_none(), "external error contract must not expose request_id");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_s3_error_code_from_client_error_response() {
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert("x-amz-request-id", "request-id".parse().expect("request ID header should parse"));
|
||||
|
||||
let response = http_resp_to_error_response(
|
||||
StatusCode::NOT_FOUND,
|
||||
&headers,
|
||||
b"<Error><Code>NoSuchVersion</Code><Message>remote detail</Message></Error>".to_vec(),
|
||||
"bucket",
|
||||
"object",
|
||||
);
|
||||
|
||||
assert_eq!(response.code, S3ErrorCode::NoSuchVersion);
|
||||
assert_eq!(response.status_code, StatusCode::NOT_FOUND);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -18,7 +18,7 @@
|
||||
#![allow(unused_must_use)]
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use http::{HeaderMap, HeaderValue, StatusCode};
|
||||
use http::{HeaderMap, HeaderValue};
|
||||
use http_body_util::BodyExt;
|
||||
use hyper::body::Body;
|
||||
use hyper::body::Bytes;
|
||||
@@ -31,62 +31,13 @@ use uuid::Uuid;
|
||||
use crate::client::{
|
||||
api_error_response::{ErrorResponse, err_invalid_argument, http_resp_to_error_response},
|
||||
api_get_options::GetObjectOptions,
|
||||
transition_api::{
|
||||
ObjectInfo, ReadCloser, ReaderImpl, RequestMetadata, TransitionClient, collect_response_body, to_object_info_for_provider,
|
||||
},
|
||||
transition_api::{ObjectInfo, ReadCloser, ReaderImpl, RequestMetadata, TransitionClient, to_object_info_for_provider},
|
||||
};
|
||||
use s3s::{
|
||||
dto::{BucketVersioningStatus, MFADelete, VersioningConfiguration},
|
||||
dto::VersioningConfiguration,
|
||||
header::{X_AMZ_DELETE_MARKER, X_AMZ_VERSION_ID},
|
||||
};
|
||||
|
||||
const S3_XML_NAMESPACE: &str = "http://s3.amazonaws.com/doc/2006-03-01/";
|
||||
|
||||
#[derive(serde::Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct StrictVersioningConfiguration {
|
||||
#[serde(rename = "@xmlns")]
|
||||
namespace: Option<String>,
|
||||
#[serde(rename = "MfaDelete")]
|
||||
mfa_delete: Option<String>,
|
||||
#[serde(rename = "Status")]
|
||||
status: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(serde::Deserialize)]
|
||||
enum StrictVersioningResponse {
|
||||
VersioningConfiguration(StrictVersioningConfiguration),
|
||||
}
|
||||
|
||||
fn parse_bucket_versioning_response(
|
||||
status: StatusCode,
|
||||
headers: &HeaderMap,
|
||||
body: Vec<u8>,
|
||||
bucket_name: &str,
|
||||
) -> Result<VersioningConfiguration, std::io::Error> {
|
||||
if status != StatusCode::OK {
|
||||
return Err(std::io::Error::other(http_resp_to_error_response(status, headers, body, bucket_name, "")));
|
||||
}
|
||||
|
||||
let StrictVersioningResponse::VersioningConfiguration(parsed) =
|
||||
quick_xml::de::from_reader(body.as_slice()).map_err(|err| std::io::Error::new(std::io::ErrorKind::InvalidData, err))?;
|
||||
if parsed
|
||||
.namespace
|
||||
.as_deref()
|
||||
.is_some_and(|namespace| namespace != S3_XML_NAMESPACE)
|
||||
{
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidData,
|
||||
"unexpected GetBucketVersioning XML namespace",
|
||||
));
|
||||
}
|
||||
Ok(VersioningConfiguration {
|
||||
mfa_delete: parsed.mfa_delete.map(MFADelete::from),
|
||||
status: parsed.status.map(BucketVersioningStatus::from),
|
||||
..Default::default()
|
||||
})
|
||||
}
|
||||
|
||||
impl TransitionClient {
|
||||
pub async fn bucket_exists(&self, bucket_name: &str) -> Result<bool, std::io::Error> {
|
||||
let resp = self
|
||||
@@ -172,8 +123,19 @@ impl TransitionClient {
|
||||
let resp_status = resp.status();
|
||||
let h = resp.headers().clone();
|
||||
|
||||
let body_vec = collect_response_body(resp.into_body(), rustfs_config::MAX_S3_CLIENT_RESPONSE_SIZE).await?;
|
||||
parse_bucket_versioning_response(resp_status, &h, body_vec, bucket_name)
|
||||
let mut body_vec = Vec::new();
|
||||
let mut body = resp.into_body();
|
||||
while let Some(frame) = body.frame().await {
|
||||
let frame = frame.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))?;
|
||||
if let Some(data) = frame.data_ref() {
|
||||
body_vec.extend_from_slice(data);
|
||||
}
|
||||
}
|
||||
let resperr = http_resp_to_error_response(resp_status, &h, body_vec, bucket_name, "");
|
||||
|
||||
warn!("get bucket versioning, resperr: {:?}", resperr);
|
||||
|
||||
Ok(VersioningConfiguration::default())
|
||||
}
|
||||
|
||||
Err(err) => Err(std::io::Error::other(err)),
|
||||
@@ -273,72 +235,3 @@ impl TransitionClient {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::parse_bucket_versioning_response;
|
||||
use http::{HeaderMap, StatusCode};
|
||||
use s3s::dto::BucketVersioningStatus;
|
||||
|
||||
#[test]
|
||||
fn parses_bucket_versioning_statuses_mfa_delete_and_unversioned_state() {
|
||||
for (xml, expected) in [
|
||||
(
|
||||
br#"<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Status>Enabled</Status></VersioningConfiguration>"#
|
||||
.as_slice(),
|
||||
Some(BucketVersioningStatus::ENABLED),
|
||||
),
|
||||
(
|
||||
br#"<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Status>Suspended</Status></VersioningConfiguration>"#
|
||||
.as_slice(),
|
||||
Some(BucketVersioningStatus::SUSPENDED),
|
||||
),
|
||||
(
|
||||
br#"<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"/>"#.as_slice(),
|
||||
None,
|
||||
),
|
||||
] {
|
||||
let config = parse_bucket_versioning_response(StatusCode::OK, &HeaderMap::new(), xml.to_vec(), "tier-bucket")
|
||||
.expect("valid GetBucketVersioning response should parse");
|
||||
assert_eq!(config.status.as_ref().map(|status| status.as_str()), expected);
|
||||
}
|
||||
|
||||
let config = parse_bucket_versioning_response(
|
||||
StatusCode::OK,
|
||||
&HeaderMap::new(),
|
||||
br#"<VersioningConfiguration><MfaDelete>Enabled</MfaDelete></VersioningConfiguration>"#.to_vec(),
|
||||
"tier-bucket",
|
||||
)
|
||||
.expect("valid MfaDelete should parse");
|
||||
assert_eq!(config.mfa_delete.as_ref().map(|status| status.as_str()), Some("Enabled"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_non_ok_and_malformed_bucket_versioning_responses() {
|
||||
let valid = br#"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>"#.to_vec();
|
||||
for status in [StatusCode::NO_CONTENT, StatusCode::PARTIAL_CONTENT] {
|
||||
let status_err = parse_bucket_versioning_response(status, &HeaderMap::new(), valid.clone(), "tier-bucket")
|
||||
.expect_err("GetBucketVersioning must return exactly HTTP 200");
|
||||
assert_eq!(status_err.kind(), std::io::ErrorKind::Other);
|
||||
}
|
||||
|
||||
let parse_err = parse_bucket_versioning_response(
|
||||
StatusCode::OK,
|
||||
&HeaderMap::new(),
|
||||
b"<VersioningConfiguration>".to_vec(),
|
||||
"tier-bucket",
|
||||
)
|
||||
.expect_err("malformed GetBucketVersioning XML must fail closed");
|
||||
assert_eq!(parse_err.kind(), std::io::ErrorKind::InvalidData);
|
||||
|
||||
for xml in [
|
||||
b"<VersioningConfiguration><Statuz>Enabled</Statuz></VersioningConfiguration>".as_slice(),
|
||||
b"<WrongRoot><Status>Enabled</Status></WrongRoot>".as_slice(),
|
||||
b"<VersioningConfiguration xmlns=\"https://example.invalid\"/>".as_slice(),
|
||||
] {
|
||||
let strict_err = parse_bucket_versioning_response(StatusCode::OK, &HeaderMap::new(), xml.to_vec(), "tier-bucket")
|
||||
.expect_err("unknown GetBucketVersioning XML must fail closed");
|
||||
assert_eq!(strict_err.kind(), std::io::ErrorKind::InvalidData);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -60,7 +60,6 @@ impl ProviderVersionCapabilities {
|
||||
|| tier_type.eq_ignore_ascii_case("rustfs")
|
||||
|| tier_type.eq_ignore_ascii_case("minio")
|
||||
|| tier_type.eq_ignore_ascii_case("r2")
|
||||
|| tier_type.eq_ignore_ascii_case("wasabi")
|
||||
{
|
||||
Self {
|
||||
raw_version_header: Some(X_AMZ_VERSION_ID),
|
||||
@@ -157,8 +156,6 @@ mod tests {
|
||||
("MinIO", "x-amz-version-id"),
|
||||
("r2", "x-amz-version-id"),
|
||||
("R2", "x-amz-version-id"),
|
||||
("wasabi", "x-amz-version-id"),
|
||||
("Wasabi", "x-amz-version-id"),
|
||||
("aliyun", "x-oss-version-id"),
|
||||
("Aliyun", "x-oss-version-id"),
|
||||
("tencent", "x-cos-version-id"),
|
||||
|
||||
@@ -42,7 +42,7 @@ use http::{
|
||||
request::{Builder, Request},
|
||||
};
|
||||
use http_body::Body;
|
||||
use http_body_util::{BodyExt, LengthLimitError, Limited};
|
||||
use http_body_util::BodyExt;
|
||||
use hyper::body::Bytes;
|
||||
use hyper::body::Incoming;
|
||||
use hyper_rustls::{ConfigBuilderExt, HttpsConnector};
|
||||
@@ -55,7 +55,9 @@ use rustfs_rio::HashReader;
|
||||
use rustfs_utils::HashAlgorithm;
|
||||
use rustfs_utils::{
|
||||
net::get_endpoint_url,
|
||||
retry::{DEFAULT_RETRY_CAP, DEFAULT_RETRY_UNIT, MAX_JITTER, MAX_RETRY, RetryTimer},
|
||||
retry::{
|
||||
DEFAULT_RETRY_CAP, DEFAULT_RETRY_UNIT, MAX_JITTER, MAX_RETRY, RetryTimer, is_http_status_retryable, is_s3code_retryable,
|
||||
},
|
||||
};
|
||||
use rustls_pki_types::PrivateKeyDer;
|
||||
use rustls_pki_types::pem::PemObject;
|
||||
@@ -80,25 +82,9 @@ use url::{Url, form_urlencoded};
|
||||
use uuid::Uuid;
|
||||
|
||||
const C_USER_AGENT: &str = "RustFS (linux; x86)";
|
||||
pub(crate) const MAX_S3_ERROR_RESPONSE_SIZE: usize = 64 * 1024;
|
||||
|
||||
const SUCCESS_STATUS: [StatusCode; 3] = [StatusCode::OK, StatusCode::NO_CONTENT, StatusCode::PARTIAL_CONTENT];
|
||||
|
||||
pub(crate) async fn collect_response_body<B>(body: B, limit: usize) -> Result<Vec<u8>, std::io::Error>
|
||||
where
|
||||
B: Body<Data = Bytes>,
|
||||
B::Error: Into<Box<dyn std::error::Error + Send + Sync>>,
|
||||
{
|
||||
let body = Limited::new(body, limit).collect().await.map_err(|err| {
|
||||
if err.is::<LengthLimitError>() {
|
||||
std::io::Error::new(std::io::ErrorKind::InvalidData, "remote tier response body exceeds limit")
|
||||
} else {
|
||||
std::io::Error::other(err)
|
||||
}
|
||||
})?;
|
||||
Ok(body.to_bytes().to_vec())
|
||||
}
|
||||
|
||||
const C_UNKNOWN: i32 = -1;
|
||||
const C_OFFLINE: i32 = 0;
|
||||
const C_ONLINE: i32 = 1;
|
||||
@@ -411,17 +397,20 @@ impl TransitionClient {
|
||||
pub async fn doit(&self, req: Request<s3s::Body>) -> Result<Response<Incoming>, std::io::Error> {
|
||||
let req_method;
|
||||
let req_uri;
|
||||
let req_headers;
|
||||
let resp;
|
||||
let http_client = self.http_client.clone();
|
||||
{
|
||||
req_method = req.method().clone();
|
||||
req_uri = req.uri().clone();
|
||||
req_headers = req.headers().clone();
|
||||
|
||||
debug!("endpoint_url: {}", self.endpoint_url.as_str().to_string());
|
||||
resp = http_client.request(req);
|
||||
}
|
||||
let resp = resp.await;
|
||||
debug!("http_client url: {} {}", req_method, req_uri);
|
||||
debug!("http_client headers: {:?}", req_headers);
|
||||
if let Err(err) = resp {
|
||||
error!("http_client call error: {:?}", err);
|
||||
return Err(std::io::Error::other(err));
|
||||
@@ -431,23 +420,28 @@ impl TransitionClient {
|
||||
Ok(r) => r,
|
||||
Err(_) => return Err(std::io::Error::other("Unexpected error in response")),
|
||||
};
|
||||
debug!(status = %resp.status(), "remote tier response received");
|
||||
debug!("http_resp: {:?}", resp);
|
||||
|
||||
//let b = resp.body_mut().store_all_unlimited().await.unwrap().to_vec();
|
||||
//debug!("http_resp_body: {}", String::from_utf8(b).unwrap());
|
||||
|
||||
//if self.is_trace_enabled && !(self.trace_errors_only && resp.status() == StatusCode::OK) {
|
||||
if !resp.status().is_success() {
|
||||
let status = resp.status();
|
||||
let request_id = resp
|
||||
.headers()
|
||||
.get("x-amz-request-id")
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.unwrap_or_default()
|
||||
.to_string();
|
||||
warn!(status = %status, request_id, "remote tier request rejected");
|
||||
//self.dump_http(&cloned_req, &resp)?;
|
||||
let mut body_vec = Vec::new();
|
||||
let mut body = resp.into_body();
|
||||
while let Some(frame) = body.frame().await {
|
||||
let frame = frame.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))?;
|
||||
if let Some(data) = frame.data_ref() {
|
||||
body_vec.extend_from_slice(data);
|
||||
}
|
||||
}
|
||||
let body_str = String::from_utf8_lossy(&body_vec);
|
||||
warn!("err_body: {}", body_str);
|
||||
Err(std::io::Error::other(format!("http_client call error: {}", body_str)))
|
||||
} else {
|
||||
Ok(resp)
|
||||
}
|
||||
Ok(resp)
|
||||
}
|
||||
|
||||
pub async fn execute_method(
|
||||
@@ -489,22 +483,17 @@ impl TransitionClient {
|
||||
let resp_status = resp.status();
|
||||
let h = resp.headers().clone();
|
||||
|
||||
let body_vec = collect_response_body(resp.into_body(), MAX_S3_ERROR_RESPONSE_SIZE).await?;
|
||||
let parsed_error =
|
||||
http_resp_to_error_response(resp_status, &h, body_vec, &metadata.bucket_name, &metadata.object_name);
|
||||
let routing_region = parsed_error.region;
|
||||
let code = match parsed_error.code {
|
||||
S3ErrorCode::Custom(_) => S3ErrorCode::ResponseInterrupted,
|
||||
code => code,
|
||||
};
|
||||
let err_response = ErrorResponse {
|
||||
status_code: resp_status,
|
||||
message: format!("remote tier request failed with status {resp_status}: {}", code.as_str()),
|
||||
code,
|
||||
bucket_name: metadata.bucket_name.clone(),
|
||||
key: metadata.object_name.clone(),
|
||||
..Default::default()
|
||||
};
|
||||
let mut body_vec = Vec::new();
|
||||
let mut body = resp.into_body();
|
||||
while let Some(frame) = body.frame().await {
|
||||
let frame = frame.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))?;
|
||||
if let Some(data) = frame.data_ref() {
|
||||
body_vec.extend_from_slice(data);
|
||||
}
|
||||
}
|
||||
let mut err_response =
|
||||
http_resp_to_error_response(resp_status, &h, body_vec.clone(), &metadata.bucket_name, &metadata.object_name);
|
||||
err_response.message = format!("remote tier error: {}", err_response.message);
|
||||
|
||||
if self.region == "" {
|
||||
return match err_response.code {
|
||||
@@ -513,20 +502,20 @@ impl TransitionClient {
|
||||
Err(std::io::Error::other(err_response))
|
||||
}
|
||||
S3ErrorCode::AccessDenied => {
|
||||
if routing_region.is_empty() {
|
||||
if err_response.region == "" {
|
||||
return Err(std::io::Error::other(err_response));
|
||||
}
|
||||
if metadata.bucket_name != "" {
|
||||
if let Ok(mut bucket_loc_cache) = self.bucket_loc_cache.lock() {
|
||||
if let Some(location) = bucket_loc_cache.get(&metadata.bucket_name) {
|
||||
if location != routing_region {
|
||||
bucket_loc_cache.set(&metadata.bucket_name, &routing_region);
|
||||
if location != err_response.region {
|
||||
bucket_loc_cache.set(&metadata.bucket_name, &err_response.region);
|
||||
//continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if routing_region != metadata.bucket_location {
|
||||
metadata.bucket_location = routing_region;
|
||||
} else if err_response.region != metadata.bucket_location {
|
||||
metadata.bucket_location = err_response.region.clone();
|
||||
//continue;
|
||||
}
|
||||
Err(std::io::Error::other(err_response))
|
||||
@@ -537,10 +526,18 @@ impl TransitionClient {
|
||||
};
|
||||
}
|
||||
|
||||
return Err(std::io::Error::other(err_response));
|
||||
if is_s3code_retryable(err_response.code.as_str()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if is_http_status_retryable(&resp_status) {
|
||||
continue;
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
|
||||
Err(std::io::Error::other("remote tier request did not produce a response"))
|
||||
Err(std::io::Error::other("resp err"))
|
||||
}
|
||||
|
||||
async fn new_request(
|
||||
@@ -1411,43 +1408,13 @@ pub struct CreateBucketConfiguration {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{
|
||||
MAX_S3_CLIENT_RESPONSE_SIZE, MAX_S3_ERROR_RESPONSE_SIZE, SignatureType, build_tls_config, collect_response_body,
|
||||
signer_error_to_io_error, to_object_info_for_provider, validate_header_values, with_rustls_init_guard,
|
||||
SignatureType, build_tls_config, signer_error_to_io_error, to_object_info_for_provider, validate_header_values,
|
||||
with_rustls_init_guard,
|
||||
};
|
||||
use crate::client::provider_versions::ProviderVersionCapabilities;
|
||||
use http::{HeaderMap, HeaderValue};
|
||||
use http_body_util::Full;
|
||||
use hyper::body::Bytes;
|
||||
use uuid::Uuid;
|
||||
|
||||
#[tokio::test]
|
||||
async fn response_body_limit_rejects_the_first_byte_past_the_boundary() {
|
||||
let exact = collect_response_body(
|
||||
Full::new(Bytes::from(vec![0_u8; MAX_S3_CLIENT_RESPONSE_SIZE])),
|
||||
MAX_S3_CLIENT_RESPONSE_SIZE,
|
||||
)
|
||||
.await
|
||||
.expect("a response exactly at the limit should be accepted");
|
||||
assert_eq!(exact.len(), MAX_S3_CLIENT_RESPONSE_SIZE);
|
||||
|
||||
let err = collect_response_body(
|
||||
Full::new(Bytes::from(vec![0_u8; MAX_S3_CLIENT_RESPONSE_SIZE + 1])),
|
||||
MAX_S3_CLIENT_RESPONSE_SIZE,
|
||||
)
|
||||
.await
|
||||
.expect_err("oversized remote response must fail closed");
|
||||
|
||||
assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
|
||||
|
||||
let err = collect_response_body(
|
||||
Full::new(Bytes::from(vec![0_u8; MAX_S3_ERROR_RESPONSE_SIZE + 1])),
|
||||
MAX_S3_ERROR_RESPONSE_SIZE,
|
||||
)
|
||||
.await
|
||||
.expect_err("oversized S3 error response must use the smaller error limit");
|
||||
assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rustls_guard_converts_panics_to_io_errors() {
|
||||
let err = with_rustls_init_guard(|| -> Result<(), std::io::Error> { panic!("missing provider") })
|
||||
|
||||
@@ -32,9 +32,6 @@ const FAILURE_DOMAIN_NOT_REPORTED: &str = "failure domain labels not reported by
|
||||
const NUMA_NOT_WIRED: &str = "NUMA topology not wired into runtime";
|
||||
const PROFILING_NOT_WIRED: &str = "profiling capability not wired into ECStore";
|
||||
const PEER_HEALTH_NOT_REPORTED: &str = "peer health not reported by endpoints";
|
||||
const PEER_HEALTH_LOCAL_NODE: &str = "local node does not require peer health probing";
|
||||
const PEER_HEALTH_REACHABLE: &str = "peer marked reachable by internode health tracker";
|
||||
const PEER_HEALTH_UNREACHABLE: &str = "peer marked unreachable by internode health tracker";
|
||||
const CONTROL_RPC_SEPARATED: &str = "control RPC remains on the gRPC control plane";
|
||||
const DATA_STREAM_RPC_SEPARATED: &str = "remote disk data streams remain on the internode data transport";
|
||||
|
||||
@@ -344,24 +341,12 @@ pub fn peer_health_snapshot_from_membership(membership: &ClusterMembershipSnapsh
|
||||
.map(|node| ClusterPeerHealth {
|
||||
node_id: node.node_id.clone(),
|
||||
is_local: node.is_local,
|
||||
status: peer_health_status_for_node(node),
|
||||
status: CapabilityStatus::unknown().with_reason(PEER_HEALTH_NOT_REPORTED),
|
||||
})
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
|
||||
fn peer_health_status_for_node(node: &ClusterNodeMembership) -> CapabilityStatus {
|
||||
if node.is_local {
|
||||
return CapabilityStatus::supported().with_reason(PEER_HEALTH_LOCAL_NODE);
|
||||
}
|
||||
|
||||
match rustfs_io_metrics::internode_metrics::cluster_peer_observed_online_status(&node.grid_host) {
|
||||
Some(true) => CapabilityStatus::supported().with_reason(PEER_HEALTH_REACHABLE),
|
||||
Some(false) => CapabilityStatus::unknown().with_reason(PEER_HEALTH_UNREACHABLE),
|
||||
None => CapabilityStatus::disabled().with_reason(PEER_HEALTH_NOT_REPORTED),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn rpc_boundary_snapshot() -> ClusterRpcBoundarySnapshot {
|
||||
ClusterRpcBoundarySnapshot {
|
||||
control_channels: ["metadata", "lock", "health", "administrative"]
|
||||
@@ -513,7 +498,6 @@ mod tests {
|
||||
use std::collections::BTreeSet;
|
||||
|
||||
use crate::layout::endpoints::{Endpoints, PoolEndpoints};
|
||||
use crate::storage_api_contracts::topology::CapabilityState;
|
||||
|
||||
#[test]
|
||||
fn topology_snapshot_maps_endpoint_sets_without_local_paths() {
|
||||
@@ -604,55 +588,16 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn peer_health_snapshot_reports_observed_peer_status() {
|
||||
let online_host = "http://cluster-control-plane-peer-online-test:9000";
|
||||
let offline_host = "http://cluster-control-plane-peer-offline-test:9000";
|
||||
rustfs_io_metrics::internode_metrics::record_peer_reachable(online_host);
|
||||
rustfs_io_metrics::internode_metrics::record_peer_unreachable(offline_host, 1);
|
||||
let membership = ClusterMembershipSnapshot {
|
||||
nodes: vec![
|
||||
ClusterNodeMembership {
|
||||
node_id: LOCAL_NODE_ID.to_string(),
|
||||
grid_host: String::new(),
|
||||
is_local: true,
|
||||
pools: vec![0],
|
||||
},
|
||||
ClusterNodeMembership {
|
||||
node_id: "cluster-control-plane-peer-online-test:9000".to_string(),
|
||||
grid_host: online_host.to_string(),
|
||||
is_local: false,
|
||||
pools: vec![0],
|
||||
},
|
||||
ClusterNodeMembership {
|
||||
node_id: "cluster-control-plane-peer-offline-test:9000".to_string(),
|
||||
grid_host: offline_host.to_string(),
|
||||
is_local: false,
|
||||
pools: vec![0],
|
||||
},
|
||||
ClusterNodeMembership {
|
||||
node_id: "cluster-control-plane-peer-unknown-test:9000".to_string(),
|
||||
grid_host: "http://cluster-control-plane-peer-unknown-test:9000".to_string(),
|
||||
is_local: false,
|
||||
pools: vec![0],
|
||||
},
|
||||
],
|
||||
drives: Vec::new(),
|
||||
};
|
||||
fn peer_health_snapshot_reports_static_unknown_status() {
|
||||
let membership = membership_snapshot_from_endpoint_pools(&sample_url_endpoint_pools());
|
||||
let snapshot = peer_health_snapshot_from_membership(&membership);
|
||||
|
||||
assert_eq!(snapshot.peers.len(), 4);
|
||||
assert_eq!(snapshot.peers[0].node_id, LOCAL_NODE_ID);
|
||||
assert_eq!(snapshot.peers.len(), 2);
|
||||
assert_eq!(snapshot.peers[0].node_id, "node1.example:9000");
|
||||
assert!(snapshot.peers[0].is_local);
|
||||
assert!(snapshot.peers[0].status.state.is_supported());
|
||||
assert_eq!(snapshot.peers[0].status.reason.as_deref(), Some(PEER_HEALTH_LOCAL_NODE));
|
||||
assert_eq!(snapshot.peers[1].node_id, "cluster-control-plane-peer-online-test:9000");
|
||||
assert_eq!(snapshot.peers[0].status.reason.as_deref(), Some(PEER_HEALTH_NOT_REPORTED));
|
||||
assert_eq!(snapshot.peers[1].node_id, "node2.example:9000");
|
||||
assert!(!snapshot.peers[1].is_local);
|
||||
assert!(snapshot.peers[1].status.state.is_supported());
|
||||
assert_eq!(snapshot.peers[1].status.reason.as_deref(), Some(PEER_HEALTH_REACHABLE));
|
||||
assert_eq!(snapshot.peers[2].status.reason.as_deref(), Some(PEER_HEALTH_UNREACHABLE));
|
||||
assert!(!snapshot.peers[2].status.state.is_supported());
|
||||
assert_eq!(snapshot.peers[3].status.reason.as_deref(), Some(PEER_HEALTH_NOT_REPORTED));
|
||||
assert_eq!(snapshot.peers[3].status.state, CapabilityState::Disabled);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -19,10 +19,7 @@ use crate::runtime::sources as runtime_sources;
|
||||
use http::Uri;
|
||||
use rustfs_protos::{
|
||||
ChannelClass, create_new_channel, get_channel_for_class,
|
||||
proto_gen::node_service::{
|
||||
heal_control_service_client::HealControlServiceClient, node_service_client::NodeServiceClient,
|
||||
tier_mutation_control_service_client::TierMutationControlServiceClient,
|
||||
},
|
||||
proto_gen::node_service::{heal_control_service_client::HealControlServiceClient, node_service_client::NodeServiceClient},
|
||||
};
|
||||
use std::{error::Error, io::ErrorKind};
|
||||
use tonic::{service::interceptor::InterceptedService, transport::Channel};
|
||||
@@ -56,21 +53,6 @@ pub async fn heal_control_time_out_client(
|
||||
.max_encoding_message_size(max_message_size))
|
||||
}
|
||||
|
||||
pub async fn tier_mutation_control_time_out_client(
|
||||
addr: &str,
|
||||
interceptor: TonicInterceptor,
|
||||
) -> Result<TierMutationControlServiceClient<InterceptedService<Channel, TonicInterceptor>>, Box<dyn Error>> {
|
||||
let interceptor = interceptor.with_rpc_audience(addr)?;
|
||||
let channel = match runtime_sources::cached_node_channel(addr).await {
|
||||
Some(channel) => channel,
|
||||
None => create_new_channel(addr).await?,
|
||||
};
|
||||
let max_message_size = rustfs_protos::TIER_MUTATION_RPC_MAX_MESSAGE_SIZE;
|
||||
Ok(TierMutationControlServiceClient::with_interceptor(channel, interceptor)
|
||||
.max_decoding_message_size(max_message_size)
|
||||
.max_encoding_message_size(max_message_size))
|
||||
}
|
||||
|
||||
/// Build a `NodeServiceClient` bound to the [`ChannelClass`]-appropriate channel for `addr`.
|
||||
///
|
||||
/// Bulk `bytes`-carrying RPCs (ReadAll/WriteAll/ReadMultiple/BatchReadVersion) pass
|
||||
|
||||
@@ -1144,56 +1144,6 @@ mod tests {
|
||||
assert_eq!(tampered.to_string(), "RPC content SHA-256 mismatch");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier_mutation_rpc_contract_requires_method_bound_v2_body_digest() {
|
||||
ensure_test_rpc_secret();
|
||||
let mutation_id = uuid::uuid!("12345678-1234-5678-9abc-def012345678");
|
||||
let body = rustfs_protos::canonical_tier_mutation_rpc_body(
|
||||
rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION,
|
||||
rustfs_protos::TierMutationRpcPhase::Prepare,
|
||||
mutation_id,
|
||||
b"canonical-tier-mutation-prepare",
|
||||
)
|
||||
.expect("small tier mutation body should encode");
|
||||
let mut request = tonic::Request::new(());
|
||||
set_tonic_canonical_body_digest(&mut request, &body).expect("canonical body digest should be attached");
|
||||
let content_sha256 = request
|
||||
.metadata()
|
||||
.get(RPC_CONTENT_SHA256_HEADER)
|
||||
.and_then(|value| value.to_str().ok());
|
||||
let headers = gen_tonic_signature_headers(
|
||||
"node-a:9000",
|
||||
"node_service.TierMutationControlService",
|
||||
"PrepareTierMutation",
|
||||
content_sha256,
|
||||
)
|
||||
.expect("body-bound tier mutation auth headers should build");
|
||||
request.metadata_mut().as_mut().extend(headers.clone());
|
||||
|
||||
assert!(
|
||||
verify_tonic_rpc_signature("node-a:9000", "/node_service.TierMutationControlService/PrepareTierMutation", &headers)
|
||||
.is_ok(),
|
||||
"tier mutation RPC signature must bind destination, service, method, nonce, and body digest"
|
||||
);
|
||||
let method_replay =
|
||||
verify_tonic_rpc_signature("node-a:9000", "/node_service.TierMutationControlService/CommitTierMutation", &headers)
|
||||
.expect_err("prepare auth must not replay to commit");
|
||||
assert_eq!(method_replay.to_string(), "Invalid RPC v2 signature");
|
||||
let service_replay = verify_tonic_rpc_signature("node-a:9000", "/node_service.NodeService/PrepareTierMutation", &headers)
|
||||
.expect_err("tier mutation auth must not replay to the legacy node service path");
|
||||
assert_eq!(service_replay.to_string(), "Invalid RPC v2 signature");
|
||||
let tampered_body = rustfs_protos::canonical_tier_mutation_rpc_body(
|
||||
rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION,
|
||||
rustfs_protos::TierMutationRpcPhase::Commit,
|
||||
mutation_id,
|
||||
b"canonical-tier-mutation-prepare",
|
||||
)
|
||||
.expect("small tier mutation body should encode");
|
||||
let tampered =
|
||||
verify_tonic_canonical_body_digest(&request, &tampered_body).expect_err("commit body must not match prepare digest");
|
||||
assert_eq!(tampered.to_string(), "RPC content SHA-256 mismatch");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn partial_v2_metadata_fails_closed() {
|
||||
ensure_test_rpc_secret();
|
||||
|
||||
@@ -14,7 +14,6 @@
|
||||
|
||||
use crate::cluster::rpc::client::{
|
||||
TonicInterceptor, gen_tonic_signature_interceptor, heal_control_time_out_client, node_service_time_out_client,
|
||||
tier_mutation_control_time_out_client,
|
||||
};
|
||||
use crate::cluster::rpc::{set_tonic_canonical_body_digest, verify_tonic_rpc_response_proof};
|
||||
use crate::error::{Error, Result};
|
||||
@@ -24,7 +23,6 @@ use crate::{
|
||||
runtime::sources as runtime_sources,
|
||||
services::metrics_realtime::{CollectMetricsOpts, MetricType},
|
||||
};
|
||||
use bytes::Bytes;
|
||||
use rmp_serde::{Deserializer, Serializer};
|
||||
use rustfs_madmin::{
|
||||
ServerProperties,
|
||||
@@ -32,6 +30,7 @@ use rustfs_madmin::{
|
||||
metrics::RealtimeMetrics,
|
||||
net::NetInfo,
|
||||
};
|
||||
use rustfs_protos::evict_failed_connection;
|
||||
use rustfs_protos::proto_gen::node_service::{
|
||||
BackgroundHealStatusRequest, CancelDecommissionRequest, ClearDecommissionRequest, DeleteBucketMetadataRequest,
|
||||
DeletePolicyRequest, DeleteServiceAccountRequest, DeleteUserRequest, GetCpusRequest, GetLiveEventsRequest, GetMemInfoRequest,
|
||||
@@ -40,11 +39,8 @@ use rustfs_protos::proto_gen::node_service::{
|
||||
LoadPolicyMappingRequest, LoadPolicyRequest, LoadRebalanceMetaRequest, LoadServiceAccountRequest,
|
||||
LoadTransitionTierConfigRequest, LoadUserRequest, LocalStorageInfoRequest, Mss, ReloadPoolMetaRequest,
|
||||
ReloadSiteReplicationConfigRequest, ScannerActivityRequest, ScannerActivityResponse, ServerInfoRequest, SignalServiceRequest,
|
||||
StartDecommissionRequest, StartProfilingRequest, StopRebalanceRequest, TierMutationAbortRequest, TierMutationCommitRequest,
|
||||
TierMutationControlResponse, TierMutationPeerState, TierMutationPrepareRequest, node_service_client::NodeServiceClient,
|
||||
tier_mutation_control_service_client::TierMutationControlServiceClient,
|
||||
StartDecommissionRequest, StartProfilingRequest, StopRebalanceRequest, node_service_client::NodeServiceClient,
|
||||
};
|
||||
use rustfs_protos::{TierMutationRpcPhase, evict_failed_connection};
|
||||
use rustfs_utils::XHost;
|
||||
use serde::{Deserialize, Serialize as _};
|
||||
use std::{
|
||||
@@ -61,7 +57,6 @@ use tonic::Request;
|
||||
use tonic::service::interceptor::InterceptedService;
|
||||
use tonic::transport::Channel;
|
||||
use tracing::{debug, info, warn};
|
||||
use uuid::Uuid;
|
||||
|
||||
pub const PEER_RESTSIGNAL: &str = "signal";
|
||||
pub const PEER_RESTSUB_SYS: &str = "sub-sys";
|
||||
@@ -124,82 +119,6 @@ pub struct PeerRestClient {
|
||||
recovery_running: Arc<AtomicBool>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum PeerTierMutationState {
|
||||
Prepared,
|
||||
Committed,
|
||||
Aborted,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub struct PeerTierMutationOutcome {
|
||||
pub state: PeerTierMutationState,
|
||||
pub applied: bool,
|
||||
}
|
||||
|
||||
fn validate_tier_mutation_response_proof(
|
||||
version: u32,
|
||||
phase: TierMutationRpcPhase,
|
||||
mutation_id: Uuid,
|
||||
canonical_payload: &[u8],
|
||||
response: &TierMutationControlResponse,
|
||||
) -> Result<()> {
|
||||
let canonical_response =
|
||||
rustfs_protos::canonical_tier_mutation_rpc_response_body(rustfs_protos::TierMutationRpcResponseProofInput {
|
||||
version,
|
||||
phase,
|
||||
mutation_id,
|
||||
canonical_payload,
|
||||
success: response.success,
|
||||
state: response.state,
|
||||
applied: response.applied,
|
||||
error_info: response.error_info.as_deref(),
|
||||
})
|
||||
.map_err(|_| Error::other("tier mutation response length cannot be represented"))?;
|
||||
verify_tonic_rpc_response_proof(&canonical_response, &response.response_proof)
|
||||
.map_err(|_| Error::other("peer returned an invalid tier mutation response proof"))
|
||||
}
|
||||
|
||||
fn decode_tier_mutation_peer_state(state: i32) -> Result<PeerTierMutationState> {
|
||||
match TierMutationPeerState::try_from(state).map_err(|_| Error::other("peer returned an invalid tier mutation state"))? {
|
||||
TierMutationPeerState::Prepared => Ok(PeerTierMutationState::Prepared),
|
||||
TierMutationPeerState::Committed => Ok(PeerTierMutationState::Committed),
|
||||
TierMutationPeerState::Aborted => Ok(PeerTierMutationState::Aborted),
|
||||
TierMutationPeerState::Unspecified => Err(Error::other("peer returned an unspecified tier mutation state")),
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_tier_mutation_payload_len(phase: TierMutationRpcPhase, payload_len: usize) -> Result<()> {
|
||||
let limit = match phase {
|
||||
TierMutationRpcPhase::Prepare => rustfs_protos::TIER_MUTATION_RPC_MAX_PREPARE_PAYLOAD_SIZE,
|
||||
TierMutationRpcPhase::Commit => rustfs_protos::TIER_MUTATION_RPC_MAX_COMMIT_PAYLOAD_SIZE,
|
||||
TierMutationRpcPhase::Abort => {
|
||||
if payload_len == 0 {
|
||||
return Ok(());
|
||||
}
|
||||
return Err(Error::other("tier mutation abort payload must be empty"));
|
||||
}
|
||||
_ => return Err(Error::other("tier mutation rpc phase is unsupported")),
|
||||
};
|
||||
if payload_len > limit {
|
||||
return Err(Error::other("tier mutation payload exceeds size limit"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn tier_mutation_phase_label(phase: TierMutationRpcPhase) -> &'static str {
|
||||
match phase {
|
||||
TierMutationRpcPhase::Prepare => "prepare",
|
||||
TierMutationRpcPhase::Commit => "commit",
|
||||
TierMutationRpcPhase::Abort => "abort",
|
||||
_ => "unknown",
|
||||
}
|
||||
}
|
||||
|
||||
fn tier_mutation_control_status_error(phase: TierMutationRpcPhase, status: tonic::Status) -> Error {
|
||||
Error::other(format!("peer tier mutation {} RPC failed: {status}", tier_mutation_phase_label(phase)))
|
||||
}
|
||||
|
||||
impl PeerRestClient {
|
||||
fn recovery_monitor_span(grid_host: &str) -> tracing::Span {
|
||||
tracing::info_span!(
|
||||
@@ -286,25 +205,6 @@ impl PeerRestClient {
|
||||
})
|
||||
}
|
||||
|
||||
async fn get_tier_mutation_control_client(
|
||||
&self,
|
||||
) -> Result<TierMutationControlServiceClient<InterceptedService<Channel, TonicInterceptor>>> {
|
||||
if self.offline.load(Ordering::Acquire) {
|
||||
self.mark_offline_and_spawn_recovery();
|
||||
return Err(Error::other(format!("peer {} is temporarily offline", self.grid_host)));
|
||||
}
|
||||
|
||||
tier_mutation_control_time_out_client(&self.grid_host, TonicInterceptor::Signature(gen_tonic_signature_interceptor()))
|
||||
.await
|
||||
.map_err(|err| {
|
||||
let storage_err = Error::other(format!("can not get tier mutation control client, err: {err}"));
|
||||
if Self::is_network_like_error(&storage_err) {
|
||||
self.mark_offline_and_spawn_recovery();
|
||||
}
|
||||
storage_err
|
||||
})
|
||||
}
|
||||
|
||||
/// Evict the connection to this peer from the global cache.
|
||||
/// This should be called when communication with this peer fails.
|
||||
pub async fn evict_connection(&self) {
|
||||
@@ -835,101 +735,6 @@ impl PeerRestClient {
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn prepare_tier_mutation(&self, mutation_id: Uuid, canonical_payload: Bytes) -> Result<PeerTierMutationOutcome> {
|
||||
self.tier_mutation_control(TierMutationRpcPhase::Prepare, mutation_id, canonical_payload)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn commit_tier_mutation(&self, mutation_id: Uuid, canonical_payload: Bytes) -> Result<PeerTierMutationOutcome> {
|
||||
self.tier_mutation_control(TierMutationRpcPhase::Commit, mutation_id, canonical_payload)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn abort_tier_mutation(&self, mutation_id: Uuid) -> Result<PeerTierMutationOutcome> {
|
||||
self.tier_mutation_control(TierMutationRpcPhase::Abort, mutation_id, Bytes::new())
|
||||
.await
|
||||
}
|
||||
|
||||
async fn tier_mutation_control(
|
||||
&self,
|
||||
phase: TierMutationRpcPhase,
|
||||
mutation_id: Uuid,
|
||||
canonical_payload: Bytes,
|
||||
) -> Result<PeerTierMutationOutcome> {
|
||||
validate_tier_mutation_payload_len(phase, canonical_payload.len())?;
|
||||
let version = rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION;
|
||||
self.finalize_result(
|
||||
async {
|
||||
let mut client = self
|
||||
.get_tier_mutation_control_client()
|
||||
.await?
|
||||
.max_encoding_message_size(rustfs_protos::TIER_MUTATION_RPC_MAX_MESSAGE_SIZE)
|
||||
.max_decoding_message_size(rustfs_protos::TIER_MUTATION_RPC_MAX_MESSAGE_SIZE);
|
||||
let canonical_body =
|
||||
rustfs_protos::canonical_tier_mutation_rpc_body(version, phase, mutation_id, canonical_payload.as_ref())
|
||||
.map_err(|_| Error::other("tier mutation request length cannot be represented"))?;
|
||||
let mutation_id_text = mutation_id.to_string();
|
||||
let response = match phase {
|
||||
TierMutationRpcPhase::Prepare => {
|
||||
let mut request = Request::new(TierMutationPrepareRequest {
|
||||
version,
|
||||
mutation_id: mutation_id_text.clone(),
|
||||
canonical_payload: canonical_payload.clone(),
|
||||
});
|
||||
set_tonic_canonical_body_digest(&mut request, &canonical_body)?;
|
||||
client
|
||||
.prepare_tier_mutation(request)
|
||||
.await
|
||||
.map_err(|status| tier_mutation_control_status_error(phase, status))?
|
||||
.into_inner()
|
||||
}
|
||||
TierMutationRpcPhase::Commit => {
|
||||
let mut request = Request::new(TierMutationCommitRequest {
|
||||
version,
|
||||
mutation_id: mutation_id_text.clone(),
|
||||
canonical_payload: canonical_payload.clone(),
|
||||
});
|
||||
set_tonic_canonical_body_digest(&mut request, &canonical_body)?;
|
||||
client
|
||||
.commit_tier_mutation(request)
|
||||
.await
|
||||
.map_err(|status| tier_mutation_control_status_error(phase, status))?
|
||||
.into_inner()
|
||||
}
|
||||
TierMutationRpcPhase::Abort => {
|
||||
let mut request = Request::new(TierMutationAbortRequest {
|
||||
version,
|
||||
mutation_id: mutation_id_text,
|
||||
canonical_payload: canonical_payload.clone(),
|
||||
});
|
||||
set_tonic_canonical_body_digest(&mut request, &canonical_body)?;
|
||||
client
|
||||
.abort_tier_mutation(request)
|
||||
.await
|
||||
.map_err(|status| tier_mutation_control_status_error(phase, status))?
|
||||
.into_inner()
|
||||
}
|
||||
_ => return Err(Error::other("tier mutation rpc phase is unsupported")),
|
||||
};
|
||||
validate_tier_mutation_response_proof(version, phase, mutation_id, &canonical_payload, &response)?;
|
||||
if !response.success {
|
||||
return Err(Error::other(
|
||||
response
|
||||
.error_info
|
||||
.unwrap_or_else(|| "peer tier mutation failed without an error".to_string()),
|
||||
));
|
||||
}
|
||||
let state = decode_tier_mutation_peer_state(response.state)?;
|
||||
Ok(PeerTierMutationOutcome {
|
||||
state,
|
||||
applied: response.applied,
|
||||
})
|
||||
}
|
||||
.await,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn heal_control(&self, version: u32, topology_fingerprint: String, command: Vec<u8>) -> Result<Vec<u8>> {
|
||||
if topology_fingerprint.len() > HEAL_CONTROL_FINGERPRINT_MAX_SIZE {
|
||||
return Err(Error::other("heal control topology fingerprint exceeds size limit"));
|
||||
@@ -1638,190 +1443,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
struct TierMutationResponseFixture<'a> {
|
||||
version: u32,
|
||||
phase: TierMutationRpcPhase,
|
||||
mutation_id: Uuid,
|
||||
canonical_payload: &'a [u8],
|
||||
success: bool,
|
||||
state: i32,
|
||||
applied: bool,
|
||||
error_info: Option<&'a str>,
|
||||
}
|
||||
|
||||
fn signed_tier_mutation_response(input: TierMutationResponseFixture<'_>) -> TierMutationControlResponse {
|
||||
let canonical =
|
||||
rustfs_protos::canonical_tier_mutation_rpc_response_body(rustfs_protos::TierMutationRpcResponseProofInput {
|
||||
version: input.version,
|
||||
phase: input.phase,
|
||||
mutation_id: input.mutation_id,
|
||||
canonical_payload: input.canonical_payload,
|
||||
success: input.success,
|
||||
state: input.state,
|
||||
applied: input.applied,
|
||||
error_info: input.error_info,
|
||||
})
|
||||
.expect("small tier mutation response should encode");
|
||||
let response_proof =
|
||||
crate::cluster::rpc::sign_tonic_rpc_response_proof(&canonical).expect("tier mutation response should sign");
|
||||
TierMutationControlResponse {
|
||||
success: input.success,
|
||||
state: input.state,
|
||||
applied: input.applied,
|
||||
error_info: input.error_info.map(str::to_string),
|
||||
response_proof: response_proof.into(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier_mutation_response_proof_binds_phase_payload_state_and_error() {
|
||||
runtime_sources::ensure_test_rpc_secret();
|
||||
let mutation_id = Uuid::new_v4();
|
||||
let payload = b"tier-mutation-prepare";
|
||||
let response = signed_tier_mutation_response(TierMutationResponseFixture {
|
||||
version: rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION,
|
||||
phase: TierMutationRpcPhase::Prepare,
|
||||
mutation_id,
|
||||
canonical_payload: payload,
|
||||
success: true,
|
||||
state: TierMutationPeerState::Prepared as i32,
|
||||
applied: true,
|
||||
error_info: None,
|
||||
});
|
||||
validate_tier_mutation_response_proof(
|
||||
rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION,
|
||||
TierMutationRpcPhase::Prepare,
|
||||
mutation_id,
|
||||
payload,
|
||||
&response,
|
||||
)
|
||||
.expect("matching tier mutation response proof should verify");
|
||||
|
||||
for tampered in [
|
||||
TierMutationControlResponse {
|
||||
success: false,
|
||||
error_info: Some("peer failed".to_string()),
|
||||
..response.clone()
|
||||
},
|
||||
TierMutationControlResponse {
|
||||
state: TierMutationPeerState::Committed as i32,
|
||||
..response.clone()
|
||||
},
|
||||
TierMutationControlResponse {
|
||||
applied: false,
|
||||
..response.clone()
|
||||
},
|
||||
] {
|
||||
let err = validate_tier_mutation_response_proof(
|
||||
rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION,
|
||||
TierMutationRpcPhase::Prepare,
|
||||
mutation_id,
|
||||
payload,
|
||||
&tampered,
|
||||
)
|
||||
.expect_err("tampered tier mutation response proof must fail");
|
||||
assert!(err.to_string().contains("invalid tier mutation response proof"));
|
||||
}
|
||||
|
||||
let err = validate_tier_mutation_response_proof(
|
||||
rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION,
|
||||
TierMutationRpcPhase::Commit,
|
||||
mutation_id,
|
||||
payload,
|
||||
&response,
|
||||
)
|
||||
.expect_err("response proof must bind request phase");
|
||||
assert!(err.to_string().contains("invalid tier mutation response proof"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier_mutation_peer_state_decode_fails_closed() {
|
||||
assert_eq!(
|
||||
decode_tier_mutation_peer_state(TierMutationPeerState::Prepared as i32).expect("prepared state should decode"),
|
||||
PeerTierMutationState::Prepared
|
||||
);
|
||||
assert_eq!(
|
||||
decode_tier_mutation_peer_state(TierMutationPeerState::Committed as i32).expect("committed state should decode"),
|
||||
PeerTierMutationState::Committed
|
||||
);
|
||||
assert_eq!(
|
||||
decode_tier_mutation_peer_state(TierMutationPeerState::Aborted as i32).expect("aborted state should decode"),
|
||||
PeerTierMutationState::Aborted
|
||||
);
|
||||
assert!(decode_tier_mutation_peer_state(TierMutationPeerState::Unspecified as i32).is_err());
|
||||
assert!(decode_tier_mutation_peer_state(99).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier_mutation_payload_guard_rejects_invalid_lengths() {
|
||||
validate_tier_mutation_payload_len(
|
||||
TierMutationRpcPhase::Prepare,
|
||||
rustfs_protos::TIER_MUTATION_RPC_MAX_PREPARE_PAYLOAD_SIZE,
|
||||
)
|
||||
.expect("max prepare payload should fit");
|
||||
assert!(
|
||||
validate_tier_mutation_payload_len(
|
||||
TierMutationRpcPhase::Prepare,
|
||||
rustfs_protos::TIER_MUTATION_RPC_MAX_PREPARE_PAYLOAD_SIZE + 1,
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
validate_tier_mutation_payload_len(
|
||||
TierMutationRpcPhase::Commit,
|
||||
rustfs_protos::TIER_MUTATION_RPC_MAX_COMMIT_PAYLOAD_SIZE,
|
||||
)
|
||||
.expect("max commit payload should fit");
|
||||
assert!(
|
||||
validate_tier_mutation_payload_len(
|
||||
TierMutationRpcPhase::Commit,
|
||||
rustfs_protos::TIER_MUTATION_RPC_MAX_COMMIT_PAYLOAD_SIZE + 1,
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
validate_tier_mutation_payload_len(TierMutationRpcPhase::Abort, 0).expect("empty abort payload should fit");
|
||||
assert!(validate_tier_mutation_payload_len(TierMutationRpcPhase::Abort, 1).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier_mutation_rpc_status_matrix_fails_closed_for_old_or_unresponsive_peers() {
|
||||
for (phase, label) in [
|
||||
(TierMutationRpcPhase::Prepare, "prepare"),
|
||||
(TierMutationRpcPhase::Commit, "commit"),
|
||||
(TierMutationRpcPhase::Abort, "abort"),
|
||||
] {
|
||||
for status in [
|
||||
tonic::Status::unimplemented("old peer has no tier mutation control service"),
|
||||
tonic::Status::deadline_exceeded("peer tier mutation control timed out"),
|
||||
tonic::Status::unavailable("peer tier mutation control unavailable"),
|
||||
] {
|
||||
let err = tier_mutation_control_status_error(phase, status);
|
||||
let rendered = err.to_string();
|
||||
assert!(rendered.contains(&format!("peer tier mutation {label} RPC failed")), "{rendered}");
|
||||
assert!(
|
||||
rendered.contains("old peer")
|
||||
|| rendered.contains("timed out")
|
||||
|| rendered.contains("unavailable")
|
||||
|| rendered.contains("Unavailable"),
|
||||
"{rendered}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn peer_rest_client_rejects_oversized_tier_prepare_before_dialing() {
|
||||
let client = test_peer_client();
|
||||
let err = client
|
||||
.prepare_tier_mutation(
|
||||
Uuid::new_v4(),
|
||||
Bytes::from(vec![0; rustfs_protos::TIER_MUTATION_RPC_MAX_PREPARE_PAYLOAD_SIZE + 1]),
|
||||
)
|
||||
.await
|
||||
.expect_err("oversized tier prepare should fail before dialing");
|
||||
assert!(err.to_string().contains("tier mutation payload exceeds size limit"));
|
||||
assert!(!client.offline.load(Ordering::Acquire));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn peer_rest_client_prepare_retry_clears_offline_gate() {
|
||||
// finalize_result sets the offline gate on a network error; without
|
||||
|
||||
@@ -23,7 +23,7 @@ use crate::disk::{DiskAPI, DiskStore, disk_store::get_max_timeout_duration};
|
||||
use crate::runtime::instance::{InstanceContext, bootstrap_ctx};
|
||||
use crate::runtime::sources as runtime_sources;
|
||||
use crate::storage_api_contracts::bucket::{BucketInfo, BucketOptions, DeleteBucketOptions, MakeBucketOptions};
|
||||
use crate::store::{has_xlmeta_files, utils::is_reserved_or_invalid_bucket};
|
||||
use crate::store::utils::is_reserved_or_invalid_bucket;
|
||||
use crate::{
|
||||
disk::{
|
||||
self, VolumeInfo,
|
||||
@@ -614,24 +614,13 @@ impl PeerS3Client for LocalPeerS3Client {
|
||||
return Err(Error::ErasureWriteQuorum);
|
||||
}
|
||||
|
||||
let force = if opts.force_if_empty && !opts.force {
|
||||
for disk in local_disks.iter() {
|
||||
if has_xlmeta_files(&disk.path().join(bucket)).await.map_err(Error::Io)? {
|
||||
return Err(Error::VolumeNotEmpty);
|
||||
}
|
||||
}
|
||||
true
|
||||
} else {
|
||||
opts.force
|
||||
};
|
||||
|
||||
let mut futures = Vec::with_capacity(local_disks.len());
|
||||
|
||||
for disk in local_disks.iter() {
|
||||
// Non-force delete refuses a non-empty bucket (VolumeNotEmpty), which
|
||||
// the recreate loop below turns into BucketNotEmpty; only an explicit
|
||||
// force delete removes recursively (backlog#799 B1).
|
||||
futures.push(disk.delete_volume(bucket, force));
|
||||
futures.push(disk.delete_volume(bucket, opts.force));
|
||||
}
|
||||
|
||||
let results = join_all(futures).await;
|
||||
@@ -999,15 +988,13 @@ impl PeerS3Client for RemotePeerS3Client {
|
||||
.await
|
||||
}
|
||||
|
||||
async fn delete_bucket(&self, bucket: &str, opts: &DeleteBucketOptions) -> Result<()> {
|
||||
async fn delete_bucket(&self, bucket: &str, _opts: &DeleteBucketOptions) -> Result<()> {
|
||||
self.execute_with_timeout(
|
||||
|| async {
|
||||
let options = serde_json::to_string(opts)?;
|
||||
let mut client = self.get_client().await?;
|
||||
|
||||
let request = Request::new(DeleteBucketRequest {
|
||||
bucket: bucket.to_string(),
|
||||
options,
|
||||
});
|
||||
let response = client.delete_bucket(request).await?.into_inner();
|
||||
if !response.success {
|
||||
|
||||
@@ -209,16 +209,6 @@ pub fn get_drive_walkdir_stall_timeout() -> Duration {
|
||||
)
|
||||
}
|
||||
|
||||
pub fn get_drive_walkdir_peek_timeout() -> Duration {
|
||||
let stall_timeout = get_drive_walkdir_stall_timeout();
|
||||
let configured = get_drive_timeout_duration(
|
||||
rustfs_config::ENV_DRIVE_WALKDIR_PEEK_TIMEOUT_SECS,
|
||||
rustfs_config::DEFAULT_DRIVE_WALKDIR_PEEK_TIMEOUT_SECS,
|
||||
Some(rustfs_config::DRIVE_TIMEOUT_PROFILE_HIGH_LATENCY_SECS.saturating_mul(2)),
|
||||
);
|
||||
configured.max(stall_timeout)
|
||||
}
|
||||
|
||||
pub fn get_object_disk_read_timeout() -> Duration {
|
||||
get_drive_timeout_duration(
|
||||
rustfs_config::ENV_OBJECT_DISK_READ_TIMEOUT,
|
||||
@@ -1666,64 +1656,6 @@ mod tests {
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn drive_walkdir_peek_timeout_uses_wider_default_when_unset() {
|
||||
temp_env::with_var_unset(rustfs_config::ENV_DRIVE_WALKDIR_PEEK_TIMEOUT_SECS, || {
|
||||
temp_env::with_var_unset(rustfs_config::ENV_DRIVE_WALKDIR_STALL_TIMEOUT_SECS, || {
|
||||
temp_env::with_var_unset(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, || {
|
||||
temp_env::with_var_unset(rustfs_config::ENV_DRIVE_TIMEOUT_PROFILE, || {
|
||||
assert_eq!(
|
||||
get_drive_walkdir_peek_timeout(),
|
||||
Duration::from_secs(rustfs_config::DEFAULT_DRIVE_WALKDIR_PEEK_TIMEOUT_SECS)
|
||||
);
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn drive_walkdir_peek_timeout_is_never_stricter_than_stall_timeout() {
|
||||
temp_env::with_var(rustfs_config::ENV_DRIVE_WALKDIR_PEEK_TIMEOUT_SECS, Some("3"), || {
|
||||
temp_env::with_var(rustfs_config::ENV_DRIVE_WALKDIR_STALL_TIMEOUT_SECS, Some("13"), || {
|
||||
temp_env::with_var_unset(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, || {
|
||||
assert_eq!(get_drive_walkdir_peek_timeout(), Duration::from_secs(13));
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn drive_walkdir_peek_timeout_prefers_canonical_over_legacy() {
|
||||
temp_env::with_var(rustfs_config::ENV_DRIVE_WALKDIR_PEEK_TIMEOUT_SECS, Some("23"), || {
|
||||
temp_env::with_var(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, Some("17"), || {
|
||||
temp_env::with_var_unset(rustfs_config::ENV_DRIVE_WALKDIR_STALL_TIMEOUT_SECS, || {
|
||||
assert_eq!(get_drive_walkdir_peek_timeout(), Duration::from_secs(23));
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn drive_walkdir_peek_timeout_uses_high_latency_profile_default() {
|
||||
temp_env::with_var_unset(rustfs_config::ENV_DRIVE_WALKDIR_PEEK_TIMEOUT_SECS, || {
|
||||
temp_env::with_var_unset(rustfs_config::ENV_DRIVE_WALKDIR_STALL_TIMEOUT_SECS, || {
|
||||
temp_env::with_var_unset(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, || {
|
||||
temp_env::with_var(
|
||||
rustfs_config::ENV_DRIVE_TIMEOUT_PROFILE,
|
||||
Some(rustfs_config::DRIVE_TIMEOUT_PROFILE_HIGH_LATENCY),
|
||||
|| {
|
||||
assert_eq!(
|
||||
get_drive_walkdir_peek_timeout(),
|
||||
Duration::from_secs(rustfs_config::DRIVE_TIMEOUT_PROFILE_HIGH_LATENCY_SECS * 2)
|
||||
);
|
||||
},
|
||||
);
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn drive_walkdir_timeout_prefers_canonical_over_legacy() {
|
||||
temp_env::with_var(rustfs_config::ENV_DRIVE_WALKDIR_TIMEOUT_SECS, Some("11"), || {
|
||||
|
||||
@@ -5172,25 +5172,7 @@ impl LocalDisk {
|
||||
// larger `RUSTFS_DRIVE_WALKDIR_STALL_TIMEOUT_SECS` or the high-latency
|
||||
// drive-timeout profile; a streaming readdir rewrite is a separate,
|
||||
// higher-risk follow-up and is intentionally not done here.
|
||||
let read_dir_started = rustfs_io_metrics::get_stage_metrics_enabled().then(std::time::Instant::now);
|
||||
let read_dir_result = with_walk_stall_timeout(stall, self.list_dir("", &opts.bucket, ¤t, -1)).await;
|
||||
if let Some(started) = read_dir_started {
|
||||
rustfs_io_metrics::record_list_objects_local_read_dir(rustfs_io_metrics::ListObjectsLocalReadDirObservation {
|
||||
outcome: if read_dir_result.is_ok() {
|
||||
rustfs_io_metrics::LIST_OBJECTS_LOCAL_READ_DIR_OUTCOME_OK
|
||||
} else {
|
||||
rustfs_io_metrics::LIST_OBJECTS_LOCAL_READ_DIR_OUTCOME_ERROR
|
||||
},
|
||||
requested_count: -1,
|
||||
returned_entries: read_dir_result.as_ref().map_or(0, Vec::len),
|
||||
duration_ms: started.elapsed().as_secs_f64() * 1000.0,
|
||||
is_root: current.trim_matches('/').is_empty(),
|
||||
has_filter_prefix: !prefix.is_empty(),
|
||||
has_forward: forward.is_some(),
|
||||
});
|
||||
}
|
||||
|
||||
let mut entries = match read_dir_result {
|
||||
let mut entries = match with_walk_stall_timeout(stall, self.list_dir("", &opts.bucket, ¤t, -1)).await {
|
||||
Ok(res) => res,
|
||||
Err(e) => {
|
||||
if e != DiskError::VolumeNotFound && e != Error::FileNotFound {
|
||||
@@ -11981,90 +11963,6 @@ mod test {
|
||||
assert!(names.contains(&"quux/thud".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn scan_dir_records_whole_parent_read_dir_before_page_limit() {
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("test runtime should be created");
|
||||
let recorder = crate::test_metrics::CapturingRecorder::default();
|
||||
let previous_gate = rustfs_io_metrics::get_stage_metrics_enabled();
|
||||
rustfs_io_metrics::set_get_stage_metrics_enabled(true);
|
||||
|
||||
metrics::with_local_recorder(&recorder, || {
|
||||
runtime.block_on(async {
|
||||
use rustfs_filemeta::MetacacheReader;
|
||||
use tempfile::tempdir;
|
||||
|
||||
let dir = tempdir().expect("tempdir should be created");
|
||||
let bucket = "test-bucket";
|
||||
let bucket_dir = dir.path().join(bucket);
|
||||
const OBJECTS: usize = 12;
|
||||
|
||||
for index in 0..OBJECTS {
|
||||
let object_dir = bucket_dir.join(format!("object-{index:04}"));
|
||||
fs::create_dir_all(&object_dir)
|
||||
.await
|
||||
.expect("object directory should be created");
|
||||
fs::write(object_dir.join(STORAGE_FORMAT_FILE), b"meta")
|
||||
.await
|
||||
.expect("object metadata should be written");
|
||||
}
|
||||
|
||||
let endpoint =
|
||||
Endpoint::try_from(dir.path().to_str().expect("tempdir path should be utf8")).expect("endpoint should parse");
|
||||
let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created");
|
||||
|
||||
let (reader, mut writer) = tokio::io::duplex(4096);
|
||||
let mut out = MetacacheWriter::new(&mut writer);
|
||||
let opts = WalkDirOptions {
|
||||
bucket: bucket.to_string(),
|
||||
base_dir: String::new(),
|
||||
recursive: true,
|
||||
limit: 1,
|
||||
..Default::default()
|
||||
};
|
||||
let mut objs_returned = 0;
|
||||
|
||||
disk.scan_dir(String::new(), String::new(), &opts, &mut out, &mut objs_returned, false, None)
|
||||
.await
|
||||
.expect("scan_dir should succeed");
|
||||
out.close().await.expect("metacache writer should close");
|
||||
drop(out);
|
||||
drop(writer);
|
||||
|
||||
let mut reader = MetacacheReader::new(reader);
|
||||
let visible_objects = reader
|
||||
.read_all()
|
||||
.await
|
||||
.expect("scan output should decode")
|
||||
.into_iter()
|
||||
.filter(|entry| !entry.metadata.is_empty())
|
||||
.count();
|
||||
|
||||
assert_eq!(visible_objects, 1);
|
||||
assert_eq!(objs_returned, 1);
|
||||
});
|
||||
});
|
||||
rustfs_io_metrics::set_get_stage_metrics_enabled(previous_gate);
|
||||
|
||||
assert_eq!(
|
||||
recorder.counter_value(
|
||||
"rustfs_s3_list_objects_local_read_dir_total",
|
||||
&[("outcome", "ok"), ("count_mode", "whole"), ("is_root", "true")]
|
||||
),
|
||||
1
|
||||
);
|
||||
assert_eq!(
|
||||
recorder.histogram_values(
|
||||
"rustfs_s3_list_objects_local_read_dir_entries",
|
||||
&[("outcome", "ok"), ("count_mode", "whole")]
|
||||
),
|
||||
vec![12.0]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_scan_dir_reports_base_dir_object_metadata() {
|
||||
use rustfs_filemeta::MetacacheReader;
|
||||
|
||||
@@ -50,7 +50,7 @@ use crate::services::event_notification::EventNotifier;
|
||||
use crate::services::tier::tier::TierConfigMgr;
|
||||
use rustfs_lock::{GlobalLockManager, get_global_lock_manager};
|
||||
use s3s::region::Region;
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::collections::HashMap;
|
||||
use std::sync::{Arc, OnceLock};
|
||||
use tokio::sync::{OnceCell, RwLock};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
@@ -159,10 +159,6 @@ pub struct InstanceContext {
|
||||
/// workers (scanner/heal/tier/lifecycle) without touching another instance.
|
||||
/// Replaces the process-global cancel-token static.
|
||||
background_cancel_token: OnceLock<CancellationToken>,
|
||||
tier_delete_journal_recovery_stores: std::sync::Mutex<HashSet<Uuid>>,
|
||||
transition_transaction_recovery_stores: std::sync::Mutex<HashSet<Uuid>>,
|
||||
#[cfg(test)]
|
||||
tier_delete_journal_recovery_wakeup: tokio::sync::Notify,
|
||||
}
|
||||
|
||||
impl InstanceContext {
|
||||
@@ -197,10 +193,6 @@ impl InstanceContext {
|
||||
local_disk_set_drives: Arc::new(RwLock::new(Vec::new())),
|
||||
bucket_metadata_sys: std::sync::Mutex::new(None),
|
||||
background_cancel_token: OnceLock::new(),
|
||||
tier_delete_journal_recovery_stores: std::sync::Mutex::new(HashSet::new()),
|
||||
transition_transaction_recovery_stores: std::sync::Mutex::new(HashSet::new()),
|
||||
#[cfg(test)]
|
||||
tier_delete_journal_recovery_wakeup: tokio::sync::Notify::new(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -361,34 +353,6 @@ impl InstanceContext {
|
||||
self.background_cancel_token.get().cloned()
|
||||
}
|
||||
|
||||
pub(crate) fn bind_background_cancel_token(&self, token: CancellationToken) -> CancellationToken {
|
||||
self.background_cancel_token.get_or_init(|| token).clone()
|
||||
}
|
||||
|
||||
pub(crate) fn mark_tier_delete_journal_recovery_started(&self, store_id: Uuid) -> bool {
|
||||
self.tier_delete_journal_recovery_stores
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.insert(store_id)
|
||||
}
|
||||
|
||||
pub(crate) fn mark_transition_transaction_recovery_started(&self, store_id: Uuid) -> bool {
|
||||
self.transition_transaction_recovery_stores
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.insert(store_id)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn wake_tier_delete_journal_recovery(&self) {
|
||||
self.tier_delete_journal_recovery_wakeup.notify_one();
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn wait_for_tier_delete_journal_recovery(&self) {
|
||||
self.tier_delete_journal_recovery_wakeup.notified().await;
|
||||
}
|
||||
|
||||
/// Update this instance's erasure setup type.
|
||||
pub async fn update_erasure_type(&self, setup_type: SetupType) {
|
||||
*self.erasure_kind.write().await = setup_type;
|
||||
@@ -442,20 +406,6 @@ impl std::fmt::Debug for InstanceContext {
|
||||
.field("replication_stats_set", &self.replication_stats.get().is_some())
|
||||
.field("replication_pool_set", &self.replication_pool.get().is_some())
|
||||
.field("background_cancel_token_set", &self.background_cancel_token.get().is_some())
|
||||
.field(
|
||||
"tier_delete_journal_recovery_store_count",
|
||||
&self
|
||||
.tier_delete_journal_recovery_stores
|
||||
.lock()
|
||||
.map_or(0, |stores| stores.len()),
|
||||
)
|
||||
.field(
|
||||
"transition_transaction_recovery_store_count",
|
||||
&self
|
||||
.transition_transaction_recovery_stores
|
||||
.lock()
|
||||
.map_or(0, |stores| stores.len()),
|
||||
)
|
||||
.finish_non_exhaustive()
|
||||
}
|
||||
}
|
||||
@@ -790,36 +740,6 @@ mod tests {
|
||||
assert!(ctx_a.background_cancel_token().unwrap().is_cancelled());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier_delete_journal_recovery_is_deduplicated_per_store_and_instance() {
|
||||
let ctx_a = InstanceContext::new();
|
||||
let ctx_b = InstanceContext::new();
|
||||
let store_a = Uuid::new_v4();
|
||||
let store_b = Uuid::new_v4();
|
||||
|
||||
assert!(ctx_a.mark_tier_delete_journal_recovery_started(store_a));
|
||||
assert!(!ctx_a.mark_tier_delete_journal_recovery_started(store_a));
|
||||
assert!(ctx_a.mark_tier_delete_journal_recovery_started(store_b));
|
||||
assert!(ctx_b.mark_tier_delete_journal_recovery_started(store_a));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn background_cancel_token_binds_the_provided_shutdown_token() {
|
||||
let ctx = InstanceContext::new();
|
||||
let shutdown = CancellationToken::new();
|
||||
let first = ctx.bind_background_cancel_token(shutdown.clone());
|
||||
let second = ctx.bind_background_cancel_token(CancellationToken::new());
|
||||
|
||||
shutdown.cancel();
|
||||
assert!(first.is_cancelled());
|
||||
assert!(second.is_cancelled());
|
||||
assert!(
|
||||
ctx.background_cancel_token()
|
||||
.expect("shutdown token should be published")
|
||||
.is_cancelled()
|
||||
);
|
||||
}
|
||||
|
||||
// Phase 5 acceptance (backlog#939): two independent instance contexts share
|
||||
// NONE of the runtime state that used to live in process globals. This is
|
||||
// the end-to-end proof that the object-graph isolation carrier works — every
|
||||
|
||||
@@ -21,7 +21,6 @@ use crate::runtime::sources as runtime_sources;
|
||||
use crate::services::metrics_realtime::{CollectMetricsOpts, MetricType};
|
||||
use crate::services::rebalance::RebalSaveOpt;
|
||||
use crate::storage_api_contracts::admin::StorageAdminApi;
|
||||
use bytes::Bytes;
|
||||
use futures::future::join_all;
|
||||
use lazy_static::lazy_static;
|
||||
use rustfs_madmin::health::{Cpus, MemInfo, OsInfo, Partitions, ProcInfo, SysConfig, SysErrors, SysServices};
|
||||
@@ -36,7 +35,6 @@ use std::sync::{Arc, Mutex, OnceLock};
|
||||
use std::time::{Duration, SystemTime};
|
||||
use tokio::time::timeout;
|
||||
use tracing::{debug, error, info, warn};
|
||||
use uuid::Uuid;
|
||||
|
||||
/// After this many consecutive admin-call failures, mark the peer as offline.
|
||||
const CONSECUTIVE_FAILURE_THRESHOLD: u32 = 3;
|
||||
@@ -115,17 +113,6 @@ pub struct NotificationPeerErr {
|
||||
pub err: Option<Error>,
|
||||
}
|
||||
|
||||
fn notification_peer_result<T>(host: String, result: Result<T>) -> NotificationPeerErr {
|
||||
NotificationPeerErr { host, err: result.err() }
|
||||
}
|
||||
|
||||
fn unreachable_notification_peer_err() -> NotificationPeerErr {
|
||||
NotificationPeerErr {
|
||||
host: String::new(),
|
||||
err: Some(Error::other("peer is not reachable")),
|
||||
}
|
||||
}
|
||||
|
||||
impl NotificationSys {
|
||||
pub fn rest_client_from_hash(&self, s: &str) -> Option<PeerRestClient> {
|
||||
if self.all_peer_clients.is_empty() {
|
||||
@@ -1083,50 +1070,6 @@ impl NotificationSys {
|
||||
}
|
||||
join_all(futures).await
|
||||
}
|
||||
|
||||
pub async fn prepare_tier_mutation(&self, mutation_id: Uuid, canonical_payload: Bytes) -> Vec<NotificationPeerErr> {
|
||||
let mut futures = Vec::with_capacity(self.peer_clients.len());
|
||||
for client in self.peer_clients.iter().cloned() {
|
||||
let payload = canonical_payload.clone();
|
||||
futures.push(async move {
|
||||
if let Some(client) = client {
|
||||
notification_peer_result(client.host.to_string(), client.prepare_tier_mutation(mutation_id, payload).await)
|
||||
} else {
|
||||
unreachable_notification_peer_err()
|
||||
}
|
||||
});
|
||||
}
|
||||
join_all(futures).await
|
||||
}
|
||||
|
||||
pub async fn commit_tier_mutation(&self, mutation_id: Uuid, canonical_payload: Bytes) -> Vec<NotificationPeerErr> {
|
||||
let mut futures = Vec::with_capacity(self.peer_clients.len());
|
||||
for client in self.peer_clients.iter().cloned() {
|
||||
let payload = canonical_payload.clone();
|
||||
futures.push(async move {
|
||||
if let Some(client) = client {
|
||||
notification_peer_result(client.host.to_string(), client.commit_tier_mutation(mutation_id, payload).await)
|
||||
} else {
|
||||
unreachable_notification_peer_err()
|
||||
}
|
||||
});
|
||||
}
|
||||
join_all(futures).await
|
||||
}
|
||||
|
||||
pub async fn abort_tier_mutation(&self, mutation_id: Uuid) -> Vec<NotificationPeerErr> {
|
||||
let mut futures = Vec::with_capacity(self.peer_clients.len());
|
||||
for client in self.peer_clients.iter().cloned() {
|
||||
futures.push(async move {
|
||||
if let Some(client) = client {
|
||||
notification_peer_result(client.host.to_string(), client.abort_tier_mutation(mutation_id).await)
|
||||
} else {
|
||||
unreachable_notification_peer_err()
|
||||
}
|
||||
});
|
||||
}
|
||||
join_all(futures).await
|
||||
}
|
||||
}
|
||||
|
||||
async fn scanner_activity_with_timeout<F>(timeout_duration: Duration, host: &str, activity: F) -> Result<ScannerPeerActivity>
|
||||
@@ -1783,36 +1726,6 @@ mod tests {
|
||||
assert!(results[0].err.as_ref().unwrap().to_string().contains("peer is not reachable"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn tier_mutation_fanout_reports_unreachable_peers_fail_closed() {
|
||||
let sys = NotificationSys {
|
||||
peer_clients: vec![None],
|
||||
all_peer_clients: Vec::new(),
|
||||
peer_admin_caches: vec![Mutex::new(PeerAdminCache::new())],
|
||||
};
|
||||
let mutation_id = Uuid::from_u128(1);
|
||||
|
||||
let prepare = sys.prepare_tier_mutation(mutation_id, Bytes::from_static(b"prepare")).await;
|
||||
assert_eq!(prepare.len(), 1);
|
||||
assert!(prepare[0].host.is_empty());
|
||||
assert!(
|
||||
prepare[0]
|
||||
.err
|
||||
.as_ref()
|
||||
.expect("unreachable prepare peer should carry an error")
|
||||
.to_string()
|
||||
.contains("peer is not reachable")
|
||||
);
|
||||
|
||||
let commit = sys.commit_tier_mutation(mutation_id, Bytes::from_static(b"commit")).await;
|
||||
assert_eq!(commit.len(), 1);
|
||||
assert!(commit[0].err.is_some());
|
||||
|
||||
let abort = sys.abort_tier_mutation(mutation_id).await;
|
||||
assert_eq!(abort.len(), 1);
|
||||
assert!(abort[0].err.is_some());
|
||||
}
|
||||
|
||||
// --- Tests for handle_peer_failure / handle_server_info_failure caching ---
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -19,8 +19,6 @@ pub mod tier_admin;
|
||||
pub mod tier_config;
|
||||
pub mod tier_gen;
|
||||
pub mod tier_handlers;
|
||||
pub(crate) mod tier_mutation_intent;
|
||||
pub mod tier_mutation_peer;
|
||||
pub mod warm_backend;
|
||||
pub mod warm_backend_aliyun;
|
||||
pub mod warm_backend_azure;
|
||||
@@ -32,4 +30,3 @@ pub mod warm_backend_rustfs;
|
||||
pub mod warm_backend_s3;
|
||||
pub mod warm_backend_s3sdk;
|
||||
pub mod warm_backend_tencent;
|
||||
pub mod warm_backend_wasabi;
|
||||
|
||||
@@ -57,10 +57,7 @@
|
||||
use std::collections::HashMap;
|
||||
use std::io::Cursor;
|
||||
use std::path::Path;
|
||||
use std::sync::{
|
||||
Arc,
|
||||
atomic::{AtomicBool, AtomicUsize, Ordering},
|
||||
};
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use async_trait::async_trait;
|
||||
@@ -78,21 +75,6 @@ use crate::services::tier::warm_backend::{WarmBackend, WarmBackendGetOpts, build
|
||||
use rustfs_filemeta::FileMeta;
|
||||
use rustfs_utils::path::path_join_buf;
|
||||
|
||||
/// One-shot barrier before rejected transition cleanup resolves its ECStore.
|
||||
pub struct TransitionCleanupStoreBarrier(crate::set_disk::SetDiskTransitionCleanupStoreBarrier);
|
||||
|
||||
impl TransitionCleanupStoreBarrier {
|
||||
/// Install the barrier for the next rejected transition cleanup.
|
||||
pub fn install() -> Self {
|
||||
Self(crate::set_disk::SetDiskTransitionCleanupStoreBarrier::install())
|
||||
}
|
||||
|
||||
/// Wait until the rejected transition reaches cleanup-store resolution.
|
||||
pub async fn wait_until_paused(&self) {
|
||||
self.0.wait_until_paused().await;
|
||||
}
|
||||
}
|
||||
|
||||
/// Default polling cadence used by the `wait_for_*` helpers.
|
||||
const POLL_INTERVAL: Duration = Duration::from_millis(50);
|
||||
|
||||
@@ -160,15 +142,11 @@ struct MockWarmBackendInner {
|
||||
faults: Mutex<FaultConfig>,
|
||||
put_read_limit: Mutex<Option<usize>>,
|
||||
put_remote_version: Mutex<Option<String>>,
|
||||
reject_non_empty_remote_versions: AtomicBool,
|
||||
fail_remove: AtomicBool,
|
||||
exact_remove_count: AtomicUsize,
|
||||
op_log: Mutex<Vec<MockWarmOp>>,
|
||||
put_versions: Mutex<Vec<(String, String)>>,
|
||||
remove_versions: Mutex<Vec<(String, String)>>,
|
||||
put_barrier: Mutex<Option<Arc<MockPutBarrierState>>>,
|
||||
get_barrier: Mutex<Option<Arc<MockGetBarrierState>>>,
|
||||
remove_barrier: Mutex<Option<Arc<MockRemoveBarrierState>>>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
@@ -184,23 +162,6 @@ struct MockGetBarrierState {
|
||||
fail_after_release: bool,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct MockRemoveBarrierState {
|
||||
arrived: Notify,
|
||||
release: Notify,
|
||||
operation_dropped: Notify,
|
||||
}
|
||||
|
||||
struct MockRemoveOperationGuard {
|
||||
state: Arc<MockRemoveBarrierState>,
|
||||
}
|
||||
|
||||
impl Drop for MockRemoveOperationGuard {
|
||||
fn drop(&mut self) {
|
||||
self.state.operation_dropped.notify_one();
|
||||
}
|
||||
}
|
||||
|
||||
/// One-shot barrier that pauses a mock tier PUT after storing its remote body.
|
||||
pub struct MockPutBarrier {
|
||||
state: Arc<MockPutBarrierState>,
|
||||
@@ -251,38 +212,6 @@ impl Drop for MockGetBarrier {
|
||||
}
|
||||
}
|
||||
|
||||
/// One-shot barrier that pauses and then fails a mock tier DELETE.
|
||||
pub struct MockRemoveBarrier {
|
||||
state: Arc<MockRemoveBarrierState>,
|
||||
}
|
||||
|
||||
impl MockRemoveBarrier {
|
||||
/// Wait until DELETE reaches the deterministic failure point.
|
||||
pub async fn wait_until_paused(&self) {
|
||||
tokio::time::timeout(Duration::from_secs(30), self.state.arrived.notified())
|
||||
.await
|
||||
.expect("mock tier DELETE should reach the deterministic barrier");
|
||||
}
|
||||
|
||||
/// Release the paused DELETE, which then returns an injected error.
|
||||
pub fn release(&self) {
|
||||
self.state.release.notify_one();
|
||||
}
|
||||
|
||||
/// Wait until the paused DELETE future completes or is cancelled.
|
||||
pub async fn wait_until_operation_dropped(&self) {
|
||||
tokio::time::timeout(Duration::from_secs(30), self.state.operation_dropped.notified())
|
||||
.await
|
||||
.expect("mock tier DELETE operation should be dropped");
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for MockRemoveBarrier {
|
||||
fn drop(&mut self) {
|
||||
self.state.release.notify_one();
|
||||
}
|
||||
}
|
||||
|
||||
/// In-memory [`WarmBackend`] for lifecycle / tiering integration tests.
|
||||
///
|
||||
/// Cloning shares the same underlying storage, fault configuration, and
|
||||
@@ -306,15 +235,6 @@ impl MockWarmBackend {
|
||||
MockPutBarrier { state }
|
||||
}
|
||||
|
||||
/// Pause and then fail the next DELETE after it reaches the backend.
|
||||
pub async fn arm_failing_remove_barrier(&self) -> MockRemoveBarrier {
|
||||
let state = Arc::new(MockRemoveBarrierState::default());
|
||||
let mut barrier = self.inner.remove_barrier.lock().await;
|
||||
assert!(barrier.is_none(), "mock tier DELETE barrier is already armed");
|
||||
*barrier = Some(state.clone());
|
||||
MockRemoveBarrier { state }
|
||||
}
|
||||
|
||||
/// Arm a one-shot pause before the next tier GET, then return an error
|
||||
/// after the test releases it.
|
||||
pub async fn arm_failing_get_barrier(&self) -> MockGetBarrier {
|
||||
@@ -326,14 +246,6 @@ impl MockWarmBackend {
|
||||
MockGetBarrier { state }
|
||||
}
|
||||
|
||||
/// Arm a one-shot pause before the next tier GET, then continue normally
|
||||
/// after the test releases it.
|
||||
pub async fn arm_get_barrier(&self) -> MockGetBarrier {
|
||||
let state = Arc::new(MockGetBarrierState::default());
|
||||
*self.inner.get_barrier.lock().await = Some(Arc::clone(&state));
|
||||
MockGetBarrier { state }
|
||||
}
|
||||
|
||||
// ---- fault injection -------------------------------------------------
|
||||
|
||||
/// Replace the entire fault configuration.
|
||||
@@ -378,16 +290,6 @@ impl MockWarmBackend {
|
||||
*self.inner.put_remote_version.lock().await = remote_version;
|
||||
}
|
||||
|
||||
/// Reject non-empty remote versions before transition metadata is committed.
|
||||
pub fn set_reject_non_empty_remote_versions(&self, reject: bool) {
|
||||
self.inner.reject_non_empty_remote_versions.store(reject, Ordering::Release);
|
||||
}
|
||||
|
||||
/// Enable or disable a persistent remove failure for durability tests.
|
||||
pub fn set_remove_failure(&self, fail: bool) {
|
||||
self.inner.fail_remove.store(fail, Ordering::Release);
|
||||
}
|
||||
|
||||
async fn precondition(&self) -> Result<(), std::io::Error> {
|
||||
let (latency, error) = {
|
||||
let faults = self.inner.faults.lock().await;
|
||||
@@ -429,11 +331,6 @@ impl MockWarmBackend {
|
||||
.count()
|
||||
}
|
||||
|
||||
/// Number of exact-version trait remove calls, including failed attempts.
|
||||
pub fn exact_remove_count(&self) -> usize {
|
||||
self.inner.exact_remove_count.load(Ordering::Acquire)
|
||||
}
|
||||
|
||||
/// Return the exact object/version pairs produced by successful tier PUTs.
|
||||
pub async fn put_versions(&self) -> Vec<(String, String)> {
|
||||
self.inner.put_versions.lock().await.clone()
|
||||
@@ -579,13 +476,6 @@ impl MockWarmBackend {
|
||||
|
||||
#[async_trait]
|
||||
impl WarmBackend for MockWarmBackend {
|
||||
fn validate_remote_version_id(&self, remote_version_id: &str) -> Result<(), std::io::Error> {
|
||||
if self.inner.reject_non_empty_remote_versions.load(Ordering::Acquire) && !remote_version_id.is_empty() {
|
||||
return Err(std::io::Error::other("mock warm backend requires an unversioned remote object"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn put(&self, object: &str, r: ReaderImpl, _length: i64) -> Result<String, std::io::Error> {
|
||||
self.precondition().await?;
|
||||
let bytes = self.read_bytes(r).await?;
|
||||
@@ -684,15 +574,6 @@ impl WarmBackend for MockWarmBackend {
|
||||
|
||||
async fn remove(&self, object: &str, rv: &str) -> Result<(), std::io::Error> {
|
||||
self.precondition().await?;
|
||||
if let Some(barrier) = self.inner.remove_barrier.lock().await.take() {
|
||||
let _operation = MockRemoveOperationGuard { state: barrier.clone() };
|
||||
barrier.arrived.notify_one();
|
||||
barrier.release.notified().await;
|
||||
return Err(std::io::Error::other("mock warm backend remove failure after barrier"));
|
||||
}
|
||||
if self.inner.fail_remove.load(Ordering::Acquire) {
|
||||
return Err(std::io::Error::other("mock warm backend remove failure"));
|
||||
}
|
||||
let mut objects = self.inner.objects.lock().await;
|
||||
if let Some(stored) = objects.get(object)
|
||||
&& !rv.is_empty()
|
||||
@@ -714,17 +595,6 @@ impl WarmBackend for MockWarmBackend {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn remove_exact(&self, object: &str, rv: &str) -> Result<(), std::io::Error> {
|
||||
if rv.is_empty() {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidInput,
|
||||
"an exact mock tier delete requires a remote version ID",
|
||||
));
|
||||
}
|
||||
self.inner.exact_remove_count.fetch_add(1, Ordering::AcqRel);
|
||||
self.remove(object, rv).await
|
||||
}
|
||||
|
||||
async fn in_use(&self) -> Result<bool, std::io::Error> {
|
||||
self.precondition().await?;
|
||||
self.record(MockWarmOp::InUse).await;
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -13,22 +13,12 @@
|
||||
// limitations under the License.
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::{fmt::Display, io};
|
||||
use std::fmt::Display;
|
||||
use tracing::info;
|
||||
|
||||
const C_TIER_CONFIG_VER: &str = "v1";
|
||||
|
||||
const ERR_TIER_NAME_EMPTY: &str = "remote tier name empty";
|
||||
const WASABI_US_EAST_ENDPOINT: &str = "https://s3.wasabisys.com";
|
||||
const WASABI_ALTERNATIVE_ENDPOINTS: &[(&str, &str)] = &[
|
||||
("us-east-1", "https://s3.us-east-1.wasabisys.com"),
|
||||
("eu-central-1", "https://s3.nl-1.wasabisys.com"),
|
||||
("eu-central-2", "https://s3.de-1.wasabisys.com"),
|
||||
("eu-west-1", "https://s3.uk-1.wasabisys.com"),
|
||||
("eu-west-2", "https://s3.fr-1.wasabisys.com"),
|
||||
("eu-west-3", "https://s3.uk-2.wasabisys.com"),
|
||||
("eu-south-1", "https://s3.it-1.wasabisys.com"),
|
||||
];
|
||||
|
||||
#[derive(Serialize, Deserialize, Default, Debug, Clone)]
|
||||
pub enum TierType {
|
||||
@@ -36,8 +26,6 @@ pub enum TierType {
|
||||
Unsupported,
|
||||
#[serde(rename = "s3")]
|
||||
S3,
|
||||
#[serde(rename = "wasabi")]
|
||||
Wasabi,
|
||||
#[serde(rename = "rustfs")]
|
||||
RustFS,
|
||||
#[serde(rename = "minio")]
|
||||
@@ -62,9 +50,6 @@ impl Display for TierType {
|
||||
TierType::S3 => {
|
||||
write!(f, "S3")
|
||||
}
|
||||
TierType::Wasabi => {
|
||||
write!(f, "Wasabi")
|
||||
}
|
||||
TierType::RustFS => {
|
||||
write!(f, "RustFS")
|
||||
}
|
||||
@@ -100,7 +85,6 @@ impl TierType {
|
||||
pub fn new(sc_type: &str) -> Self {
|
||||
match sc_type {
|
||||
"S3" => TierType::S3,
|
||||
"Wasabi" => TierType::Wasabi,
|
||||
"RustFS" => TierType::RustFS,
|
||||
"MinIO" => TierType::MinIO,
|
||||
"Aliyun" => TierType::Aliyun,
|
||||
@@ -116,7 +100,6 @@ impl TierType {
|
||||
pub fn as_lowercase(&self) -> String {
|
||||
match self {
|
||||
TierType::S3 => "s3".to_string(),
|
||||
TierType::Wasabi => "wasabi".to_string(),
|
||||
TierType::RustFS => "rustfs".to_string(),
|
||||
TierType::MinIO => "minio".to_string(),
|
||||
TierType::Aliyun => "aliyun".to_string(),
|
||||
@@ -141,8 +124,6 @@ pub struct TierConfig {
|
||||
pub name: String,
|
||||
#[serde(rename = "s3", skip_serializing_if = "Option::is_none")]
|
||||
pub s3: Option<TierS3>,
|
||||
#[serde(rename = "wasabi", skip_serializing_if = "Option::is_none")]
|
||||
pub wasabi: Option<TierWasabi>,
|
||||
#[serde(rename = "aliyun", skip_serializing_if = "Option::is_none")]
|
||||
pub aliyun: Option<TierAliyun>,
|
||||
#[serde(rename = "tencent", skip_serializing_if = "Option::is_none")]
|
||||
@@ -164,7 +145,6 @@ pub struct TierConfig {
|
||||
impl Clone for TierConfig {
|
||||
fn clone(&self) -> TierConfig {
|
||||
let mut s3 = None;
|
||||
let mut wasabi = None;
|
||||
let mut r = None;
|
||||
let mut compatible_backend = None;
|
||||
let mut aliyun = None;
|
||||
@@ -181,13 +161,6 @@ impl Clone for TierConfig {
|
||||
s3 = Some(s3_clone);
|
||||
}
|
||||
}
|
||||
TierType::Wasabi => {
|
||||
if let Some(wasabi_) = self.wasabi.as_ref() {
|
||||
let mut wasabi_clone = wasabi_.clone();
|
||||
wasabi_clone.secret_key = "REDACTED".to_string();
|
||||
wasabi = Some(wasabi_clone);
|
||||
}
|
||||
}
|
||||
TierType::RustFS => {
|
||||
if let Some(r_) = self.rustfs.as_ref() {
|
||||
let mut r_clone = r_.clone();
|
||||
@@ -251,7 +224,6 @@ impl Clone for TierConfig {
|
||||
tier_type: self.tier_type.clone(),
|
||||
name: self.name.clone(),
|
||||
s3,
|
||||
wasabi,
|
||||
rustfs: r,
|
||||
minio: compatible_backend,
|
||||
aliyun,
|
||||
@@ -272,7 +244,6 @@ impl TierConfig {
|
||||
tier_type: self.tier_type.clone(),
|
||||
name: self.name.clone(),
|
||||
s3: self.s3.clone(),
|
||||
wasabi: self.wasabi.clone(),
|
||||
aliyun: self.aliyun.clone(),
|
||||
tencent: self.tencent.clone(),
|
||||
huaweicloud: self.huaweicloud.clone(),
|
||||
@@ -287,7 +258,6 @@ impl TierConfig {
|
||||
fn endpoint(&self) -> String {
|
||||
match self.tier_type {
|
||||
TierType::S3 => self.s3.as_ref().map(|s| s.endpoint.clone()).unwrap_or_default(),
|
||||
TierType::Wasabi => self.wasabi.as_ref().map(|w| w.endpoint.clone()).unwrap_or_default(),
|
||||
TierType::RustFS => self.rustfs.as_ref().map(|r| r.endpoint.clone()).unwrap_or_default(),
|
||||
TierType::MinIO => self.minio.as_ref().map(|m| m.endpoint.clone()).unwrap_or_default(),
|
||||
TierType::Aliyun => self.aliyun.as_ref().map(|a| a.endpoint.clone()).unwrap_or_default(),
|
||||
@@ -306,7 +276,6 @@ impl TierConfig {
|
||||
fn bucket(&self) -> String {
|
||||
match self.tier_type {
|
||||
TierType::S3 => self.s3.as_ref().map(|s| s.bucket.clone()).unwrap_or_default(),
|
||||
TierType::Wasabi => self.wasabi.as_ref().map(|w| w.bucket.clone()).unwrap_or_default(),
|
||||
TierType::RustFS => self.rustfs.as_ref().map(|r| r.bucket.clone()).unwrap_or_default(),
|
||||
TierType::MinIO => self.minio.as_ref().map(|m| m.bucket.clone()).unwrap_or_default(),
|
||||
TierType::Aliyun => self.aliyun.as_ref().map(|a| a.bucket.clone()).unwrap_or_default(),
|
||||
@@ -325,7 +294,6 @@ impl TierConfig {
|
||||
fn prefix(&self) -> String {
|
||||
match self.tier_type {
|
||||
TierType::S3 => self.s3.as_ref().map(|s| s.prefix.clone()).unwrap_or_default(),
|
||||
TierType::Wasabi => self.wasabi.as_ref().map(|w| w.prefix.clone()).unwrap_or_default(),
|
||||
TierType::RustFS => self.rustfs.as_ref().map(|r| r.prefix.clone()).unwrap_or_default(),
|
||||
TierType::MinIO => self.minio.as_ref().map(|m| m.prefix.clone()).unwrap_or_default(),
|
||||
TierType::Aliyun => self.aliyun.as_ref().map(|a| a.prefix.clone()).unwrap_or_default(),
|
||||
@@ -344,7 +312,6 @@ impl TierConfig {
|
||||
fn region(&self) -> String {
|
||||
match self.tier_type {
|
||||
TierType::S3 => self.s3.as_ref().map(|s| s.region.clone()).unwrap_or_default(),
|
||||
TierType::Wasabi => self.wasabi.as_ref().map(|w| w.region.clone()).unwrap_or_default(),
|
||||
TierType::RustFS => self.rustfs.as_ref().map(|r| r.region.clone()).unwrap_or_default(),
|
||||
TierType::MinIO => self.minio.as_ref().map(|m| m.region.clone()).unwrap_or_default(),
|
||||
TierType::Aliyun => self.aliyun.as_ref().map(|a| a.region.clone()).unwrap_or_default(),
|
||||
@@ -389,73 +356,6 @@ pub struct TierS3 {
|
||||
pub aws_role_duration_seconds: i32,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Default, Clone)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct TierWasabi {
|
||||
pub name: String,
|
||||
#[serde(default)]
|
||||
pub endpoint: String,
|
||||
#[serde(rename = "accessKey")]
|
||||
pub access_key: String,
|
||||
#[serde(rename = "secretKey")]
|
||||
pub secret_key: String,
|
||||
pub bucket: String,
|
||||
#[serde(default)]
|
||||
pub prefix: String,
|
||||
pub region: String,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for TierWasabi {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("TierWasabi")
|
||||
.field("name", &self.name)
|
||||
.field("endpoint", &self.endpoint)
|
||||
.field("access_key", &self.access_key)
|
||||
.field("secret_key", &"REDACTED")
|
||||
.field("bucket", &self.bucket)
|
||||
.field("prefix", &self.prefix)
|
||||
.field("region", &self.region)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
impl TierWasabi {
|
||||
pub(crate) fn alternative_endpoint(&self) -> Option<&'static str> {
|
||||
WASABI_ALTERNATIVE_ENDPOINTS
|
||||
.iter()
|
||||
.find_map(|(region, endpoint)| (*region == self.region).then_some(*endpoint))
|
||||
}
|
||||
|
||||
pub(crate) fn canonical_endpoint(&self) -> io::Result<String> {
|
||||
let region = self.region.as_bytes();
|
||||
let is_alphanumeric = |byte: &u8| byte.is_ascii_lowercase() || byte.is_ascii_digit();
|
||||
if !(1..=63).contains(®ion.len())
|
||||
|| !region.first().is_some_and(is_alphanumeric)
|
||||
|| !region.last().is_some_and(is_alphanumeric)
|
||||
|| !region
|
||||
.iter()
|
||||
.all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || *byte == b'-')
|
||||
{
|
||||
return Err(io::Error::other("invalid Wasabi region"));
|
||||
}
|
||||
|
||||
let endpoint = if self.region == "us-east-1" {
|
||||
WASABI_US_EAST_ENDPOINT.to_string()
|
||||
} else {
|
||||
format!("https://s3.{}.wasabisys.com", self.region)
|
||||
};
|
||||
if !self.endpoint.is_empty() && self.endpoint != endpoint && self.alternative_endpoint() != Some(self.endpoint.as_str()) {
|
||||
return Err(io::Error::other(format!("Wasabi endpoint must be {endpoint}")));
|
||||
}
|
||||
Ok(endpoint)
|
||||
}
|
||||
|
||||
pub(crate) fn normalize_endpoint(&mut self) -> io::Result<()> {
|
||||
self.endpoint = self.canonical_endpoint()?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl TierS3 {
|
||||
#[allow(dead_code)]
|
||||
fn create<F>(
|
||||
@@ -715,200 +615,3 @@ pub struct TierR2 {
|
||||
pub prefix: String,
|
||||
pub region: String,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn wasabi_config() -> TierWasabi {
|
||||
TierWasabi {
|
||||
name: "COLD-WASABI".to_string(),
|
||||
access_key: "access".to_string(),
|
||||
secret_key: "secret".to_string(),
|
||||
bucket: "archive".to_string(),
|
||||
prefix: "objects".to_string(),
|
||||
region: "ap-northeast-1".to_string(),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn wasabi_type_and_endpoint_are_canonical() {
|
||||
let mut wasabi = wasabi_config();
|
||||
let endpoint = wasabi.canonical_endpoint().expect("valid region should produce an endpoint");
|
||||
assert_eq!(endpoint, "https://s3.ap-northeast-1.wasabisys.com");
|
||||
|
||||
wasabi
|
||||
.normalize_endpoint()
|
||||
.expect("valid region should normalize the endpoint");
|
||||
assert_eq!(wasabi.endpoint, endpoint);
|
||||
assert_eq!(
|
||||
wasabi.canonical_endpoint().expect("canonical endpoint should be accepted"),
|
||||
"https://s3.ap-northeast-1.wasabisys.com"
|
||||
);
|
||||
assert_eq!(TierType::new("Wasabi").to_string(), "Wasabi");
|
||||
assert_eq!(TierType::Wasabi.as_lowercase(), "wasabi");
|
||||
|
||||
for (region, alternative, canonical) in [
|
||||
("us-east-1", "https://s3.us-east-1.wasabisys.com", "https://s3.wasabisys.com"),
|
||||
("eu-central-1", "https://s3.nl-1.wasabisys.com", "https://s3.eu-central-1.wasabisys.com"),
|
||||
("eu-central-2", "https://s3.de-1.wasabisys.com", "https://s3.eu-central-2.wasabisys.com"),
|
||||
("eu-west-1", "https://s3.uk-1.wasabisys.com", "https://s3.eu-west-1.wasabisys.com"),
|
||||
("eu-west-2", "https://s3.fr-1.wasabisys.com", "https://s3.eu-west-2.wasabisys.com"),
|
||||
("eu-west-3", "https://s3.uk-2.wasabisys.com", "https://s3.eu-west-3.wasabisys.com"),
|
||||
("eu-south-1", "https://s3.it-1.wasabisys.com", "https://s3.eu-south-1.wasabisys.com"),
|
||||
] {
|
||||
let mut config = wasabi_config();
|
||||
config.region = region.to_string();
|
||||
config.endpoint = alternative.to_string();
|
||||
config
|
||||
.normalize_endpoint()
|
||||
.expect("a documented Wasabi alternative endpoint should normalize");
|
||||
assert_eq!(config.endpoint, canonical, "region {region}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn wasabi_endpoint_rejects_unsafe_region_and_custom_endpoint() {
|
||||
for region in [
|
||||
"",
|
||||
"US-EAST-1",
|
||||
"-us-east-1",
|
||||
"us-east-1-",
|
||||
"us.east-1",
|
||||
"us/east-1",
|
||||
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
||||
] {
|
||||
let mut wasabi = wasabi_config();
|
||||
wasabi.region = region.to_string();
|
||||
let err = wasabi.canonical_endpoint().expect_err("unsafe region must be rejected");
|
||||
assert_eq!(err.to_string(), "invalid Wasabi region", "region {region}");
|
||||
}
|
||||
|
||||
let mut wasabi = wasabi_config();
|
||||
wasabi.endpoint = "https://s3.example.invalid".to_string();
|
||||
let err = wasabi.canonical_endpoint().expect_err("custom endpoint must be rejected");
|
||||
assert!(err.to_string().contains("https://s3.ap-northeast-1.wasabisys.com"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn wasabi_config_rejects_unknown_fields_and_redacts_secret() {
|
||||
let omitted_optional = serde_json::from_value::<TierWasabi>(serde_json::json!({
|
||||
"name": "COLD-WASABI",
|
||||
"accessKey": "access",
|
||||
"secretKey": "secret",
|
||||
"bucket": "archive",
|
||||
"region": "us-east-1"
|
||||
}))
|
||||
.expect("endpoint and prefix should remain optional in Admin JSON");
|
||||
assert!(omitted_optional.endpoint.is_empty());
|
||||
assert!(omitted_optional.prefix.is_empty());
|
||||
assert_eq!(
|
||||
omitted_optional
|
||||
.canonical_endpoint()
|
||||
.expect("an omitted endpoint should be derived from the region"),
|
||||
WASABI_US_EAST_ENDPOINT
|
||||
);
|
||||
|
||||
let err = serde_json::from_value::<TierWasabi>(serde_json::json!({
|
||||
"name": "COLD-WASABI",
|
||||
"accessKey": "access",
|
||||
"secretKey": "secret",
|
||||
"bucket": "archive",
|
||||
"region": "us-east-1",
|
||||
"unexpected": true
|
||||
}))
|
||||
.expect_err("unknown Wasabi fields must be rejected");
|
||||
assert!(err.to_string().contains("unknown field `unexpected`"), "{err}");
|
||||
|
||||
let err = serde_json::from_value::<TierWasabi>(serde_json::json!({
|
||||
"name": "COLD-WASABI",
|
||||
"accessKey": "access",
|
||||
"bucket": "archive",
|
||||
"region": "us-east-1"
|
||||
}))
|
||||
.expect_err("missing Wasabi credentials must be rejected during decoding");
|
||||
assert!(err.to_string().contains("missing field `secretKey`"), "{err}");
|
||||
|
||||
let config = TierConfig {
|
||||
tier_type: TierType::Wasabi,
|
||||
wasabi: Some(wasabi_config()),
|
||||
..Default::default()
|
||||
};
|
||||
let redacted = config.clone();
|
||||
assert_eq!(
|
||||
redacted
|
||||
.wasabi
|
||||
.as_ref()
|
||||
.expect("redacted Wasabi payload should remain")
|
||||
.secret_key,
|
||||
"REDACTED"
|
||||
);
|
||||
assert_eq!(
|
||||
config
|
||||
.clone_with_credentials()
|
||||
.wasabi
|
||||
.as_ref()
|
||||
.expect("credential-bearing Wasabi payload should remain")
|
||||
.secret_key,
|
||||
"secret"
|
||||
);
|
||||
|
||||
let mut debug_config = wasabi_config();
|
||||
debug_config.secret_key = "wasabi-debug-secret-value".to_string();
|
||||
let debug = format!("{debug_config:?}");
|
||||
assert!(debug.contains("REDACTED"));
|
||||
assert!(!debug.contains("wasabi-debug-secret-value"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn wasabi_admin_json_shape_roundtrips_and_redacts() {
|
||||
let mut wasabi = wasabi_config();
|
||||
wasabi
|
||||
.normalize_endpoint()
|
||||
.expect("valid Wasabi configuration should normalize");
|
||||
let config = TierConfig {
|
||||
tier_type: TierType::Wasabi,
|
||||
wasabi: Some(wasabi),
|
||||
..Default::default()
|
||||
};
|
||||
let expected = serde_json::json!({
|
||||
"type": "wasabi",
|
||||
"wasabi": {
|
||||
"name": "COLD-WASABI",
|
||||
"endpoint": "https://s3.ap-northeast-1.wasabisys.com",
|
||||
"accessKey": "access",
|
||||
"secretKey": "secret",
|
||||
"bucket": "archive",
|
||||
"prefix": "objects",
|
||||
"region": "ap-northeast-1"
|
||||
}
|
||||
});
|
||||
|
||||
let encoded = serde_json::to_value(config.clone_with_credentials()).expect("Wasabi Admin JSON should encode");
|
||||
assert_eq!(encoded, expected);
|
||||
let decoded: TierConfig = serde_json::from_value(encoded).expect("Wasabi Admin JSON should decode");
|
||||
assert!(matches!(decoded.tier_type, TierType::Wasabi));
|
||||
let redacted = config.clone();
|
||||
assert_eq!(
|
||||
config
|
||||
.wasabi
|
||||
.as_ref()
|
||||
.expect("source Wasabi payload should remain")
|
||||
.secret_key,
|
||||
"secret"
|
||||
);
|
||||
assert_eq!(
|
||||
redacted
|
||||
.wasabi
|
||||
.as_ref()
|
||||
.expect("redacted Wasabi payload should remain")
|
||||
.secret_key,
|
||||
"REDACTED"
|
||||
);
|
||||
assert_eq!(
|
||||
serde_json::to_value(redacted).expect("redacted Wasabi JSON should encode")["wasabi"]["secretKey"],
|
||||
"REDACTED"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,785 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use rustfs_utils::crypto::{hex_sha256, is_sha256_checksum};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::config::com;
|
||||
use crate::disk::RUSTFS_META_BUCKET;
|
||||
use crate::error::{Error, Result as EcstoreResult};
|
||||
use crate::object_api::ObjectOptions;
|
||||
use crate::services::tier::tier::TierDestinationId;
|
||||
use crate::storage_api_contracts::{list::ListOperations as _, object::HTTPPreconditions};
|
||||
use crate::store::ECStore;
|
||||
|
||||
pub(crate) const TIER_MUTATION_INTENT_SCHEMA: &str = "rustfs-tier-mutation-intent-v1";
|
||||
pub(crate) const MAX_TIER_MUTATION_INTENT_SIZE: usize = rustfs_protos::TIER_MUTATION_RPC_MAX_PREPARE_PAYLOAD_SIZE;
|
||||
pub(crate) const TIER_MUTATION_INTENT_RECORD_PREFIX: &str = "tier/mutation-intents/records";
|
||||
pub(crate) type TierMutationDigest = [u8; 32];
|
||||
|
||||
pub(crate) type Result<T> = std::result::Result<T, TierMutationIntentError>;
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub(crate) enum TierMutationIntentError {
|
||||
#[error("tier mutation intent is corrupt: {0}")]
|
||||
Corrupt(&'static str),
|
||||
#[error("tier mutation intent schema is unsupported: {0}")]
|
||||
UnsupportedSchema(String),
|
||||
#[error("tier mutation intent checksum mismatch")]
|
||||
ChecksumMismatch,
|
||||
#[error("invalid tier mutation intent state change from {from:?} to {to:?}")]
|
||||
InvalidStateChange {
|
||||
from: TierMutationIntentState,
|
||||
to: TierMutationIntentState,
|
||||
},
|
||||
#[error("tier mutation intent json error: {0}")]
|
||||
Json(#[from] serde_json::Error),
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub(crate) enum TierMutationIntentState {
|
||||
Prepared,
|
||||
Committed,
|
||||
Aborted,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub(crate) enum TierMutationIntentKind {
|
||||
Add,
|
||||
Edit,
|
||||
Remove,
|
||||
Clear,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub(crate) struct TierMutationIntentTarget {
|
||||
pub tier_name: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub old_backend_identity: Option<TierDestinationId>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub new_backend_identity: Option<TierDestinationId>,
|
||||
}
|
||||
|
||||
impl TierMutationIntentTarget {
|
||||
pub(crate) fn validate(&self) -> Result<()> {
|
||||
if self.tier_name.is_empty() {
|
||||
return Err(TierMutationIntentError::Corrupt("target tier name is empty"));
|
||||
}
|
||||
if self.old_backend_identity.is_none() && self.new_backend_identity.is_none() {
|
||||
return Err(TierMutationIntentError::Corrupt("target has no backend identity"));
|
||||
}
|
||||
if self.old_backend_identity.as_ref().is_some_and(identity_is_empty) {
|
||||
return Err(TierMutationIntentError::Corrupt("target old backend identity is empty"));
|
||||
}
|
||||
if self.new_backend_identity.as_ref().is_some_and(identity_is_empty) {
|
||||
return Err(TierMutationIntentError::Corrupt("target new backend identity is empty"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub(crate) struct TierMutationIntent {
|
||||
pub mutation_id: Uuid,
|
||||
pub revision: u64,
|
||||
pub kind: TierMutationIntentKind,
|
||||
pub state: TierMutationIntentState,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub old_config_etag: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub committed_config_etag: Option<String>,
|
||||
pub candidate_digest: TierMutationDigest,
|
||||
pub affected_targets: Vec<TierMutationIntentTarget>,
|
||||
pub expires_at_unix_nanos: i64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub(crate) struct TierMutationIntentRecordScan {
|
||||
pub intents: Vec<TierMutationIntent>,
|
||||
pub scanned: usize,
|
||||
pub failed: usize,
|
||||
pub next_marker: Option<String>,
|
||||
pub truncated: bool,
|
||||
}
|
||||
|
||||
impl TierMutationIntent {
|
||||
pub(crate) fn validate(&self) -> Result<()> {
|
||||
if self.mutation_id.is_nil() {
|
||||
return Err(TierMutationIntentError::Corrupt("mutation_id is nil"));
|
||||
}
|
||||
if self.revision == 0 {
|
||||
return Err(TierMutationIntentError::Corrupt("revision is zero"));
|
||||
}
|
||||
if self.old_config_etag.as_deref().is_some_and(str::is_empty) {
|
||||
return Err(TierMutationIntentError::Corrupt("old config etag is empty"));
|
||||
}
|
||||
if self.old_config_etag.is_none() && self.kind != TierMutationIntentKind::Add {
|
||||
return Err(TierMutationIntentError::Corrupt("old config etag is required for mutation kind"));
|
||||
}
|
||||
if self.committed_config_etag.as_deref().is_some_and(str::is_empty) {
|
||||
return Err(TierMutationIntentError::Corrupt("committed config etag is empty"));
|
||||
}
|
||||
if self.state != TierMutationIntentState::Committed && self.committed_config_etag.is_some() {
|
||||
return Err(TierMutationIntentError::Corrupt("non-committed intent carries committed config etag"));
|
||||
}
|
||||
if self.state == TierMutationIntentState::Committed && self.committed_config_etag.is_none() {
|
||||
return Err(TierMutationIntentError::Corrupt("committed intent is missing committed config etag"));
|
||||
}
|
||||
if digest_is_empty(&self.candidate_digest) {
|
||||
return Err(TierMutationIntentError::Corrupt("candidate digest is empty"));
|
||||
}
|
||||
if self.affected_targets.is_empty() {
|
||||
return Err(TierMutationIntentError::Corrupt("affected targets are empty"));
|
||||
}
|
||||
if self.expires_at_unix_nanos <= 0 {
|
||||
return Err(TierMutationIntentError::Corrupt("expiry timestamp is not positive"));
|
||||
}
|
||||
|
||||
let mut previous: Option<&str> = None;
|
||||
for target in &self.affected_targets {
|
||||
target.validate()?;
|
||||
self.validate_target_shape(target)?;
|
||||
if previous.is_some_and(|previous| previous >= target.tier_name.as_str()) {
|
||||
return Err(TierMutationIntentError::Corrupt("affected targets are not canonical"));
|
||||
}
|
||||
previous = Some(&target.tier_name);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_target_shape(&self, target: &TierMutationIntentTarget) -> Result<()> {
|
||||
match self.kind {
|
||||
TierMutationIntentKind::Add if target.old_backend_identity.is_none() && target.new_backend_identity.is_some() => {
|
||||
Ok(())
|
||||
}
|
||||
TierMutationIntentKind::Edit if target.old_backend_identity.is_some() && target.new_backend_identity.is_some() => {
|
||||
Ok(())
|
||||
}
|
||||
TierMutationIntentKind::Remove | TierMutationIntentKind::Clear
|
||||
if target.old_backend_identity.is_some() && target.new_backend_identity.is_none() =>
|
||||
{
|
||||
Ok(())
|
||||
}
|
||||
_ => Err(TierMutationIntentError::Corrupt(
|
||||
"target backend identity shape does not match mutation kind",
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn advance(&mut self, next: TierMutationIntentState, committed_config_etag: Option<String>) -> Result<()> {
|
||||
match (self.state, next) {
|
||||
(TierMutationIntentState::Prepared, TierMutationIntentState::Committed) => {
|
||||
let committed_config_etag =
|
||||
committed_config_etag.ok_or(TierMutationIntentError::Corrupt("commit requires config etag"))?;
|
||||
if committed_config_etag.is_empty() {
|
||||
return Err(TierMutationIntentError::Corrupt("commit config etag is empty"));
|
||||
}
|
||||
self.state = next;
|
||||
self.committed_config_etag = Some(committed_config_etag);
|
||||
}
|
||||
(TierMutationIntentState::Prepared, TierMutationIntentState::Aborted) => {
|
||||
if committed_config_etag.is_some() {
|
||||
return Err(TierMutationIntentError::Corrupt("abort must not carry committed config etag"));
|
||||
}
|
||||
self.state = next;
|
||||
self.committed_config_etag = None;
|
||||
}
|
||||
_ => {
|
||||
return Err(TierMutationIntentError::InvalidStateChange {
|
||||
from: self.state,
|
||||
to: next,
|
||||
});
|
||||
}
|
||||
}
|
||||
self.revision = self
|
||||
.revision
|
||||
.checked_add(1)
|
||||
.ok_or(TierMutationIntentError::Corrupt("revision overflow"))?;
|
||||
self.validate()
|
||||
}
|
||||
|
||||
pub(crate) fn advance_idempotent(
|
||||
&mut self,
|
||||
next: TierMutationIntentState,
|
||||
committed_config_etag: Option<String>,
|
||||
) -> Result<bool> {
|
||||
match (self.state, next) {
|
||||
(TierMutationIntentState::Prepared, TierMutationIntentState::Committed | TierMutationIntentState::Aborted) => {
|
||||
self.advance(next, committed_config_etag)?;
|
||||
Ok(true)
|
||||
}
|
||||
(TierMutationIntentState::Committed, TierMutationIntentState::Committed) => {
|
||||
let committed_config_etag =
|
||||
committed_config_etag.ok_or(TierMutationIntentError::Corrupt("commit requires config etag"))?;
|
||||
if committed_config_etag.is_empty() {
|
||||
return Err(TierMutationIntentError::Corrupt("commit config etag is empty"));
|
||||
}
|
||||
if self.committed_config_etag.as_deref() != Some(committed_config_etag.as_str()) {
|
||||
return Err(TierMutationIntentError::Corrupt("committed config etag does not match"));
|
||||
}
|
||||
self.validate()?;
|
||||
Ok(false)
|
||||
}
|
||||
(TierMutationIntentState::Aborted, TierMutationIntentState::Aborted) => {
|
||||
if committed_config_etag.is_some() {
|
||||
return Err(TierMutationIntentError::Corrupt("abort must not carry committed config etag"));
|
||||
}
|
||||
self.validate()?;
|
||||
Ok(false)
|
||||
}
|
||||
_ => Err(TierMutationIntentError::InvalidStateChange {
|
||||
from: self.state,
|
||||
to: next,
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn encode(&self) -> Result<Vec<u8>> {
|
||||
self.validate()?;
|
||||
let intent_bytes = serde_json::to_vec(self)?;
|
||||
let content_sha256 = hex_sha256(&intent_bytes, ToOwned::to_owned);
|
||||
let persisted = PersistedTierMutationIntent {
|
||||
schema: TIER_MUTATION_INTENT_SCHEMA.to_string(),
|
||||
content_sha256,
|
||||
intent: self.clone(),
|
||||
};
|
||||
let encoded = serde_json::to_vec(&persisted)?;
|
||||
if encoded.len() > MAX_TIER_MUTATION_INTENT_SIZE {
|
||||
return Err(TierMutationIntentError::Corrupt("encoded intent exceeds maximum size"));
|
||||
}
|
||||
Ok(encoded)
|
||||
}
|
||||
|
||||
pub(crate) fn decode(expected_mutation_id: Uuid, data: &[u8]) -> Result<Self> {
|
||||
if data.len() > MAX_TIER_MUTATION_INTENT_SIZE {
|
||||
return Err(TierMutationIntentError::Corrupt("encoded intent exceeds maximum size"));
|
||||
}
|
||||
let persisted: PersistedTierMutationIntent = serde_json::from_slice(data)?;
|
||||
if persisted.schema != TIER_MUTATION_INTENT_SCHEMA {
|
||||
return Err(TierMutationIntentError::UnsupportedSchema(persisted.schema));
|
||||
}
|
||||
if !is_sha256_checksum(&persisted.content_sha256) {
|
||||
return Err(TierMutationIntentError::Corrupt("content checksum is not a sha256 checksum"));
|
||||
}
|
||||
let intent_bytes = serde_json::to_vec(&persisted.intent)?;
|
||||
let actual_checksum = hex_sha256(&intent_bytes, ToOwned::to_owned);
|
||||
if persisted.content_sha256 != actual_checksum {
|
||||
return Err(TierMutationIntentError::ChecksumMismatch);
|
||||
}
|
||||
if persisted.intent.mutation_id != expected_mutation_id {
|
||||
return Err(TierMutationIntentError::Corrupt("mutation_id does not match intent key"));
|
||||
}
|
||||
persisted.intent.validate()?;
|
||||
Ok(persisted.intent)
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn tier_mutation_intent_record_object_name(mutation_id: Uuid) -> Result<String> {
|
||||
if mutation_id.is_nil() {
|
||||
return Err(TierMutationIntentError::Corrupt("mutation_id is nil"));
|
||||
}
|
||||
let mutation_key = mutation_id.simple().to_string();
|
||||
Ok(format!(
|
||||
"{}/{}/{}/{}.json",
|
||||
TIER_MUTATION_INTENT_RECORD_PREFIX,
|
||||
&mutation_key[..2],
|
||||
&mutation_key[2..4],
|
||||
mutation_key
|
||||
))
|
||||
}
|
||||
|
||||
pub(crate) fn tier_mutation_intent_id_from_record_object_name(object: &str) -> Result<Uuid> {
|
||||
let prefix = format!("{TIER_MUTATION_INTENT_RECORD_PREFIX}/");
|
||||
let suffix = object
|
||||
.strip_prefix(&prefix)
|
||||
.ok_or(TierMutationIntentError::Corrupt("intent record path has wrong prefix"))?;
|
||||
let mut parts = suffix.split('/');
|
||||
let shard_a = parts
|
||||
.next()
|
||||
.ok_or(TierMutationIntentError::Corrupt("intent record path is incomplete"))?;
|
||||
let shard_b = parts
|
||||
.next()
|
||||
.ok_or(TierMutationIntentError::Corrupt("intent record path is incomplete"))?;
|
||||
let file_name = parts
|
||||
.next()
|
||||
.ok_or(TierMutationIntentError::Corrupt("intent record path is incomplete"))?;
|
||||
if parts.next().is_some() {
|
||||
return Err(TierMutationIntentError::Corrupt("intent record path is not canonical"));
|
||||
}
|
||||
let mutation_key = file_name
|
||||
.strip_suffix(".json")
|
||||
.ok_or(TierMutationIntentError::Corrupt("intent record path has wrong suffix"))?;
|
||||
if mutation_key.len() != 32
|
||||
|| !mutation_key
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase())
|
||||
{
|
||||
return Err(TierMutationIntentError::Corrupt("intent record path has invalid mutation id"));
|
||||
}
|
||||
if shard_a != &mutation_key[..2] || shard_b != &mutation_key[2..4] {
|
||||
return Err(TierMutationIntentError::Corrupt("intent record path shard does not match mutation id"));
|
||||
}
|
||||
Uuid::parse_str(mutation_key).map_err(|_| TierMutationIntentError::Corrupt("intent record path has invalid uuid"))
|
||||
}
|
||||
|
||||
pub(crate) async fn save_tier_mutation_intent_record(api: Arc<ECStore>, intent: &TierMutationIntent) -> EcstoreResult<()> {
|
||||
let object = tier_mutation_intent_record_object_name(intent.mutation_id).map_err(tier_mutation_intent_store_error)?;
|
||||
let data = intent.encode().map_err(tier_mutation_intent_store_error)?;
|
||||
com::save_config(api, &object, data).await
|
||||
}
|
||||
|
||||
pub(crate) async fn save_tier_mutation_intent_record_if_absent(
|
||||
api: Arc<ECStore>,
|
||||
intent: &TierMutationIntent,
|
||||
) -> EcstoreResult<()> {
|
||||
let object = tier_mutation_intent_record_object_name(intent.mutation_id).map_err(tier_mutation_intent_store_error)?;
|
||||
let data = intent.encode().map_err(tier_mutation_intent_store_error)?;
|
||||
com::save_config_with_opts(
|
||||
api,
|
||||
&object,
|
||||
data,
|
||||
&ObjectOptions {
|
||||
max_parity: true,
|
||||
http_preconditions: Some(HTTPPreconditions {
|
||||
if_none_match: Some("*".to_string()),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn load_tier_mutation_intent_record(api: Arc<ECStore>, mutation_id: Uuid) -> EcstoreResult<TierMutationIntent> {
|
||||
let (intent, _) = load_tier_mutation_intent_record_with_etag(api, mutation_id).await?;
|
||||
Ok(intent)
|
||||
}
|
||||
|
||||
pub(crate) async fn load_tier_mutation_intent_record_with_etag(
|
||||
api: Arc<ECStore>,
|
||||
mutation_id: Uuid,
|
||||
) -> EcstoreResult<(TierMutationIntent, String)> {
|
||||
let object = tier_mutation_intent_record_object_name(mutation_id).map_err(tier_mutation_intent_store_error)?;
|
||||
let (data, object_info) = com::read_config_with_metadata(api, &object, &ObjectOptions::default()).await?;
|
||||
let etag = object_info
|
||||
.etag
|
||||
.filter(|etag| !etag.trim().is_empty())
|
||||
.ok_or_else(|| Error::other("tier mutation intent record is missing an ETag"))?;
|
||||
let intent = TierMutationIntent::decode(mutation_id, &data).map_err(tier_mutation_intent_store_error)?;
|
||||
Ok((intent, etag))
|
||||
}
|
||||
|
||||
pub(crate) async fn save_tier_mutation_intent_record_if_current(
|
||||
api: Arc<ECStore>,
|
||||
intent: &TierMutationIntent,
|
||||
current_etag: &str,
|
||||
) -> EcstoreResult<()> {
|
||||
if current_etag.trim().is_empty() {
|
||||
return Err(Error::other("tier mutation intent current ETag is empty"));
|
||||
}
|
||||
let object = tier_mutation_intent_record_object_name(intent.mutation_id).map_err(tier_mutation_intent_store_error)?;
|
||||
let data = intent.encode().map_err(tier_mutation_intent_store_error)?;
|
||||
com::save_config_with_opts(
|
||||
api,
|
||||
&object,
|
||||
data,
|
||||
&ObjectOptions {
|
||||
max_parity: true,
|
||||
http_preconditions: Some(HTTPPreconditions {
|
||||
if_match: Some(current_etag.to_string()),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn delete_tier_mutation_intent_record(api: Arc<ECStore>, mutation_id: Uuid) -> EcstoreResult<()> {
|
||||
let object = tier_mutation_intent_record_object_name(mutation_id).map_err(tier_mutation_intent_store_error)?;
|
||||
match com::delete_config(api, &object).await {
|
||||
Ok(()) | Err(Error::ConfigNotFound) => Ok(()),
|
||||
Err(err) => Err(err),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn advance_tier_mutation_intent_record_idempotent(
|
||||
api: Arc<ECStore>,
|
||||
mutation_id: Uuid,
|
||||
next: TierMutationIntentState,
|
||||
committed_config_etag: Option<String>,
|
||||
) -> EcstoreResult<(TierMutationIntent, bool)> {
|
||||
let (mut intent, current_etag) = load_tier_mutation_intent_record_with_etag(api.clone(), mutation_id).await?;
|
||||
let advanced = intent
|
||||
.advance_idempotent(next, committed_config_etag)
|
||||
.map_err(tier_mutation_intent_store_error)?;
|
||||
if advanced {
|
||||
save_tier_mutation_intent_record_if_current(api, &intent, ¤t_etag).await?;
|
||||
}
|
||||
Ok((intent, advanced))
|
||||
}
|
||||
|
||||
pub(crate) async fn list_tier_mutation_intent_records(
|
||||
api: Arc<ECStore>,
|
||||
limit: usize,
|
||||
marker: Option<String>,
|
||||
) -> EcstoreResult<TierMutationIntentRecordScan> {
|
||||
if limit == 0 {
|
||||
return Err(Error::other("tier mutation intent scan limit must be greater than zero"));
|
||||
}
|
||||
|
||||
let list = api
|
||||
.clone()
|
||||
.list_objects_v2(
|
||||
RUSTFS_META_BUCKET,
|
||||
TIER_MUTATION_INTENT_RECORD_PREFIX,
|
||||
marker,
|
||||
None,
|
||||
i32::try_from(limit).map_or(i32::MAX, |value| value),
|
||||
false,
|
||||
None,
|
||||
false,
|
||||
)
|
||||
.await?;
|
||||
let mut scan = TierMutationIntentRecordScan {
|
||||
intents: Vec::with_capacity(list.objects.len()),
|
||||
scanned: 0,
|
||||
failed: 0,
|
||||
next_marker: list.next_continuation_token,
|
||||
truncated: list.is_truncated,
|
||||
};
|
||||
|
||||
for object in list.objects {
|
||||
scan.scanned += 1;
|
||||
let mutation_id = match tier_mutation_intent_id_from_record_object_name(&object.name) {
|
||||
Ok(mutation_id) => mutation_id,
|
||||
Err(_) => {
|
||||
scan.failed += 1;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
match load_tier_mutation_intent_record(api.clone(), mutation_id).await {
|
||||
Ok(intent) => scan.intents.push(intent),
|
||||
Err(Error::ConfigNotFound) => {}
|
||||
Err(_) => scan.failed += 1,
|
||||
}
|
||||
}
|
||||
|
||||
Ok(scan)
|
||||
}
|
||||
|
||||
fn tier_mutation_intent_store_error(err: TierMutationIntentError) -> Error {
|
||||
Error::other(err)
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct PersistedTierMutationIntent {
|
||||
schema: String,
|
||||
content_sha256: String,
|
||||
intent: TierMutationIntent,
|
||||
}
|
||||
|
||||
fn identity_is_empty(identity: &TierDestinationId) -> bool {
|
||||
identity.iter().all(|byte| *byte == 0)
|
||||
}
|
||||
|
||||
fn digest_is_empty(digest: &TierMutationDigest) -> bool {
|
||||
digest.iter().all(|byte| *byte == 0)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
const OLD_IDENTITY: TierDestinationId = [1; 32];
|
||||
const NEW_IDENTITY: TierDestinationId = [2; 32];
|
||||
const CANDIDATE_DIGEST: TierMutationDigest = [3; 32];
|
||||
|
||||
fn prepared_intent() -> TierMutationIntent {
|
||||
TierMutationIntent {
|
||||
mutation_id: Uuid::new_v4(),
|
||||
revision: 1,
|
||||
kind: TierMutationIntentKind::Edit,
|
||||
state: TierMutationIntentState::Prepared,
|
||||
old_config_etag: Some("old-etag".to_string()),
|
||||
committed_config_etag: None,
|
||||
candidate_digest: CANDIDATE_DIGEST,
|
||||
affected_targets: vec![TierMutationIntentTarget {
|
||||
tier_name: "COLD-A".to_string(),
|
||||
old_backend_identity: Some(OLD_IDENTITY),
|
||||
new_backend_identity: Some(NEW_IDENTITY),
|
||||
}],
|
||||
expires_at_unix_nanos: 1_780_000_000_000_000_000,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn intent_round_trip_preserves_committed_state() {
|
||||
let mut intent = prepared_intent();
|
||||
intent
|
||||
.advance(TierMutationIntentState::Committed, Some("new-etag".to_string()))
|
||||
.expect("intent should commit");
|
||||
|
||||
let encoded = intent.encode().expect("intent should encode");
|
||||
let decoded = TierMutationIntent::decode(intent.mutation_id, &encoded).expect("intent should decode");
|
||||
|
||||
assert_eq!(decoded, intent);
|
||||
assert_eq!(decoded.committed_config_etag.as_deref(), Some("new-etag"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn intent_decode_rejects_unknown_fields_and_checksum_mismatch() {
|
||||
let intent = prepared_intent();
|
||||
let encoded = intent.encode().expect("intent should encode");
|
||||
let mut persisted: serde_json::Value = serde_json::from_slice(&encoded).expect("encoded intent should be json");
|
||||
persisted["intent"]["unexpected"] = serde_json::Value::Bool(true);
|
||||
let unknown = serde_json::to_vec(&persisted).expect("mutated intent should encode");
|
||||
assert!(matches!(
|
||||
TierMutationIntent::decode(intent.mutation_id, &unknown),
|
||||
Err(TierMutationIntentError::Json(_))
|
||||
));
|
||||
|
||||
let mut persisted: serde_json::Value = serde_json::from_slice(&encoded).expect("encoded intent should be json");
|
||||
persisted["intent"]["revision"] = serde_json::Value::from(2_u64);
|
||||
let mismatched = serde_json::to_vec(&persisted).expect("mutated intent should encode");
|
||||
assert!(matches!(
|
||||
TierMutationIntent::decode(intent.mutation_id, &mismatched),
|
||||
Err(TierMutationIntentError::ChecksumMismatch)
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn intent_validation_requires_canonical_targets() {
|
||||
let mut duplicate = prepared_intent();
|
||||
duplicate.affected_targets.push(TierMutationIntentTarget {
|
||||
tier_name: "COLD-A".to_string(),
|
||||
old_backend_identity: Some(OLD_IDENTITY),
|
||||
new_backend_identity: Some(NEW_IDENTITY),
|
||||
});
|
||||
assert!(matches!(
|
||||
duplicate.validate(),
|
||||
Err(TierMutationIntentError::Corrupt("affected targets are not canonical"))
|
||||
));
|
||||
|
||||
let mut unsorted = prepared_intent();
|
||||
unsorted.affected_targets.insert(
|
||||
0,
|
||||
TierMutationIntentTarget {
|
||||
tier_name: "COLD-Z".to_string(),
|
||||
old_backend_identity: Some(OLD_IDENTITY),
|
||||
new_backend_identity: Some(NEW_IDENTITY),
|
||||
},
|
||||
);
|
||||
assert!(matches!(
|
||||
unsorted.validate(),
|
||||
Err(TierMutationIntentError::Corrupt("affected targets are not canonical"))
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn intent_state_machine_rejects_late_abort_after_commit() {
|
||||
let mut intent = prepared_intent();
|
||||
intent
|
||||
.advance(TierMutationIntentState::Committed, Some("new-etag".to_string()))
|
||||
.expect("intent should commit");
|
||||
|
||||
assert!(matches!(
|
||||
intent.advance(TierMutationIntentState::Aborted, None),
|
||||
Err(TierMutationIntentError::InvalidStateChange {
|
||||
from: TierMutationIntentState::Committed,
|
||||
to: TierMutationIntentState::Aborted,
|
||||
})
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn intent_idempotent_commit_retry_keeps_revision_and_rejects_conflicts() {
|
||||
let mut intent = prepared_intent();
|
||||
|
||||
assert!(
|
||||
intent
|
||||
.advance_idempotent(TierMutationIntentState::Committed, Some("new-etag".to_string()))
|
||||
.expect("initial commit should advance")
|
||||
);
|
||||
assert_eq!(intent.revision, 2);
|
||||
assert!(
|
||||
!intent
|
||||
.advance_idempotent(TierMutationIntentState::Committed, Some("new-etag".to_string()))
|
||||
.expect("same commit retry should be idempotent")
|
||||
);
|
||||
assert_eq!(intent.revision, 2);
|
||||
assert!(matches!(
|
||||
intent.advance_idempotent(TierMutationIntentState::Committed, Some("other-etag".to_string())),
|
||||
Err(TierMutationIntentError::Corrupt("committed config etag does not match"))
|
||||
));
|
||||
assert!(matches!(
|
||||
intent.advance_idempotent(TierMutationIntentState::Aborted, None),
|
||||
Err(TierMutationIntentError::InvalidStateChange {
|
||||
from: TierMutationIntentState::Committed,
|
||||
to: TierMutationIntentState::Aborted,
|
||||
})
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn intent_idempotent_abort_retry_keeps_revision_and_rejects_commit() {
|
||||
let mut intent = prepared_intent();
|
||||
|
||||
assert!(
|
||||
intent
|
||||
.advance_idempotent(TierMutationIntentState::Aborted, None)
|
||||
.expect("initial abort should advance")
|
||||
);
|
||||
assert_eq!(intent.revision, 2);
|
||||
assert!(
|
||||
!intent
|
||||
.advance_idempotent(TierMutationIntentState::Aborted, None)
|
||||
.expect("same abort retry should be idempotent")
|
||||
);
|
||||
assert_eq!(intent.revision, 2);
|
||||
assert!(matches!(
|
||||
intent.advance_idempotent(TierMutationIntentState::Aborted, Some("unexpected-etag".to_string())),
|
||||
Err(TierMutationIntentError::Corrupt("abort must not carry committed config etag"))
|
||||
));
|
||||
assert!(matches!(
|
||||
intent.advance_idempotent(TierMutationIntentState::Committed, Some("new-etag".to_string())),
|
||||
Err(TierMutationIntentError::InvalidStateChange {
|
||||
from: TierMutationIntentState::Aborted,
|
||||
to: TierMutationIntentState::Committed,
|
||||
})
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn intent_validation_rejects_placeholder_identity() {
|
||||
let mut intent = prepared_intent();
|
||||
intent.affected_targets[0].old_backend_identity = Some([0; 32]);
|
||||
|
||||
assert!(matches!(
|
||||
intent.validate(),
|
||||
Err(TierMutationIntentError::Corrupt("target old backend identity is empty"))
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn intent_allows_initial_config_create_without_old_etag() {
|
||||
let mut intent = prepared_intent();
|
||||
intent.kind = TierMutationIntentKind::Add;
|
||||
intent.old_config_etag = None;
|
||||
intent.affected_targets[0].old_backend_identity = None;
|
||||
|
||||
let encoded = intent.encode().expect("initial create intent should encode");
|
||||
let decoded = TierMutationIntent::decode(intent.mutation_id, &encoded).expect("initial create intent should decode");
|
||||
|
||||
assert_eq!(decoded.old_config_etag, None);
|
||||
assert_eq!(decoded.kind, TierMutationIntentKind::Add);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn intent_record_object_name_is_canonical_and_reversible() {
|
||||
let mutation_id = Uuid::parse_str("12345678-1234-5678-9abc-def012345678").expect("test uuid should parse");
|
||||
|
||||
let object = tier_mutation_intent_record_object_name(mutation_id).expect("record object should build");
|
||||
let parsed = tier_mutation_intent_id_from_record_object_name(&object).expect("record object should parse");
|
||||
|
||||
assert_eq!(object, "tier/mutation-intents/records/12/34/12345678123456789abcdef012345678.json");
|
||||
assert_eq!(parsed, mutation_id);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn intent_record_object_name_rejects_nil_and_malformed_paths() {
|
||||
assert!(matches!(
|
||||
tier_mutation_intent_record_object_name(Uuid::nil()),
|
||||
Err(TierMutationIntentError::Corrupt("mutation_id is nil"))
|
||||
));
|
||||
assert!(matches!(
|
||||
tier_mutation_intent_id_from_record_object_name("tier/mutation-intents/records/12/34/not-a-uuid.json"),
|
||||
Err(TierMutationIntentError::Corrupt("intent record path has invalid mutation id"))
|
||||
));
|
||||
assert!(matches!(
|
||||
tier_mutation_intent_id_from_record_object_name(
|
||||
"tier/mutation-intents/records/12/34/12345678123456789abcdef012345678"
|
||||
),
|
||||
Err(TierMutationIntentError::Corrupt("intent record path has wrong suffix"))
|
||||
));
|
||||
assert!(matches!(
|
||||
tier_mutation_intent_id_from_record_object_name(
|
||||
"tier/mutation-intents/records/12/35/12345678123456789abcdef012345678.json"
|
||||
),
|
||||
Err(TierMutationIntentError::Corrupt("intent record path shard does not match mutation id"))
|
||||
));
|
||||
assert!(matches!(
|
||||
tier_mutation_intent_id_from_record_object_name(
|
||||
"tier/mutation-intents/records/12/34/12345678123456789abcdef012345678/extra.json"
|
||||
),
|
||||
Err(TierMutationIntentError::Corrupt("intent record path is not canonical"))
|
||||
));
|
||||
assert!(matches!(
|
||||
tier_mutation_intent_id_from_record_object_name(
|
||||
"tier/mutation-intents/records/12/34/12345678123456789ABCDEF012345678.json"
|
||||
),
|
||||
Err(TierMutationIntentError::Corrupt("intent record path has invalid mutation id"))
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn intent_decode_rejects_record_key_mismatch() {
|
||||
let intent = prepared_intent();
|
||||
let encoded = intent.encode().expect("intent should encode");
|
||||
let wrong_id = Uuid::new_v4();
|
||||
|
||||
assert!(matches!(
|
||||
TierMutationIntent::decode(wrong_id, &encoded),
|
||||
Err(TierMutationIntentError::Corrupt("mutation_id does not match intent key"))
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn intent_validation_rejects_target_shape_mismatch() {
|
||||
let mut add = prepared_intent();
|
||||
add.kind = TierMutationIntentKind::Add;
|
||||
assert!(matches!(
|
||||
add.validate(),
|
||||
Err(TierMutationIntentError::Corrupt(
|
||||
"target backend identity shape does not match mutation kind"
|
||||
))
|
||||
));
|
||||
|
||||
let mut remove = prepared_intent();
|
||||
remove.kind = TierMutationIntentKind::Remove;
|
||||
assert!(matches!(
|
||||
remove.validate(),
|
||||
Err(TierMutationIntentError::Corrupt(
|
||||
"target backend identity shape does not match mutation kind"
|
||||
))
|
||||
));
|
||||
|
||||
let mut clear_without_etag = prepared_intent();
|
||||
clear_without_etag.kind = TierMutationIntentKind::Clear;
|
||||
clear_without_etag.old_config_etag = None;
|
||||
clear_without_etag.affected_targets[0].new_backend_identity = None;
|
||||
assert!(matches!(
|
||||
clear_without_etag.validate(),
|
||||
Err(TierMutationIntentError::Corrupt("old config etag is required for mutation kind"))
|
||||
));
|
||||
}
|
||||
}
|
||||
@@ -1,278 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use rustfs_protos::{TIER_MUTATION_RPC_PROTOCOL_VERSION, TierMutationRpcPhase};
|
||||
use uuid::Uuid;
|
||||
|
||||
use super::tier::TierConfigMgr;
|
||||
use super::tier_mutation_intent::{
|
||||
MAX_TIER_MUTATION_INTENT_SIZE, TierMutationIntent, TierMutationIntentState, advance_tier_mutation_intent_record_idempotent,
|
||||
load_tier_mutation_intent_record, save_tier_mutation_intent_record_if_absent,
|
||||
};
|
||||
use crate::client::admin_handler_utils::AdminError;
|
||||
use crate::error::{Error, StorageError};
|
||||
use crate::store::ECStore;
|
||||
|
||||
pub const MAX_TIER_MUTATION_PEER_COMMIT_ETAG_SIZE: usize = rustfs_protos::TIER_MUTATION_RPC_MAX_COMMIT_PAYLOAD_SIZE;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum TierMutationPeerState {
|
||||
Prepared,
|
||||
Committed,
|
||||
Aborted,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct TierMutationPeerOutcome {
|
||||
pub state: TierMutationPeerState,
|
||||
pub applied: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum TierMutationPeerError {
|
||||
#[error("unsupported tier mutation peer protocol version: {0}")]
|
||||
UnsupportedProtocolVersion(u32),
|
||||
#[error("tier mutation peer mutation_id is nil")]
|
||||
NilMutationId,
|
||||
#[error("tier mutation peer payload is too large: {len}/{max}")]
|
||||
PayloadTooLarge { len: usize, max: usize },
|
||||
#[error("tier mutation peer payload is invalid: {0}")]
|
||||
InvalidPayload(String),
|
||||
#[error("tier mutation peer intent conflicts with existing record")]
|
||||
ConflictingIntent,
|
||||
#[error("tier mutation peer runtime error: {0}")]
|
||||
Runtime(#[source] AdminError),
|
||||
#[error("tier mutation peer store error: {0}")]
|
||||
Store(#[source] StorageError),
|
||||
}
|
||||
|
||||
impl From<Error> for TierMutationPeerError {
|
||||
fn from(error: Error) -> Self {
|
||||
Self::Store(error)
|
||||
}
|
||||
}
|
||||
|
||||
pub type TierMutationPeerResult<T> = std::result::Result<T, TierMutationPeerError>;
|
||||
|
||||
pub async fn handle_tier_mutation_peer_request(
|
||||
api: Arc<ECStore>,
|
||||
protocol_version: u32,
|
||||
phase: TierMutationRpcPhase,
|
||||
mutation_id: Uuid,
|
||||
canonical_payload: &[u8],
|
||||
) -> TierMutationPeerResult<TierMutationPeerOutcome> {
|
||||
validate_peer_request_envelope(protocol_version, mutation_id, canonical_payload)?;
|
||||
match phase {
|
||||
TierMutationRpcPhase::Prepare => handle_prepare(api, mutation_id, canonical_payload).await,
|
||||
TierMutationRpcPhase::Commit => handle_commit(api, mutation_id, canonical_payload).await,
|
||||
TierMutationRpcPhase::Abort => handle_abort(api, mutation_id, canonical_payload).await,
|
||||
_ => Err(TierMutationPeerError::InvalidPayload(
|
||||
"tier mutation rpc phase is unsupported".to_string(),
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
async fn handle_prepare(
|
||||
api: Arc<ECStore>,
|
||||
mutation_id: Uuid,
|
||||
canonical_payload: &[u8],
|
||||
) -> TierMutationPeerResult<TierMutationPeerOutcome> {
|
||||
let intent = TierMutationIntent::decode(mutation_id, canonical_payload)
|
||||
.map_err(|err| TierMutationPeerError::InvalidPayload(err.to_string()))?;
|
||||
if intent.state != TierMutationIntentState::Prepared {
|
||||
return Err(TierMutationPeerError::InvalidPayload(
|
||||
"prepare intent must be in prepared state".to_string(),
|
||||
));
|
||||
}
|
||||
let tier_config_mgr = api.tier_config_mgr();
|
||||
|
||||
match save_tier_mutation_intent_record_if_absent(api.clone(), &intent).await {
|
||||
Ok(()) => {
|
||||
TierConfigMgr::apply_prepared_mutation_intent_block(&tier_config_mgr, &intent)
|
||||
.await
|
||||
.map_err(TierMutationPeerError::Runtime)?;
|
||||
Ok(TierMutationPeerOutcome {
|
||||
state: TierMutationPeerState::Prepared,
|
||||
applied: true,
|
||||
})
|
||||
}
|
||||
Err(Error::PreconditionFailed) => {
|
||||
let existing = load_tier_mutation_intent_record(api, mutation_id).await?;
|
||||
if !same_mutation_identity(&existing, &intent) {
|
||||
return Err(TierMutationPeerError::ConflictingIntent);
|
||||
}
|
||||
match existing.state {
|
||||
TierMutationIntentState::Prepared => {
|
||||
TierConfigMgr::apply_prepared_mutation_intent_block(&tier_config_mgr, &existing)
|
||||
.await
|
||||
.map_err(TierMutationPeerError::Runtime)?;
|
||||
}
|
||||
TierMutationIntentState::Committed | TierMutationIntentState::Aborted => {
|
||||
TierConfigMgr::clear_prepared_mutation_intent_block(&tier_config_mgr, mutation_id).await;
|
||||
}
|
||||
}
|
||||
Ok(TierMutationPeerOutcome {
|
||||
state: peer_state_from_intent(existing.state),
|
||||
applied: false,
|
||||
})
|
||||
}
|
||||
Err(err) => Err(err.into()),
|
||||
}
|
||||
}
|
||||
|
||||
async fn handle_commit(
|
||||
api: Arc<ECStore>,
|
||||
mutation_id: Uuid,
|
||||
canonical_payload: &[u8],
|
||||
) -> TierMutationPeerResult<TierMutationPeerOutcome> {
|
||||
let committed_config_etag = parse_commit_etag(canonical_payload)?;
|
||||
let tier_config_mgr = api.tier_config_mgr();
|
||||
let (intent, applied) = advance_tier_mutation_intent_record_idempotent(
|
||||
api,
|
||||
mutation_id,
|
||||
TierMutationIntentState::Committed,
|
||||
Some(committed_config_etag),
|
||||
)
|
||||
.await?;
|
||||
if intent.state == TierMutationIntentState::Committed {
|
||||
TierConfigMgr::clear_prepared_mutation_intent_block(&tier_config_mgr, mutation_id).await;
|
||||
}
|
||||
Ok(TierMutationPeerOutcome {
|
||||
state: peer_state_from_intent(intent.state),
|
||||
applied,
|
||||
})
|
||||
}
|
||||
|
||||
async fn handle_abort(
|
||||
api: Arc<ECStore>,
|
||||
mutation_id: Uuid,
|
||||
canonical_payload: &[u8],
|
||||
) -> TierMutationPeerResult<TierMutationPeerOutcome> {
|
||||
if !canonical_payload.is_empty() {
|
||||
return Err(TierMutationPeerError::InvalidPayload("abort payload must be empty".to_string()));
|
||||
}
|
||||
let tier_config_mgr = api.tier_config_mgr();
|
||||
let (intent, applied) =
|
||||
advance_tier_mutation_intent_record_idempotent(api, mutation_id, TierMutationIntentState::Aborted, None).await?;
|
||||
if intent.state == TierMutationIntentState::Aborted {
|
||||
TierConfigMgr::clear_prepared_mutation_intent_block(&tier_config_mgr, mutation_id).await;
|
||||
}
|
||||
Ok(TierMutationPeerOutcome {
|
||||
state: peer_state_from_intent(intent.state),
|
||||
applied,
|
||||
})
|
||||
}
|
||||
|
||||
fn validate_peer_request_envelope(
|
||||
protocol_version: u32,
|
||||
mutation_id: Uuid,
|
||||
canonical_payload: &[u8],
|
||||
) -> TierMutationPeerResult<()> {
|
||||
if protocol_version != TIER_MUTATION_RPC_PROTOCOL_VERSION {
|
||||
return Err(TierMutationPeerError::UnsupportedProtocolVersion(protocol_version));
|
||||
}
|
||||
if mutation_id.is_nil() {
|
||||
return Err(TierMutationPeerError::NilMutationId);
|
||||
}
|
||||
if canonical_payload.len() > MAX_TIER_MUTATION_INTENT_SIZE {
|
||||
return Err(TierMutationPeerError::PayloadTooLarge {
|
||||
len: canonical_payload.len(),
|
||||
max: MAX_TIER_MUTATION_INTENT_SIZE,
|
||||
});
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn parse_commit_etag(canonical_payload: &[u8]) -> TierMutationPeerResult<String> {
|
||||
if canonical_payload.len() > MAX_TIER_MUTATION_PEER_COMMIT_ETAG_SIZE {
|
||||
return Err(TierMutationPeerError::PayloadTooLarge {
|
||||
len: canonical_payload.len(),
|
||||
max: MAX_TIER_MUTATION_PEER_COMMIT_ETAG_SIZE,
|
||||
});
|
||||
}
|
||||
let etag = std::str::from_utf8(canonical_payload)
|
||||
.map_err(|err| TierMutationPeerError::InvalidPayload(err.to_string()))?
|
||||
.trim();
|
||||
if etag.is_empty() {
|
||||
return Err(TierMutationPeerError::InvalidPayload(
|
||||
"commit payload must carry a committed config etag".to_string(),
|
||||
));
|
||||
}
|
||||
Ok(etag.to_string())
|
||||
}
|
||||
|
||||
fn peer_state_from_intent(state: TierMutationIntentState) -> TierMutationPeerState {
|
||||
match state {
|
||||
TierMutationIntentState::Prepared => TierMutationPeerState::Prepared,
|
||||
TierMutationIntentState::Committed => TierMutationPeerState::Committed,
|
||||
TierMutationIntentState::Aborted => TierMutationPeerState::Aborted,
|
||||
}
|
||||
}
|
||||
|
||||
fn same_mutation_identity(existing: &TierMutationIntent, expected: &TierMutationIntent) -> bool {
|
||||
existing.mutation_id == expected.mutation_id
|
||||
&& existing.kind == expected.kind
|
||||
&& existing.old_config_etag == expected.old_config_etag
|
||||
&& existing.candidate_digest == expected.candidate_digest
|
||||
&& existing.affected_targets == expected.affected_targets
|
||||
&& existing.expires_at_unix_nanos == expected.expires_at_unix_nanos
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn peer_request_envelope_fails_closed_on_old_version_nil_id_and_large_payload() {
|
||||
let mutation_id = Uuid::new_v4();
|
||||
assert!(matches!(
|
||||
validate_peer_request_envelope(TIER_MUTATION_RPC_PROTOCOL_VERSION + 1, mutation_id, b"payload"),
|
||||
Err(TierMutationPeerError::UnsupportedProtocolVersion(_))
|
||||
));
|
||||
assert!(matches!(
|
||||
validate_peer_request_envelope(TIER_MUTATION_RPC_PROTOCOL_VERSION, Uuid::nil(), b"payload"),
|
||||
Err(TierMutationPeerError::NilMutationId)
|
||||
));
|
||||
|
||||
let oversized = vec![0; MAX_TIER_MUTATION_INTENT_SIZE + 1];
|
||||
assert!(matches!(
|
||||
validate_peer_request_envelope(TIER_MUTATION_RPC_PROTOCOL_VERSION, mutation_id, &oversized),
|
||||
Err(TierMutationPeerError::PayloadTooLarge { .. })
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn commit_payload_requires_small_non_empty_utf8_etag() {
|
||||
assert_eq!(
|
||||
parse_commit_etag(b" committed-etag ").expect("etag payload should parse"),
|
||||
"committed-etag"
|
||||
);
|
||||
assert!(matches!(
|
||||
parse_commit_etag(b" "),
|
||||
Err(TierMutationPeerError::InvalidPayload(message)) if message.contains("committed config etag")
|
||||
));
|
||||
assert!(matches!(
|
||||
parse_commit_etag(&[0xff]),
|
||||
Err(TierMutationPeerError::InvalidPayload(message)) if message.contains("utf-8")
|
||||
));
|
||||
|
||||
let oversized = vec![b'a'; MAX_TIER_MUTATION_PEER_COMMIT_ETAG_SIZE + 1];
|
||||
assert!(matches!(
|
||||
parse_commit_etag(&oversized),
|
||||
Err(TierMutationPeerError::PayloadTooLarge { .. })
|
||||
));
|
||||
}
|
||||
}
|
||||
@@ -25,7 +25,7 @@ use crate::client::{
|
||||
};
|
||||
use crate::error::is_err_bucket_not_found;
|
||||
use crate::services::tier::{
|
||||
tier::{ERR_TIER_INVALID_CONFIG, ERR_TIER_TYPE_UNSUPPORTED},
|
||||
tier::ERR_TIER_TYPE_UNSUPPORTED,
|
||||
tier_config::{TierConfig, TierType},
|
||||
tier_handlers::{ERR_TIER_BUCKET_NOT_FOUND, ERR_TIER_NOT_FOUND, ERR_TIER_PERM_ERR},
|
||||
warm_backend_aliyun::WarmBackendAliyun,
|
||||
@@ -37,7 +37,6 @@ use crate::services::tier::{
|
||||
warm_backend_rustfs::WarmBackendRustFS,
|
||||
warm_backend_s3::WarmBackendS3,
|
||||
warm_backend_tencent::WarmBackendTencent,
|
||||
warm_backend_wasabi::WarmBackendWasabi,
|
||||
};
|
||||
use bytes::Bytes;
|
||||
use http::StatusCode;
|
||||
@@ -66,14 +65,6 @@ pub struct WarmBackendGetOpts {
|
||||
|
||||
#[async_trait::async_trait]
|
||||
pub trait WarmBackend {
|
||||
async fn validate(&self) -> Result<(), std::io::Error> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_remote_version_id(&self, _remote_version_id: &str) -> Result<(), std::io::Error> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Return `Ok` only after the backend has consumed the complete declared
|
||||
/// body and its storage service has acknowledged the PUT. The built-in S3
|
||||
/// family uses the transition client's declared-length request plus
|
||||
@@ -92,15 +83,6 @@ pub trait WarmBackend {
|
||||
) -> Result<String, std::io::Error>;
|
||||
async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error>;
|
||||
async fn remove(&self, object: &str, rv: &str) -> Result<(), std::io::Error>;
|
||||
async fn remove_exact(&self, object: &str, rv: &str) -> Result<(), std::io::Error> {
|
||||
if rv.is_empty() {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidInput,
|
||||
"an exact tier delete requires a remote version ID",
|
||||
));
|
||||
}
|
||||
self.remove(object, rv).await
|
||||
}
|
||||
async fn in_use(&self) -> Result<bool, std::io::Error>;
|
||||
}
|
||||
|
||||
@@ -183,23 +165,16 @@ pub fn build_transition_put_options(storage_class: String, mut metadata: HashMap
|
||||
|
||||
pub async fn check_warm_backend(w: Option<&WarmBackendImpl>) -> Result<(), AdminError> {
|
||||
let w = w.ok_or_else(|| ERR_TIER_NOT_FOUND.clone())?;
|
||||
w.validate().await.map_err(|_| ERR_TIER_INVALID_CONFIG.clone())?;
|
||||
let remote_version_id = w
|
||||
.put(PROBE_OBJECT, ReaderImpl::Body(Bytes::from("RustFS".as_bytes().to_vec())), 5)
|
||||
.await
|
||||
.map_err(|_| ERR_TIER_PERM_ERR.clone())?;
|
||||
|
||||
if w.validate_remote_version_id(&remote_version_id).is_err() {
|
||||
w.remove_exact(PROBE_OBJECT, &remote_version_id)
|
||||
.await
|
||||
.map_err(|_| ERR_TIER_PERM_ERR.clone())?;
|
||||
return Err(ERR_TIER_INVALID_CONFIG.clone());
|
||||
.await;
|
||||
if let Err(err) = remote_version_id {
|
||||
return Err(ERR_TIER_PERM_ERR.clone());
|
||||
}
|
||||
|
||||
let read_result = w.get(PROBE_OBJECT, &remote_version_id, WarmBackendGetOpts::default()).await;
|
||||
let remove_result = w.remove(PROBE_OBJECT, &remote_version_id).await;
|
||||
let r = w.get(PROBE_OBJECT, "", WarmBackendGetOpts::default()).await;
|
||||
//xhttp.DrainBody(r);
|
||||
if read_result.is_err() || remove_result.is_err() {
|
||||
if let Err(err) = r {
|
||||
//if is_err_bucket_not_found(&err) {
|
||||
// return Err(ERR_TIER_BUCKET_NOT_FOUND);
|
||||
//}
|
||||
@@ -210,6 +185,11 @@ pub async fn check_warm_backend(w: Option<&WarmBackendImpl>) -> Result<(), Admin
|
||||
return Err(ERR_TIER_PERM_ERR.clone());
|
||||
//}
|
||||
}
|
||||
if let Ok(version_id) = remote_version_id {
|
||||
if let Err(err) = w.remove(PROBE_OBJECT, &version_id).await {
|
||||
return Err(ERR_TIER_PERM_ERR.clone());
|
||||
};
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -236,27 +216,6 @@ pub async fn new_warm_backend(tier: &TierConfig, probe: bool) -> Result<WarmBack
|
||||
});
|
||||
}
|
||||
}
|
||||
TierType::Wasabi => {
|
||||
if let Some(wasabi_config) = tier.wasabi.as_ref() {
|
||||
match WarmBackendWasabi::new(wasabi_config, &tier.name).await {
|
||||
Ok(backend) => d = Some(Box::new(backend)),
|
||||
Err(err) => {
|
||||
warn!("{}", err);
|
||||
return Err(AdminError {
|
||||
code: "XRustFSAdminTierInvalidConfig".to_string(),
|
||||
message: format!("Unable to setup remote tier, check tier configuration: {err}"),
|
||||
status_code: StatusCode::BAD_REQUEST,
|
||||
});
|
||||
}
|
||||
}
|
||||
} else {
|
||||
return Err(AdminError {
|
||||
code: "XRustFSAdminTierInvalidConfig".to_string(),
|
||||
message: "Wasabi tier configuration not found".to_string(),
|
||||
status_code: StatusCode::BAD_REQUEST,
|
||||
});
|
||||
}
|
||||
}
|
||||
TierType::RustFS => {
|
||||
if let Some(rustfs_config) = tier.rustfs.as_ref() {
|
||||
let dd = WarmBackendRustFS::new(rustfs_config, &tier.name).await;
|
||||
@@ -422,284 +381,16 @@ pub async fn new_warm_backend(tier: &TierConfig, probe: bool) -> Result<WarmBack
|
||||
}
|
||||
}
|
||||
|
||||
let d = d.ok_or_else(|| AdminError {
|
||||
d.ok_or_else(|| AdminError {
|
||||
code: "XRustFSAdminTierInvalidConfig".to_string(),
|
||||
message: "Tier backend not initialized".to_string(),
|
||||
status_code: StatusCode::BAD_REQUEST,
|
||||
})?;
|
||||
|
||||
if probe {
|
||||
d.validate().await.map_err(|_| ERR_TIER_INVALID_CONFIG.clone())?;
|
||||
}
|
||||
Ok(d)
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::services::tier::tier_config::TierWasabi;
|
||||
use std::sync::{
|
||||
Arc,
|
||||
atomic::{AtomicUsize, Ordering},
|
||||
};
|
||||
|
||||
const PROBE_VERSION: &str = "remote-v2";
|
||||
|
||||
struct RejectingValidationBackend {
|
||||
validations: Arc<AtomicUsize>,
|
||||
puts: Arc<AtomicUsize>,
|
||||
removes: Arc<AtomicUsize>,
|
||||
}
|
||||
|
||||
struct RejectingProbeVersionBackend {
|
||||
gets: Arc<AtomicUsize>,
|
||||
removed_versions: Arc<tokio::sync::Mutex<Vec<String>>>,
|
||||
}
|
||||
|
||||
struct RecordingProbeBackend {
|
||||
get_versions: Arc<tokio::sync::Mutex<Vec<String>>>,
|
||||
removed_versions: Arc<tokio::sync::Mutex<Vec<String>>>,
|
||||
fail_get: bool,
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl WarmBackend for RejectingValidationBackend {
|
||||
async fn validate(&self) -> Result<(), std::io::Error> {
|
||||
self.validations.fetch_add(1, Ordering::SeqCst);
|
||||
Err(std::io::Error::other("invalid backend configuration"))
|
||||
}
|
||||
|
||||
async fn put(&self, _object: &str, _r: ReaderImpl, _length: i64) -> Result<String, std::io::Error> {
|
||||
self.puts.fetch_add(1, Ordering::SeqCst);
|
||||
Ok(String::new())
|
||||
}
|
||||
|
||||
async fn put_with_meta(
|
||||
&self,
|
||||
object: &str,
|
||||
r: ReaderImpl,
|
||||
length: i64,
|
||||
_meta: HashMap<String, String>,
|
||||
) -> Result<String, std::io::Error> {
|
||||
self.put(object, r, length).await
|
||||
}
|
||||
|
||||
async fn get(&self, _object: &str, _rv: &str, _opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
|
||||
Err(std::io::Error::other("get must not run after validation failure"))
|
||||
}
|
||||
|
||||
async fn remove(&self, _object: &str, _rv: &str) -> Result<(), std::io::Error> {
|
||||
self.removes.fetch_add(1, Ordering::SeqCst);
|
||||
Err(std::io::Error::other("remove must not run after validation failure"))
|
||||
}
|
||||
|
||||
async fn in_use(&self) -> Result<bool, std::io::Error> {
|
||||
Err(std::io::Error::other("in_use must not run after validation failure"))
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl WarmBackend for RejectingProbeVersionBackend {
|
||||
fn validate_remote_version_id(&self, remote_version_id: &str) -> Result<(), std::io::Error> {
|
||||
if remote_version_id.is_empty() {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(std::io::Error::other("probe returned a version ID"))
|
||||
}
|
||||
}
|
||||
|
||||
async fn put(&self, _object: &str, _r: ReaderImpl, _length: i64) -> Result<String, std::io::Error> {
|
||||
Ok(uuid::Uuid::nil().to_string())
|
||||
}
|
||||
|
||||
async fn put_with_meta(
|
||||
&self,
|
||||
object: &str,
|
||||
r: ReaderImpl,
|
||||
length: i64,
|
||||
_meta: HashMap<String, String>,
|
||||
) -> Result<String, std::io::Error> {
|
||||
self.put(object, r, length).await
|
||||
}
|
||||
|
||||
async fn get(&self, _object: &str, _rv: &str, _opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
|
||||
self.gets.fetch_add(1, Ordering::SeqCst);
|
||||
Err(std::io::Error::other("GET must not run for a rejected probe version"))
|
||||
}
|
||||
|
||||
async fn remove(&self, _object: &str, _rv: &str) -> Result<(), std::io::Error> {
|
||||
Err(std::io::Error::other("generic remove must not run for a rejected fresh PUT response"))
|
||||
}
|
||||
|
||||
async fn remove_exact(&self, _object: &str, rv: &str) -> Result<(), std::io::Error> {
|
||||
self.removed_versions.lock().await.push(rv.to_string());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn in_use(&self) -> Result<bool, std::io::Error> {
|
||||
Ok(false)
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl WarmBackend for RecordingProbeBackend {
|
||||
async fn put(&self, _object: &str, _r: ReaderImpl, _length: i64) -> Result<String, std::io::Error> {
|
||||
Ok(PROBE_VERSION.to_string())
|
||||
}
|
||||
|
||||
async fn put_with_meta(
|
||||
&self,
|
||||
object: &str,
|
||||
r: ReaderImpl,
|
||||
length: i64,
|
||||
_meta: HashMap<String, String>,
|
||||
) -> Result<String, std::io::Error> {
|
||||
self.put(object, r, length).await
|
||||
}
|
||||
|
||||
async fn get(&self, _object: &str, rv: &str, _opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
|
||||
self.get_versions.lock().await.push(rv.to_string());
|
||||
if self.fail_get {
|
||||
Err(std::io::Error::other("probe GET failed"))
|
||||
} else {
|
||||
Ok(ReadCloser::new(std::io::Cursor::new(Vec::new())))
|
||||
}
|
||||
}
|
||||
|
||||
async fn remove(&self, _object: &str, rv: &str) -> Result<(), std::io::Error> {
|
||||
self.removed_versions.lock().await.push(rv.to_string());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn in_use(&self) -> Result<bool, std::io::Error> {
|
||||
Ok(false)
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn check_warm_backend_validates_before_probe_io() {
|
||||
let validations = Arc::new(AtomicUsize::new(0));
|
||||
let puts = Arc::new(AtomicUsize::new(0));
|
||||
let removes = Arc::new(AtomicUsize::new(0));
|
||||
let backend: WarmBackendImpl = Box::new(RejectingValidationBackend {
|
||||
validations: validations.clone(),
|
||||
puts: puts.clone(),
|
||||
removes: removes.clone(),
|
||||
});
|
||||
|
||||
let err = check_warm_backend(Some(&backend))
|
||||
.await
|
||||
.expect_err("invalid backend configuration should fail before probe I/O");
|
||||
|
||||
assert_eq!(err.code, ERR_TIER_INVALID_CONFIG.code);
|
||||
assert_eq!(validations.load(Ordering::SeqCst), 1);
|
||||
assert_eq!(puts.load(Ordering::SeqCst), 0);
|
||||
assert_eq!(removes.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn default_exact_remove_rejects_an_empty_version() {
|
||||
let removes = Arc::new(AtomicUsize::new(0));
|
||||
let backend = RejectingValidationBackend {
|
||||
validations: Arc::new(AtomicUsize::new(0)),
|
||||
puts: Arc::new(AtomicUsize::new(0)),
|
||||
removes: removes.clone(),
|
||||
};
|
||||
|
||||
let err = backend
|
||||
.remove_exact("remote-object", "")
|
||||
.await
|
||||
.expect_err("an empty exact constraint must fail closed");
|
||||
|
||||
assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput);
|
||||
assert_eq!(removes.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn check_warm_backend_removes_exact_probe_when_versioning_drifts() {
|
||||
let gets = Arc::new(AtomicUsize::new(0));
|
||||
let removed_versions = Arc::new(tokio::sync::Mutex::new(Vec::new()));
|
||||
let backend: WarmBackendImpl = Box::new(RejectingProbeVersionBackend {
|
||||
gets: gets.clone(),
|
||||
removed_versions: removed_versions.clone(),
|
||||
});
|
||||
|
||||
let err = check_warm_backend(Some(&backend))
|
||||
.await
|
||||
.expect_err("a probe version ID must fail an unversioned backend check");
|
||||
|
||||
assert_eq!(err.code, ERR_TIER_INVALID_CONFIG.code);
|
||||
assert_eq!(gets.load(Ordering::SeqCst), 0);
|
||||
assert_eq!(removed_versions.lock().await.as_slice(), [uuid::Uuid::nil().to_string()]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn check_warm_backend_forwards_probe_version_to_get_and_remove() {
|
||||
let get_versions = Arc::new(tokio::sync::Mutex::new(Vec::new()));
|
||||
let removed_versions = Arc::new(tokio::sync::Mutex::new(Vec::new()));
|
||||
let backend: WarmBackendImpl = Box::new(RecordingProbeBackend {
|
||||
get_versions: get_versions.clone(),
|
||||
removed_versions: removed_versions.clone(),
|
||||
fail_get: false,
|
||||
});
|
||||
|
||||
check_warm_backend(Some(&backend))
|
||||
.await
|
||||
.expect("a successful probe should validate, read, and remove its object");
|
||||
|
||||
assert_eq!(get_versions.lock().await.as_slice(), [PROBE_VERSION]);
|
||||
assert_eq!(removed_versions.lock().await.as_slice(), [PROBE_VERSION]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn check_warm_backend_removes_probe_after_get_failure() {
|
||||
let get_versions = Arc::new(tokio::sync::Mutex::new(Vec::new()));
|
||||
let removed_versions = Arc::new(tokio::sync::Mutex::new(Vec::new()));
|
||||
let backend: WarmBackendImpl = Box::new(RecordingProbeBackend {
|
||||
get_versions: get_versions.clone(),
|
||||
removed_versions: removed_versions.clone(),
|
||||
fail_get: true,
|
||||
});
|
||||
|
||||
let err = check_warm_backend(Some(&backend))
|
||||
.await
|
||||
.expect_err("a failed probe GET should return a permission error after cleanup");
|
||||
|
||||
assert_eq!(err.code, ERR_TIER_PERM_ERR.code);
|
||||
assert_eq!(get_versions.lock().await.as_slice(), [PROBE_VERSION]);
|
||||
assert_eq!(removed_versions.lock().await.as_slice(), [PROBE_VERSION]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn new_wasabi_backend_honors_probe_flag() {
|
||||
let tier = TierConfig {
|
||||
name: "WASABI".to_string(),
|
||||
tier_type: TierType::Wasabi,
|
||||
wasabi: Some(TierWasabi {
|
||||
name: "WASABI".to_string(),
|
||||
access_key: "invalid\naccess-key".to_string(),
|
||||
secret_key: "secret-key".to_string(),
|
||||
bucket: "tier-bucket".to_string(),
|
||||
prefix: "archive".to_string(),
|
||||
region: "us-east-1".to_string(),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let backend = new_warm_backend(&tier, false)
|
||||
.await
|
||||
.expect("valid Wasabi config should initialize without probing the remote service");
|
||||
|
||||
assert!(backend.validate_remote_version_id("").is_ok());
|
||||
assert!(backend.validate_remote_version_id("unexpected-version").is_err());
|
||||
|
||||
let err = match new_warm_backend(&tier, true).await {
|
||||
Ok(_) => panic!("probing a Wasabi backend must validate its credentials"),
|
||||
Err(err) => err,
|
||||
};
|
||||
assert_eq!(err.code, ERR_TIER_INVALID_CONFIG.code);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_transition_put_options_preserves_content_headers() {
|
||||
|
||||
@@ -25,9 +25,9 @@ use url::Url;
|
||||
use crate::client::{
|
||||
api_get_options::GetObjectOptions,
|
||||
api_put_object::PutObjectOptions,
|
||||
api_remove::{RemoveObjectOptions, RemoveObjectResult},
|
||||
api_remove::RemoveObjectOptions,
|
||||
credentials::{Credentials, SignatureType, Static, Value},
|
||||
transition_api::{BucketLookupType, Options, TransitionClient, TransitionCore},
|
||||
transition_api::{Options, TransitionClient, TransitionCore},
|
||||
transition_api::{ReadCloser, ReaderImpl},
|
||||
};
|
||||
use crate::error::ErrorResponse;
|
||||
@@ -36,7 +36,6 @@ use crate::services::tier::{
|
||||
tier_config::TierS3,
|
||||
warm_backend::{WarmBackend, WarmBackendGetOpts, build_transition_put_options},
|
||||
};
|
||||
use http::HeaderMap;
|
||||
use rustfs_utils::egress::validate_outbound_url;
|
||||
use rustfs_utils::path::SLASH_SEPARATOR;
|
||||
|
||||
@@ -49,15 +48,7 @@ pub struct WarmBackendS3 {
|
||||
}
|
||||
|
||||
impl WarmBackendS3 {
|
||||
pub async fn new(conf: &TierS3, _tier: &str) -> Result<Self, std::io::Error> {
|
||||
Self::new_with_bucket_lookup(conf, BucketLookupType::BucketLookupAuto, "s3").await
|
||||
}
|
||||
|
||||
pub(crate) async fn new_with_bucket_lookup(
|
||||
conf: &TierS3,
|
||||
bucket_lookup: BucketLookupType,
|
||||
tier_type: &str,
|
||||
) -> Result<Self, std::io::Error> {
|
||||
pub async fn new(conf: &TierS3, tier: &str) -> Result<Self, std::io::Error> {
|
||||
let u = match Url::parse(&conf.endpoint) {
|
||||
Ok(u) => u,
|
||||
Err(err) => {
|
||||
@@ -103,13 +94,12 @@ impl WarmBackendS3 {
|
||||
creds,
|
||||
secure: u.scheme() == "https",
|
||||
region: conf.region.clone(),
|
||||
bucket_lookup,
|
||||
..Default::default()
|
||||
};
|
||||
let host = u
|
||||
.host()
|
||||
.ok_or_else(|| std::io::Error::other("Invalid endpoint URL: missing host"))?;
|
||||
let client = TransitionClient::new(&host.to_string(), opts, tier_type).await?;
|
||||
let client = TransitionClient::new(&host.to_string(), opts, "s3").await?;
|
||||
|
||||
let client = Arc::new(client);
|
||||
let core = TransitionCore(Arc::clone(&client));
|
||||
@@ -129,36 +119,6 @@ impl WarmBackendS3 {
|
||||
}
|
||||
return dest_obj;
|
||||
}
|
||||
|
||||
pub(crate) async fn remove_with_result(&self, object: &str, rv: &str) -> Result<RemoveObjectResult, std::io::Error> {
|
||||
let mut opts = RemoveObjectOptions::default();
|
||||
if !rv.is_empty() {
|
||||
opts.version_id = rv.to_string();
|
||||
}
|
||||
self.client
|
||||
.remove_object_inner(&self.bucket, &self.get_dest(object), opts)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn get_with_headers(
|
||||
&self,
|
||||
object: &str,
|
||||
rv: &str,
|
||||
opts: WarmBackendGetOpts,
|
||||
) -> Result<(HeaderMap, ReadCloser), std::io::Error> {
|
||||
let mut gopts = GetObjectOptions::default();
|
||||
|
||||
if !rv.is_empty() {
|
||||
gopts.version_id = rv.to_string();
|
||||
}
|
||||
if opts.start_offset >= 0 && opts.length > 0 {
|
||||
gopts
|
||||
.set_range(opts.start_offset, opts.start_offset + opts.length - 1)
|
||||
.map_err(std::io::Error::other)?;
|
||||
}
|
||||
let (_, headers, reader) = self.core.get_object(&self.bucket, &self.get_dest(object), &gopts).await?;
|
||||
Ok((headers, reader))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -208,11 +168,32 @@ impl WarmBackend for WarmBackendS3 {
|
||||
}
|
||||
|
||||
async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
|
||||
self.get_with_headers(object, rv, opts).await.map(|(_, reader)| reader)
|
||||
let mut gopts = GetObjectOptions::default();
|
||||
|
||||
if rv != "" {
|
||||
gopts.version_id = rv.to_string();
|
||||
}
|
||||
if opts.start_offset >= 0 && opts.length > 0 {
|
||||
if let Err(err) = gopts.set_range(opts.start_offset, opts.start_offset + opts.length - 1) {
|
||||
return Err(std::io::Error::other(err));
|
||||
}
|
||||
}
|
||||
let c = TransitionCore(Arc::clone(&self.client));
|
||||
let (_, _, r) = c.get_object(&self.bucket, &self.get_dest(object), &gopts).await?;
|
||||
|
||||
Ok(r)
|
||||
}
|
||||
|
||||
async fn remove(&self, object: &str, rv: &str) -> Result<(), std::io::Error> {
|
||||
self.remove_with_result(object, rv).await.map(|_| ())
|
||||
let mut ropts = RemoveObjectOptions::default();
|
||||
if rv != "" {
|
||||
ropts.version_id = rv.to_string();
|
||||
}
|
||||
let client = self.client.clone();
|
||||
match client.remove_object(&self.bucket, &self.get_dest(object), ropts).await {
|
||||
None => Ok(()),
|
||||
Some(err) => Err(std::io::Error::other(err)),
|
||||
}
|
||||
}
|
||||
|
||||
async fn in_use(&self) -> Result<bool, std::io::Error> {
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -210,40 +210,6 @@ pub(super) fn require_restore_operation_id(metadata: &HashMap<String, String>, e
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn restore_commit_operation_id_from_metadata(metadata: &HashMap<String, String>) -> Result<Option<Uuid>> {
|
||||
if !metadata.contains_key(X_AMZ_RESTORE.as_str()) {
|
||||
return Ok(None);
|
||||
}
|
||||
restore_operation_id_from_metadata(metadata)
|
||||
}
|
||||
|
||||
impl SetDisks {
|
||||
pub(super) async fn require_current_restore_operation_id(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
opts: &ObjectOptions,
|
||||
expected: Option<Uuid>,
|
||||
mode: &str,
|
||||
) -> Result<()> {
|
||||
let Some(expected) = expected else {
|
||||
return Ok(());
|
||||
};
|
||||
let read_opts = ObjectOptions {
|
||||
version_id: opts.version_id.clone(),
|
||||
versioned: opts.versioned,
|
||||
version_suspended: opts.version_suspended,
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
};
|
||||
let (current, _, _) = self.get_object_fileinfo(bucket, object, &read_opts, true, false).await?;
|
||||
restore_operation_id_from_metadata(¤t.metadata)?
|
||||
.filter(|actual| *actual == expected)
|
||||
.ok_or_else(|| Error::other(format!("restore operation id changed before {mode}: expected {expected}")))?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
type ListObjectsV2Info = StorageListObjectsV2Info<ObjectInfo>;
|
||||
type ListObjectVersionsInfo = StorageListObjectVersionsInfo<ObjectInfo>;
|
||||
type ObjectInfoOrErr = StorageObjectInfoOrErr<ObjectInfo, Error>;
|
||||
@@ -687,8 +653,6 @@ mod core;
|
||||
mod ctx;
|
||||
mod metadata;
|
||||
mod ops;
|
||||
#[cfg(feature = "test-util")]
|
||||
pub(crate) use ops::object::TransitionCleanupStoreBarrier as SetDiskTransitionCleanupStoreBarrier;
|
||||
pub(crate) use ops::object::body_cache_plaintext_len;
|
||||
mod read;
|
||||
mod replication;
|
||||
|
||||
@@ -1038,20 +1038,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
}
|
||||
}
|
||||
|
||||
let expected_restore_operation_id = restore_commit_operation_id_from_metadata(&opts.user_defined)?;
|
||||
let (mut fi, files_metas) = self.check_upload_id_exists(bucket, object, upload_id, true).await?;
|
||||
if expected_restore_operation_id.is_some() {
|
||||
rustfs_utils::http::metadata_compat::remove_str(&mut fi.metadata, SUFFIX_RESTORE_OPERATION_ID);
|
||||
}
|
||||
if opts.versioned {
|
||||
fi.version_id = Some(
|
||||
fi.version_id
|
||||
.filter(|version_id| !version_id.is_nil())
|
||||
.unwrap_or_else(Uuid::new_v4),
|
||||
);
|
||||
} else {
|
||||
fi.version_id = None;
|
||||
}
|
||||
let upload_id_path = Self::get_upload_id_dir(bucket, object, upload_id);
|
||||
|
||||
let write_quorum = fi.write_quorum(self.default_write_quorum());
|
||||
@@ -1389,7 +1376,6 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
meta.parts.clone_from(&fi.parts);
|
||||
meta.metadata = fi.metadata.clone();
|
||||
meta.versioned = opts.versioned || opts.version_suspended;
|
||||
meta.version_id = fi.version_id;
|
||||
meta.checksum = fi.checksum.clone();
|
||||
}
|
||||
}
|
||||
@@ -1435,15 +1421,6 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
});
|
||||
}
|
||||
|
||||
self.require_current_restore_operation_id(
|
||||
bucket,
|
||||
object,
|
||||
opts,
|
||||
expected_restore_operation_id,
|
||||
"complete_multipart_upload_commit",
|
||||
)
|
||||
.await?;
|
||||
|
||||
let complete_tail_stage_start = rustfs_io_metrics::put_stage_metrics_enabled().then(Instant::now);
|
||||
|
||||
// Crash-consistency injection: hard power loss after the upload is fully
|
||||
@@ -1639,7 +1616,7 @@ mod tests {
|
||||
#[serial(metadata_cache_invalidation_probe)]
|
||||
async fn complete_multipart_generation_retires_cached_snapshot() {
|
||||
use crate::storage_api_contracts::multipart::MultipartOperations as _;
|
||||
use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _};
|
||||
use crate::storage_api_contracts::object::ObjectIO as _;
|
||||
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "multipart-metadata-generation-bucket";
|
||||
@@ -2368,15 +2345,14 @@ mod tests {
|
||||
/// Stage a single-part multipart upload without completing it. A single
|
||||
/// part is the last part, so the 5 MiB minimum-part-size gate does not
|
||||
/// apply. Returns the upload id and the `CompletePart` list.
|
||||
async fn stage_upload_with_create_opts(
|
||||
async fn stage_upload(
|
||||
set_disks: &Arc<SetDisks>,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
content: &[u8],
|
||||
create_opts: &ObjectOptions,
|
||||
) -> (String, Vec<CompletePart>) {
|
||||
let upload = set_disks
|
||||
.new_multipart_upload(bucket, object, create_opts)
|
||||
.new_multipart_upload(bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("multipart upload should be created");
|
||||
let mut reader = PutObjReader::new(
|
||||
@@ -2404,43 +2380,6 @@ mod tests {
|
||||
)
|
||||
}
|
||||
|
||||
async fn stage_upload(
|
||||
set_disks: &Arc<SetDisks>,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
content: &[u8],
|
||||
) -> (String, Vec<CompletePart>) {
|
||||
stage_upload_with_create_opts(set_disks, bucket, object, content, &ObjectOptions::default()).await
|
||||
}
|
||||
|
||||
async fn rewrite_staged_upload_version_id(
|
||||
set_disks: &Arc<SetDisks>,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
upload_id: &str,
|
||||
version_id: Option<Uuid>,
|
||||
) {
|
||||
let upload_id_path = SetDisks::get_upload_id_dir(bucket, object, upload_id);
|
||||
let disks = set_disks.disks.read().await.clone();
|
||||
let (fi, mut files_metas) = set_disks
|
||||
.check_upload_id_exists(bucket, object, upload_id, true)
|
||||
.await
|
||||
.expect("staged upload metadata should be readable");
|
||||
for meta in files_metas.iter_mut() {
|
||||
meta.version_id = version_id;
|
||||
}
|
||||
SetDisks::write_unique_file_info(
|
||||
&disks,
|
||||
bucket,
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
&upload_id_path,
|
||||
&files_metas,
|
||||
fi.write_quorum(set_disks.default_write_quorum()),
|
||||
)
|
||||
.await
|
||||
.expect("staged upload metadata should be rewritten");
|
||||
}
|
||||
|
||||
async fn complete(
|
||||
set_disks: &Arc<SetDisks>,
|
||||
bucket: &str,
|
||||
@@ -2454,118 +2393,6 @@ mod tests {
|
||||
.await
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn complete_multipart_upload_assigns_completion_version_id() {
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "multipart-versioned-complete-bucket";
|
||||
let object = "object";
|
||||
make_bucket_on_all(&disk_stores, bucket).await;
|
||||
|
||||
let (upload_id, parts) = stage_upload(&set_disks, bucket, object, b"versioned multipart body").await;
|
||||
let complete_opts = ObjectOptions {
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let completed = set_disks
|
||||
.clone()
|
||||
.complete_multipart_upload(bucket, object, &upload_id, parts, &complete_opts)
|
||||
.await
|
||||
.expect("versioned multipart completion should succeed");
|
||||
let version_id = completed.version_id.expect("versioned completion must return a version id");
|
||||
|
||||
assert!(!version_id.is_nil(), "versioned completion must not use a null version id");
|
||||
|
||||
let lookup_opts = ObjectOptions {
|
||||
versioned: true,
|
||||
version_id: Some(version_id.to_string()),
|
||||
..Default::default()
|
||||
};
|
||||
let current = set_disks
|
||||
.get_object_info(bucket, object, &lookup_opts)
|
||||
.await
|
||||
.expect("completed version should be addressable by exact version id");
|
||||
|
||||
assert_eq!(current.version_id, Some(version_id));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn complete_multipart_upload_replaces_staged_nil_version_id() {
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "multipart-nil-versioned-complete-bucket";
|
||||
let object = "object";
|
||||
make_bucket_on_all(&disk_stores, bucket).await;
|
||||
|
||||
let (upload_id, parts) = stage_upload(&set_disks, bucket, object, b"versioned multipart body").await;
|
||||
rewrite_staged_upload_version_id(&set_disks, bucket, object, &upload_id, Some(Uuid::nil())).await;
|
||||
let complete_opts = ObjectOptions {
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let completed = set_disks
|
||||
.clone()
|
||||
.complete_multipart_upload(bucket, object, &upload_id, parts, &complete_opts)
|
||||
.await
|
||||
.expect("versioned multipart completion should succeed");
|
||||
let version_id = completed.version_id.expect("versioned completion must return a version id");
|
||||
|
||||
assert!(!version_id.is_nil(), "versioned completion must replace a staged null version id");
|
||||
|
||||
let lookup_opts = ObjectOptions {
|
||||
versioned: true,
|
||||
version_id: Some(version_id.to_string()),
|
||||
..Default::default()
|
||||
};
|
||||
let current = set_disks
|
||||
.get_object_info(bucket, object, &lookup_opts)
|
||||
.await
|
||||
.expect("completed version should be addressable by exact version id");
|
||||
|
||||
assert_eq!(current.version_id, Some(version_id));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn complete_multipart_upload_suspended_clears_staged_version_id() {
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "multipart-suspended-complete-bucket";
|
||||
let object = "object";
|
||||
make_bucket_on_all(&disk_stores, bucket).await;
|
||||
let create_opts = ObjectOptions {
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let (upload_id, parts) =
|
||||
stage_upload_with_create_opts(&set_disks, bucket, object, b"suspended multipart body", &create_opts).await;
|
||||
let (staged_fi, _) = set_disks
|
||||
.check_upload_id_exists(bucket, object, &upload_id, true)
|
||||
.await
|
||||
.expect("staged upload metadata should be readable");
|
||||
assert!(
|
||||
staged_fi.version_id.is_some_and(|version_id| !version_id.is_nil()),
|
||||
"enabled multipart create should stage a concrete version id"
|
||||
);
|
||||
let complete_opts = ObjectOptions {
|
||||
version_suspended: true,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let completed = set_disks
|
||||
.clone()
|
||||
.complete_multipart_upload(bucket, object, &upload_id, parts, &complete_opts)
|
||||
.await
|
||||
.expect("suspended multipart completion should succeed");
|
||||
|
||||
assert!(
|
||||
completed.version_id.is_some_and(|version_id| version_id.is_nil()),
|
||||
"suspended completion must publish a null-version object internally"
|
||||
);
|
||||
}
|
||||
|
||||
/// Read the whole committed object back; returns `(body, etag)`. The full
|
||||
/// body read also proves the served length matches (never a torn/short body).
|
||||
async fn read_object(set_disks: &Arc<SetDisks>, bucket: &str, object: &str) -> (Vec<u8>, Option<String>) {
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -127,7 +127,4 @@ pub(crate) mod topology {
|
||||
CapabilityStatus, DiskCapabilities, TopologyCapabilities, TopologyDisk, TopologyLabels, TopologyPool, TopologySet,
|
||||
TopologySnapshot,
|
||||
};
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_storage_api::CapabilityState;
|
||||
}
|
||||
|
||||
@@ -40,16 +40,16 @@ fn validate_table_bucket_delete_allowed(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn table_catalog_metadata_exists(ctx: &crate::runtime::instance::InstanceContext, bucket: &str) -> Result<bool> {
|
||||
async fn table_catalog_metadata_exists(ctx: &crate::runtime::instance::InstanceContext, bucket: &str) -> bool {
|
||||
let local_disks = runtime_sources::local_disks_in(ctx).await;
|
||||
for disk in local_disks.iter() {
|
||||
let catalog_path = disk.path().join(bucket).join(BUCKET_TABLE_RESERVED_PREFIX);
|
||||
if has_xlmeta_files(&catalog_path).await? {
|
||||
return Ok(true);
|
||||
if has_xlmeta_files(&catalog_path).await {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(false)
|
||||
false
|
||||
}
|
||||
|
||||
async fn validate_table_bucket_delete_guard(ctx: &crate::runtime::instance::InstanceContext, bucket: &str) -> Result<()> {
|
||||
@@ -57,7 +57,7 @@ async fn validate_table_bucket_delete_guard(ctx: &crate::runtime::instance::Inst
|
||||
.await
|
||||
.is_ok_and(|metadata| metadata.table_bucket_enabled());
|
||||
if table_bucket_enabled {
|
||||
validate_table_bucket_delete_allowed(bucket, true, table_catalog_metadata_exists(ctx, bucket).await?)?;
|
||||
validate_table_bucket_delete_allowed(bucket, true, table_catalog_metadata_exists(ctx, bucket).await)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
@@ -257,57 +257,49 @@ impl ECStore {
|
||||
None
|
||||
};
|
||||
|
||||
let sr_mark_delete = opts.srdelete_op == SRBucketDeleteOp::MarkDelete;
|
||||
let sr_purge = opts.srdelete_op == SRBucketDeleteOp::Purge;
|
||||
let sr_delete = sr_mark_delete || sr_purge;
|
||||
let mut delete_opts = opts.clone();
|
||||
let bucket_exists = match self.peer_sys.get_bucket_info(bucket, &BucketOptions::default()).await {
|
||||
Ok(_) => true,
|
||||
Err(err) => {
|
||||
let storage_err: StorageError = err.into();
|
||||
if is_err_strict_volume_not_found(&storage_err) && sr_delete {
|
||||
false
|
||||
} else if is_err_strict_volume_not_found(&storage_err) {
|
||||
return Err(StorageError::BucketNotFound(bucket.to_string()));
|
||||
} else {
|
||||
return Err(to_object_err(storage_err, vec![bucket]));
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
if bucket_exists {
|
||||
validate_table_bucket_delete_guard(&self.ctx, bucket).await?;
|
||||
|
||||
// Check bucket is empty before deletion (per S3 API spec)
|
||||
// If bucket is not empty (contains actual objects with xl.meta files) and force
|
||||
// is not set, return BucketNotEmpty error.
|
||||
// Note: Empty directories (left after object deletion) should NOT count as objects.
|
||||
if !opts.force {
|
||||
let local_disks = runtime_sources::local_disks_in(&self.ctx).await;
|
||||
for disk in local_disks.iter() {
|
||||
let bucket_path = disk.path().join(bucket);
|
||||
if has_xlmeta_files(&bucket_path).await? {
|
||||
return Err(StorageError::BucketNotEmpty(bucket.to_string()));
|
||||
}
|
||||
}
|
||||
delete_opts.force_if_empty = true;
|
||||
// Check bucket exists before deletion (per S3 API spec)
|
||||
// If bucket doesn't exist, return NoSuchBucket error
|
||||
if let Err(err) = self.peer_sys.get_bucket_info(bucket, &BucketOptions::default()).await {
|
||||
// Convert DiskError to StorageError for comparison
|
||||
let storage_err: StorageError = err.into();
|
||||
if is_err_bucket_not_found(&storage_err) {
|
||||
return Err(StorageError::BucketNotFound(bucket.to_string()));
|
||||
}
|
||||
return Err(to_object_err(storage_err, vec![bucket]));
|
||||
}
|
||||
|
||||
if sr_delete && !bucket_exists {
|
||||
delete_opts.force_if_empty = true;
|
||||
validate_table_bucket_delete_guard(&self.ctx, bucket).await?;
|
||||
|
||||
let sr_mark_delete = opts.srdelete_op == SRBucketDeleteOp::MarkDelete;
|
||||
let sr_purge = opts.srdelete_op == SRBucketDeleteOp::Purge;
|
||||
|
||||
// Check bucket is empty before deletion (per S3 API spec)
|
||||
// If bucket is not empty (contains actual objects with xl.meta files) and force
|
||||
// is not set, return BucketNotEmpty error.
|
||||
// Note: Empty directories (left after object deletion) should NOT count as objects.
|
||||
if !opts.force && !sr_mark_delete {
|
||||
let local_disks = runtime_sources::local_disks_in(&self.ctx).await;
|
||||
for disk in local_disks.iter() {
|
||||
// Check if bucket directory contains any xl.meta files (actual objects)
|
||||
// We recursively scan for xl.meta files to determine if bucket has objects
|
||||
// Use the disk's root path to construct bucket path
|
||||
let bucket_path = disk.path().join(bucket);
|
||||
if has_xlmeta_files(&bucket_path).await {
|
||||
return Err(StorageError::BucketNotEmpty(bucket.to_string()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if sr_mark_delete {
|
||||
self.mark_bucket_deleted(bucket).await?;
|
||||
self.cleanup_deleted_bucket_metadata(bucket, false).await?;
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
if let Err(err) = self.peer_sys.delete_bucket(bucket, &delete_opts).await {
|
||||
let storage_err = to_object_err(err.into(), vec![bucket]);
|
||||
if !sr_delete || !is_err_strict_volume_not_found(&storage_err) {
|
||||
return Err(storage_err);
|
||||
}
|
||||
}
|
||||
self.peer_sys
|
||||
.delete_bucket(bucket, opts)
|
||||
.await
|
||||
.map_err(|e| to_object_err(e.into(), vec![bucket]))?;
|
||||
|
||||
self.cleanup_deleted_bucket_metadata(bucket, sr_purge).await?;
|
||||
Ok(())
|
||||
@@ -327,7 +319,7 @@ mod tests {
|
||||
use crate::object_api::{ObjectOptions, PutObjReader};
|
||||
use crate::runtime::instance::InstanceContext;
|
||||
use crate::storage_api_contracts::{
|
||||
bucket::{BucketOperations as _, BucketOptions, DeleteBucketOptions, MakeBucketOptions, SRBucketDeleteOp},
|
||||
bucket::{BucketOperations as _, DeleteBucketOptions, MakeBucketOptions, SRBucketDeleteOp},
|
||||
object::{ObjectIO as _, ObjectOperations as _},
|
||||
};
|
||||
use crate::store::{ECStore, init_local_disks_with_instance_ctx};
|
||||
@@ -456,10 +448,7 @@ mod tests {
|
||||
|
||||
async fn any_disk_has_object_metadata(disk_paths: &[PathBuf], bucket: &str) -> bool {
|
||||
for disk_path in disk_paths {
|
||||
if super::has_xlmeta_files(&disk_path.join(bucket))
|
||||
.await
|
||||
.expect("object metadata scan should succeed")
|
||||
{
|
||||
if super::has_xlmeta_files(&disk_path.join(bucket)).await {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@@ -519,19 +508,12 @@ mod tests {
|
||||
// serialize them so their assertions cannot observe each other's operations.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn bucket_delete_mark_delete_removes_empty_bucket_and_keeps_deleted_marker() {
|
||||
async fn bucket_delete_mark_delete_marks_metadata_deleted_without_physical_object_delete() {
|
||||
let (disk_paths, ecstore) = setup_bucket_delete_test_env().await;
|
||||
let bucket = format!("bucket-mark-delete-{}", Uuid::new_v4().simple());
|
||||
let object = "object.txt";
|
||||
|
||||
ecstore
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket should be created");
|
||||
for disk_path in &disk_paths {
|
||||
tokio::fs::create_dir_all(disk_path.join(&bucket).join("empty-directory"))
|
||||
.await
|
||||
.expect("empty directory remnant should be created");
|
||||
}
|
||||
create_bucket_with_object(&ecstore, &bucket, object).await;
|
||||
assert!(metadata_sys::get_in(&ecstore.ctx, &bucket).await.is_ok());
|
||||
|
||||
let generation_before_delete = ecstore.scanner_namespace_mutation_generation();
|
||||
@@ -544,7 +526,7 @@ mod tests {
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("MarkDelete should remove an empty bucket");
|
||||
.expect("MarkDelete should not reject non-empty bucket data");
|
||||
assert_eq!(
|
||||
ecstore.scanner_namespace_mutation_generation(),
|
||||
generation_before_delete.saturating_add(1),
|
||||
@@ -552,8 +534,8 @@ mod tests {
|
||||
);
|
||||
|
||||
assert!(
|
||||
!any_disk_path_exists(&disk_paths, &bucket).await,
|
||||
"MarkDelete should remove the bucket volume"
|
||||
any_disk_has_object_metadata(&disk_paths, &bucket).await,
|
||||
"MarkDelete must not physically remove object xl.meta data"
|
||||
);
|
||||
assert!(
|
||||
any_disk_path_exists(&disk_paths, bucket_deleted_marker_volume(&bucket)).await,
|
||||
@@ -563,79 +545,6 @@ mod tests {
|
||||
metadata_sys::get_in(&ecstore.ctx, &bucket).await.is_err(),
|
||||
"deleted bucket metadata must be removed from the local cache"
|
||||
);
|
||||
let buckets = ecstore
|
||||
.list_bucket(&BucketOptions::default())
|
||||
.await
|
||||
.expect("bucket listing should succeed after MarkDelete");
|
||||
assert!(!buckets.iter().any(|info| info.name == bucket));
|
||||
ecstore
|
||||
.delete_all(RUSTFS_META_BUCKET, &bucket_deleted_marker_prefix(&bucket))
|
||||
.await
|
||||
.expect("deleted-bucket marker should be removed to simulate a partial failure");
|
||||
assert!(!any_disk_path_exists(&disk_paths, bucket_deleted_marker_volume(&bucket)).await);
|
||||
ecstore
|
||||
.delete_bucket(
|
||||
&bucket,
|
||||
&DeleteBucketOptions {
|
||||
srdelete_op: SRBucketDeleteOp::MarkDelete,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("retried MarkDelete should recreate a missing tombstone");
|
||||
assert!(any_disk_path_exists(&disk_paths, bucket_deleted_marker_volume(&bucket)).await);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn bucket_delete_mark_delete_rejects_non_empty_bucket_without_force() {
|
||||
let (disk_paths, ecstore) = setup_bucket_delete_test_env().await;
|
||||
let bucket = format!("bucket-mark-delete-non-empty-{}", Uuid::new_v4().simple());
|
||||
|
||||
create_bucket_with_object(&ecstore, &bucket, "object.txt").await;
|
||||
|
||||
let err = ecstore
|
||||
.delete_bucket(
|
||||
&bucket,
|
||||
&DeleteBucketOptions {
|
||||
srdelete_op: SRBucketDeleteOp::MarkDelete,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect_err("MarkDelete should reject a non-empty bucket without force");
|
||||
|
||||
assert!(matches!(err, StorageError::BucketNotEmpty(name) if name == bucket));
|
||||
assert!(any_disk_has_object_metadata(&disk_paths, &bucket).await);
|
||||
assert!(!any_disk_path_exists(&disk_paths, bucket_deleted_marker_volume(&bucket)).await);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn bucket_delete_mark_delete_rejects_hidden_object_paths_without_force() {
|
||||
let (disk_paths, ecstore) = setup_bucket_delete_test_env().await;
|
||||
let bucket = format!("bucket-mark-delete-hidden-{}", Uuid::new_v4().simple());
|
||||
|
||||
create_bucket_with_object(&ecstore, &bucket, ".well-known/acme-challenge").await;
|
||||
let mut reader = PutObjReader::from_vec(b"delete bucket semantics".to_vec());
|
||||
ecstore
|
||||
.put_object(&bucket, ".rustfs.sys/object", &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("second hidden object should be written");
|
||||
|
||||
let err = ecstore
|
||||
.delete_bucket(
|
||||
&bucket,
|
||||
&DeleteBucketOptions {
|
||||
srdelete_op: SRBucketDeleteOp::MarkDelete,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect_err("MarkDelete should reject a hidden object path without force");
|
||||
|
||||
assert!(matches!(err, StorageError::BucketNotEmpty(name) if name == bucket));
|
||||
assert!(any_disk_has_object_metadata(&disk_paths, &bucket).await);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -648,12 +557,7 @@ mod tests {
|
||||
|
||||
create_bucket_with_object(&ecstore, &bucket, object).await;
|
||||
write_bucket_metadata_marker(&disk_paths, &metadata_prefix).await;
|
||||
ecstore
|
||||
.mark_bucket_deleted(&bucket)
|
||||
.await
|
||||
.expect("deleted-bucket marker should be created");
|
||||
assert!(any_disk_path_exists(&disk_paths, &metadata_prefix).await);
|
||||
assert!(any_disk_path_exists(&disk_paths, bucket_deleted_marker_volume(&bucket)).await);
|
||||
|
||||
let generation_before_delete = ecstore.scanner_namespace_mutation_generation();
|
||||
ecstore
|
||||
@@ -678,32 +582,10 @@ mod tests {
|
||||
!any_disk_path_exists(&disk_paths, &metadata_prefix).await,
|
||||
"Purge should remove bucket metadata prefix"
|
||||
);
|
||||
assert!(
|
||||
!any_disk_path_exists(&disk_paths, bucket_deleted_marker_volume(&bucket)).await,
|
||||
"Purge should remove the deleted-bucket marker"
|
||||
);
|
||||
assert!(
|
||||
metadata_sys::get_in(&ecstore.ctx, &bucket).await.is_err(),
|
||||
"purged bucket metadata must be removed from the local cache"
|
||||
);
|
||||
write_bucket_metadata_marker(&disk_paths, &metadata_prefix).await;
|
||||
ecstore
|
||||
.mark_bucket_deleted(&bucket)
|
||||
.await
|
||||
.expect("stale deleted-bucket marker should be recreated");
|
||||
ecstore
|
||||
.delete_bucket(
|
||||
&bucket,
|
||||
&DeleteBucketOptions {
|
||||
force: true,
|
||||
srdelete_op: SRBucketDeleteOp::Purge,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("retried Purge should remove stale metadata without a bucket volume");
|
||||
assert!(!any_disk_path_exists(&disk_paths, &metadata_prefix).await);
|
||||
assert!(!any_disk_path_exists(&disk_paths, bucket_deleted_marker_volume(&bucket)).await);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -81,11 +81,6 @@ type ListObjectsV2Info = StorageListObjectsV2Info<ObjectInfo>;
|
||||
type ListObjectVersionsInfo = StorageListObjectVersionsInfo<ObjectInfo>;
|
||||
type ObjectInfoOrErr = StorageObjectInfoOrErr<ObjectInfo, Error>;
|
||||
type WalkOptions = StorageWalkOptions<fn(&rustfs_filemeta::FileInfo) -> bool>;
|
||||
const LIST_MERGED_INPUT_BUFFER: usize = 1;
|
||||
|
||||
fn list_merged_entry_channel() -> (Sender<MetaCacheEntry>, Receiver<MetaCacheEntry>) {
|
||||
mpsc::channel(LIST_MERGED_INPUT_BUFFER)
|
||||
}
|
||||
|
||||
fn normalize_max_keys(max_keys: i32) -> i32 {
|
||||
max_keys.min(MAX_OBJECT_LIST)
|
||||
@@ -4173,7 +4168,7 @@ impl ECStore {
|
||||
|
||||
for sets in self.pools.iter() {
|
||||
for set in sets.disk_set.iter() {
|
||||
let (send, recv) = list_merged_entry_channel();
|
||||
let (send, recv) = mpsc::channel(100);
|
||||
|
||||
inputs.push(recv);
|
||||
let opts = opts.clone();
|
||||
@@ -5428,7 +5423,7 @@ impl Sets {
|
||||
let mut inputs = Vec::new();
|
||||
|
||||
for set in &self.disk_set {
|
||||
let (send, recv) = list_merged_entry_channel();
|
||||
let (send, recv) = mpsc::channel(100);
|
||||
inputs.push(recv);
|
||||
let opts = opts.clone();
|
||||
let rx_clone = rx.clone();
|
||||
@@ -6692,14 +6687,14 @@ mod test {
|
||||
VerifiedIndexCandidateStats, VersionMarker, current_list_objects_mutation_sequence,
|
||||
encode_persistent_list_metadata_object, enforce_latest_listing_write_quorum, expand_ask_disks_for_object_quorum,
|
||||
fallback_entries_for_object, gather_results, latest_listing_allow_agreed_objects, latest_listing_object_quorum,
|
||||
latest_listing_raw_min_disks, latest_listing_required_object_quorum, list_marker_key, list_merged_entry_channel,
|
||||
list_metadata_resolution_params, list_objects_from_metadata_snapshot_candidates,
|
||||
list_objects_from_verified_index_candidates, list_objects_from_verified_index_candidates_with_optional_stats,
|
||||
list_objects_from_verified_index_candidates_with_stats, list_objects_index_mode_from_env,
|
||||
list_objects_index_provider_from_env, list_objects_index_provider_state_from_env, list_objects_key_only_provider_health,
|
||||
list_objects_metadata_fast_guardrails_from_env, list_objects_paginate, list_objects_quorum_from_env,
|
||||
list_quorum_from_env, load_namespace_mutation_journal_state, load_persistent_key_only_index, max_keys_plus_one,
|
||||
merge_entry_channels, namespace_mutation_journal_chaos_bucket_from_env, namespace_mutation_journal_chaos_config_from_env,
|
||||
latest_listing_raw_min_disks, latest_listing_required_object_quorum, list_marker_key, list_metadata_resolution_params,
|
||||
list_objects_from_metadata_snapshot_candidates, list_objects_from_verified_index_candidates,
|
||||
list_objects_from_verified_index_candidates_with_optional_stats, list_objects_from_verified_index_candidates_with_stats,
|
||||
list_objects_index_mode_from_env, list_objects_index_provider_from_env, list_objects_index_provider_state_from_env,
|
||||
list_objects_key_only_provider_health, list_objects_metadata_fast_guardrails_from_env, list_objects_paginate,
|
||||
list_objects_quorum_from_env, list_quorum_from_env, load_namespace_mutation_journal_state,
|
||||
load_persistent_key_only_index, max_keys_plus_one, merge_entry_channels,
|
||||
namespace_mutation_journal_chaos_bucket_from_env, namespace_mutation_journal_chaos_config_from_env,
|
||||
namespace_mutation_journal_chaos_enabled_from_env, namespace_mutation_journal_chaos_sequence_from_env,
|
||||
namespace_mutation_journal_chaos_status_from_env, normalize_list_quorum, observe_list_objects_mutations_with_store,
|
||||
parse_namespace_mutation_journal_state, parse_persistent_key_only_index, parse_persistent_list_metadata_object,
|
||||
@@ -6843,22 +6838,6 @@ mod test {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn list_merged_entry_channel_backpressures_after_one_head() {
|
||||
let (tx, mut rx) = list_merged_entry_channel();
|
||||
tx.try_send(test_meta_entry("object-a"))
|
||||
.expect("first merge head should fit in the input buffer");
|
||||
|
||||
match tx.try_send(test_meta_entry("object-b")) {
|
||||
Err(tokio::sync::mpsc::error::TrySendError::Full(entry)) => assert_eq!(entry.name, "object-b"),
|
||||
other => panic!("second merge head should wait for the k-way merge refill, got {other:?}"),
|
||||
}
|
||||
|
||||
assert_eq!(rx.recv().await.expect("first merge head should remain available").name, "object-a");
|
||||
tx.try_send(test_meta_entry("object-b"))
|
||||
.expect("input buffer should accept the next head after refill");
|
||||
}
|
||||
|
||||
fn test_live_object_info(name: &str, etag: &str) -> ObjectInfo {
|
||||
let mut fi = FileInfo {
|
||||
name: name.to_owned(),
|
||||
|
||||
@@ -40,8 +40,8 @@ use crate::disk::endpoint::{Endpoint, EndpointType};
|
||||
use crate::disk::{DiskAPI, DiskInfo, DiskInfoOptions};
|
||||
use crate::error::{Error, Result};
|
||||
use crate::error::{
|
||||
StorageError, is_err_bucket_exists, is_err_invalid_upload_id, is_err_object_not_found, is_err_read_quorum,
|
||||
is_err_strict_volume_not_found, is_err_version_not_found, to_object_err,
|
||||
StorageError, is_err_bucket_exists, is_err_bucket_not_found, is_err_invalid_upload_id, is_err_object_not_found,
|
||||
is_err_read_quorum, is_err_version_not_found, to_object_err,
|
||||
};
|
||||
use crate::runtime::global::DISK_RESERVE_FRACTION;
|
||||
use crate::runtime::instance::InstanceContext;
|
||||
@@ -91,7 +91,7 @@ type WalkOptions = StorageWalkOptions<fn(&FileInfo) -> bool>;
|
||||
|
||||
/// Check if a directory contains any xl.meta files (indicating actual S3 objects)
|
||||
/// This is used to determine if a bucket is empty for deletion purposes.
|
||||
pub(crate) async fn has_xlmeta_files(path: &std::path::Path) -> std::io::Result<bool> {
|
||||
async fn has_xlmeta_files(path: &std::path::Path) -> bool {
|
||||
use crate::disk::STORAGE_FORMAT_FILE;
|
||||
use tokio::fs;
|
||||
|
||||
@@ -100,27 +100,33 @@ pub(crate) async fn has_xlmeta_files(path: &std::path::Path) -> std::io::Result<
|
||||
while let Some(current_path) = stack.pop() {
|
||||
let mut entries = match fs::read_dir(¤t_path).await {
|
||||
Ok(entries) => entries,
|
||||
Err(err) if err.kind() == std::io::ErrorKind::NotFound => continue,
|
||||
Err(err) => return Err(err),
|
||||
Err(_) => continue,
|
||||
};
|
||||
|
||||
while let Some(entry) = entries.next_entry().await? {
|
||||
while let Ok(Some(entry)) = entries.next_entry().await {
|
||||
let file_name = entry.file_name();
|
||||
let file_name_str = file_name.to_string_lossy();
|
||||
|
||||
// Skip hidden files/directories (like .rustfs.sys)
|
||||
if file_name_str.starts_with('.') {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Check if this is an xl.meta file
|
||||
if file_name_str == STORAGE_FORMAT_FILE {
|
||||
return Ok(true);
|
||||
return true;
|
||||
}
|
||||
|
||||
// If it's a directory, add to stack for further exploration
|
||||
if entry.file_type().await?.is_dir() {
|
||||
if let Ok(file_type) = entry.file_type().await
|
||||
&& file_type.is_dir()
|
||||
{
|
||||
stack.push(entry.path());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(false)
|
||||
false
|
||||
}
|
||||
|
||||
async fn enqueue_transition_after_write(result: Result<ObjectInfo>, src: LcEventSrc) -> Result<ObjectInfo> {
|
||||
|
||||
@@ -86,19 +86,4 @@ impl CapturingRecorder {
|
||||
.map(|(_, samples)| samples.0.lock().expect("samples should be lockable").len())
|
||||
.sum()
|
||||
}
|
||||
|
||||
pub(crate) fn histogram_values(&self, name: &str, labels: &[(&str, &str)]) -> Vec<f64> {
|
||||
self.histograms
|
||||
.lock()
|
||||
.expect("histogram map should be lockable")
|
||||
.iter()
|
||||
.filter(|(key, _)| {
|
||||
key.name() == name
|
||||
&& labels
|
||||
.iter()
|
||||
.all(|(lk, lv)| key.labels().any(|label| label.key() == *lk && label.value() == *lv))
|
||||
})
|
||||
.flat_map(|(_, samples)| samples.0.lock().expect("samples should be lockable").clone())
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -532,9 +532,6 @@ impl FileMeta {
|
||||
if ver.header.version_id != vid {
|
||||
continue;
|
||||
}
|
||||
if fi.tier_free_version() && !ver.header.free_version() {
|
||||
return Err(Error::FileVersionNotFound);
|
||||
}
|
||||
|
||||
match ver.header.version_type {
|
||||
VersionType::Invalid | VersionType::Legacy => return Err(Error::other("invalid file meta version")),
|
||||
|
||||
@@ -51,7 +51,7 @@ rustfs-utils = { workspace = true, features = ["path"] }
|
||||
rustfs-io-metrics.workspace = true
|
||||
tokio-util = { workspace = true, features = ["io", "compat"] }
|
||||
pollster.workspace = true
|
||||
reqwest = { workspace = true }
|
||||
reqwest = { workspace = true, default-features = false, features = ["rustls", "http2", "system-proxy"] }
|
||||
moka = { workspace = true, features = ["future"] }
|
||||
openidconnect = { workspace = true, default-features = false, features = ["accept-rfc3339-timestamps"] }
|
||||
http = { workspace = true }
|
||||
|
||||
@@ -90,10 +90,6 @@ pub enum Error {
|
||||
|
||||
#[error("invalid access_key")]
|
||||
InvalidAccessKey,
|
||||
|
||||
#[error("access key is already in use")]
|
||||
AccessKeyAlreadyExists,
|
||||
|
||||
#[error("action not allowed")]
|
||||
IAMActionNotAllowed,
|
||||
|
||||
@@ -163,7 +159,6 @@ impl Clone for Error {
|
||||
Error::NoAccessKey => Error::NoAccessKey,
|
||||
Error::InvalidToken => Error::InvalidToken,
|
||||
Error::InvalidAccessKey => Error::InvalidAccessKey,
|
||||
Error::AccessKeyAlreadyExists => Error::AccessKeyAlreadyExists,
|
||||
Error::IAMActionNotAllowed => Error::IAMActionNotAllowed,
|
||||
Error::InvalidExpiration => Error::InvalidExpiration,
|
||||
Error::NoSecretKeyWithAccessKey => Error::NoSecretKeyWithAccessKey,
|
||||
@@ -302,7 +297,6 @@ mod tests {
|
||||
Error::NoSuchUser("testuser".to_string()),
|
||||
Error::NoSuchAccount("testaccount".to_string()),
|
||||
Error::InvalidArgument,
|
||||
Error::AccessKeyAlreadyExists,
|
||||
Error::IAMActionNotAllowed,
|
||||
Error::PolicyTooLarge,
|
||||
Error::ConfigNotFound,
|
||||
@@ -421,7 +415,6 @@ mod tests {
|
||||
(Error::NoSuchUser("testuser".to_string()), "user 'testuser' does not exist"),
|
||||
(Error::NoSuchAccount("testaccount".to_string()), "account 'testaccount' does not exist"),
|
||||
(Error::InvalidArgument, "invalid arguments specified"),
|
||||
(Error::AccessKeyAlreadyExists, "access key is already in use"),
|
||||
(Error::IAMActionNotAllowed, "action not allowed"),
|
||||
(Error::ConfigNotFound, "config not found"),
|
||||
];
|
||||
|
||||
@@ -783,10 +783,14 @@ where
|
||||
}
|
||||
|
||||
let cache = self.cache.snapshot();
|
||||
if cache.users.contains_key(&cred.access_key) || cache.sts_accounts.contains_key(&cred.access_key) {
|
||||
return Err(Error::AccessKeyAlreadyExists);
|
||||
let users = Arc::clone(&cache.users);
|
||||
if let Some(x) = users.get(&cred.access_key)
|
||||
&& x.credentials.is_service_account()
|
||||
{
|
||||
return Err(Error::IAMActionNotAllowed);
|
||||
}
|
||||
drop(cache);
|
||||
drop(users);
|
||||
|
||||
let u = UserIdentity::new(cred);
|
||||
|
||||
@@ -1332,7 +1336,7 @@ where
|
||||
if let Some(x) = users.get(access_key) {
|
||||
warn!(error = ?x, "IAM user already exists");
|
||||
if x.credentials.is_temp() {
|
||||
return Err(Error::AccessKeyAlreadyExists);
|
||||
return Err(Error::IAMActionNotAllowed);
|
||||
}
|
||||
}
|
||||
drop(cache);
|
||||
|
||||
+2
-157
@@ -456,7 +456,7 @@ impl<T: Store> IamSys<T> {
|
||||
}
|
||||
|
||||
if parent_user == opts.access_key {
|
||||
return Err(IamError::AccessKeyAlreadyExists);
|
||||
return Err(IamError::IAMActionNotAllowed);
|
||||
}
|
||||
|
||||
if opts.access_key == SITE_REPLICATOR_SERVICE_ACCOUNT && !opts.allow_site_replicator_account {
|
||||
@@ -1429,7 +1429,6 @@ mod tests {
|
||||
/// When true, parent user has no groups and no mapped policies (empty `policy_db_get`).
|
||||
empty_policies: bool,
|
||||
saved_sts_users: Arc<Mutex<HashMap<String, UserIdentity>>>,
|
||||
saved_service_account_count: Arc<Mutex<usize>>,
|
||||
}
|
||||
|
||||
impl StsTestMockStore {
|
||||
@@ -1437,16 +1436,8 @@ mod tests {
|
||||
Self {
|
||||
empty_policies,
|
||||
saved_sts_users: Arc::new(Mutex::new(HashMap::new())),
|
||||
saved_service_account_count: Arc::new(Mutex::new(0)),
|
||||
}
|
||||
}
|
||||
|
||||
fn saved_service_account_count(&self) -> usize {
|
||||
*self
|
||||
.saved_service_account_count
|
||||
.lock()
|
||||
.expect("saved_service_account_count mutex poisoned")
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
@@ -1470,16 +1461,10 @@ mod tests {
|
||||
async fn save_user_identity(
|
||||
&self,
|
||||
name: &str,
|
||||
user_type: UserType,
|
||||
_user_type: UserType,
|
||||
item: UserIdentity,
|
||||
_ttl: Option<usize>,
|
||||
) -> Result<()> {
|
||||
if user_type == UserType::Svc {
|
||||
*self
|
||||
.saved_service_account_count
|
||||
.lock()
|
||||
.expect("saved_service_account_count mutex poisoned") += 1;
|
||||
}
|
||||
self.saved_sts_users
|
||||
.lock()
|
||||
.expect("saved_sts_users mutex poisoned")
|
||||
@@ -1701,146 +1686,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
async fn test_iam_sys() -> IamSys<StsTestMockStore> {
|
||||
let store = StsTestMockStore::new(false);
|
||||
let cache = IamCache::new(store).await.expect("IAM cache should initialize");
|
||||
IamSys::new(cache)
|
||||
}
|
||||
|
||||
fn service_account_opts(access_key: &str, secret_key: &str) -> NewServiceAccountOpts {
|
||||
NewServiceAccountOpts {
|
||||
access_key: access_key.to_string(),
|
||||
secret_key: secret_key.to_string(),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn new_service_account_rejects_parent_access_key() {
|
||||
ensure_test_global_credentials();
|
||||
|
||||
let iam_sys = test_iam_sys().await;
|
||||
let access_key = "PARENTACCESSKEY123";
|
||||
|
||||
let err = iam_sys
|
||||
.new_service_account(access_key, None, service_account_opts(access_key, "parentAccessKeySecret123"))
|
||||
.await
|
||||
.expect_err("a service account must not reuse its parent access key");
|
||||
|
||||
assert_eq!(err, Error::AccessKeyAlreadyExists);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn duplicate_service_account_returns_access_key_already_exists() {
|
||||
ensure_test_global_credentials();
|
||||
|
||||
let iam_sys = test_iam_sys().await;
|
||||
let access_key = "DUPLICATESERVICEKEY1";
|
||||
|
||||
iam_sys
|
||||
.new_service_account("svc-parent-user", None, service_account_opts(access_key, "firstServiceSecret123"))
|
||||
.await
|
||||
.expect("initial service account should be created");
|
||||
|
||||
let err = iam_sys
|
||||
.new_service_account("svc-parent-user", None, service_account_opts(access_key, "secondServiceSecret123"))
|
||||
.await
|
||||
.expect_err("duplicate service account should fail");
|
||||
|
||||
assert_eq!(err, Error::AccessKeyAlreadyExists);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn service_account_creation_rejects_regular_user_access_key() {
|
||||
ensure_test_global_credentials();
|
||||
|
||||
let iam_sys = test_iam_sys().await;
|
||||
let access_key = "REGULARUSERACCESSKEY1";
|
||||
let user = AddOrUpdateUserReq {
|
||||
secret_key: "regularUserSecret123".to_string(),
|
||||
policy: None,
|
||||
status: rustfs_madmin::AccountStatus::Enabled,
|
||||
};
|
||||
|
||||
iam_sys
|
||||
.store
|
||||
.add_user(access_key, &user)
|
||||
.await
|
||||
.expect("regular user should be created");
|
||||
|
||||
let err = iam_sys
|
||||
.new_service_account("svc-parent-user", None, service_account_opts(access_key, "serviceAccountSecret123"))
|
||||
.await
|
||||
.expect_err("service-account creation must reject a regular-user access key");
|
||||
|
||||
assert_eq!(err, Error::AccessKeyAlreadyExists);
|
||||
assert_eq!(iam_sys.store.api.saved_service_account_count(), 0);
|
||||
let existing = iam_sys.get_user(access_key).await.expect("regular user should remain cached");
|
||||
assert!(!existing.credentials.is_service_account());
|
||||
assert_eq!(existing.credentials.secret_key, user.secret_key);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn service_account_creation_rejects_sts_access_key() {
|
||||
ensure_test_global_credentials();
|
||||
|
||||
let iam_sys = test_iam_sys().await;
|
||||
let access_key = "TEMPORARYSERVICEKEY12";
|
||||
let temp_cred = Credentials {
|
||||
access_key: access_key.to_string(),
|
||||
secret_key: "temporarySecretKey123".to_string(),
|
||||
session_token: "temporary-session-token".to_string(),
|
||||
expiration: Some(OffsetDateTime::now_utc() + time::Duration::hours(1)),
|
||||
status: ACCOUNT_ON.to_string(),
|
||||
parent_user: "sts-parent-user".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
iam_sys
|
||||
.set_temp_user(access_key, &temp_cred, None)
|
||||
.await
|
||||
.expect("temporary credentials should be created");
|
||||
|
||||
let err = iam_sys
|
||||
.new_service_account("svc-parent-user", None, service_account_opts(access_key, "serviceAccountSecret123"))
|
||||
.await
|
||||
.expect_err("service-account creation must reject a temporary access key");
|
||||
|
||||
assert_eq!(err, Error::AccessKeyAlreadyExists);
|
||||
assert_eq!(iam_sys.store.api.saved_service_account_count(), 0);
|
||||
let existing = iam_sys
|
||||
.get_user(access_key)
|
||||
.await
|
||||
.expect("temporary account should remain cached");
|
||||
assert!(existing.credentials.is_temp());
|
||||
assert_eq!(existing.credentials.secret_key, temp_cred.secret_key);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn user_creation_rejects_service_account_access_key() {
|
||||
ensure_test_global_credentials();
|
||||
|
||||
let iam_sys = test_iam_sys().await;
|
||||
let access_key = "SERVICEACCOUNTKEY123";
|
||||
|
||||
iam_sys
|
||||
.new_service_account("svc-parent-user", None, service_account_opts(access_key, "serviceAccountSecret123"))
|
||||
.await
|
||||
.expect("service account should be created");
|
||||
|
||||
let user = AddOrUpdateUserReq {
|
||||
secret_key: "regularUserSecret123".to_string(),
|
||||
policy: None,
|
||||
status: rustfs_madmin::AccountStatus::Enabled,
|
||||
};
|
||||
let err = iam_sys
|
||||
.create_user(access_key, &user)
|
||||
.await
|
||||
.expect_err("user creation must reject a service-account access key");
|
||||
|
||||
assert_eq!(err, Error::AccessKeyAlreadyExists);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_new_service_account_without_expiration_omits_exp_claim() {
|
||||
ensure_test_global_credentials();
|
||||
|
||||
@@ -514,12 +514,6 @@ pub fn cluster_peer_is_offline(addr: &str) -> bool {
|
||||
peers.get(normalize_peer_key(addr)).map(|peer| !peer.online).unwrap_or(false)
|
||||
}
|
||||
|
||||
/// Return the last observed online/offline state for a peer, if this process has observed one.
|
||||
pub fn cluster_peer_observed_online_status(addr: &str) -> Option<bool> {
|
||||
let peers = CLUSTER_PEER_HEALTH.read().unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
peers.get(normalize_peer_key(addr)).map(|peer| peer.online)
|
||||
}
|
||||
|
||||
/// Decide whether to fast-fail (bypass) an offline peer instead of attempting to reach it
|
||||
/// (grpc-optimization P3 offline bypass). Returns `true` to bypass.
|
||||
///
|
||||
@@ -788,18 +782,6 @@ mod tests {
|
||||
assert!(!cluster_peer_is_offline(bare), "reachable via one form must mark the shared entry online");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cluster_peer_observed_online_status_reports_known_and_unknown_peers() {
|
||||
let addr = "http://cluster-peer-snapshot-status-test:9000";
|
||||
assert_eq!(cluster_peer_observed_online_status(addr), None);
|
||||
|
||||
record_peer_reachable(addr);
|
||||
assert_eq!(cluster_peer_observed_online_status(addr), Some(true));
|
||||
|
||||
record_peer_unreachable(addr, 1);
|
||||
assert_eq!(cluster_peer_observed_online_status(addr), Some(false));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cluster_servers_offline_total_name_is_stable() {
|
||||
assert_eq!(CLUSTER_SERVERS_OFFLINE_TOTAL, "rustfs_cluster_servers_offline_total");
|
||||
|
||||
@@ -210,12 +210,10 @@ pub use io_metrics::{
|
||||
record_io_scheduler_decision, record_load_level_change, record_queue_operation, record_starvation_event,
|
||||
};
|
||||
pub use list_objects_metrics::{
|
||||
LIST_OBJECTS_GATHER_OUTCOME_INPUT_CLOSED, LIST_OBJECTS_GATHER_OUTCOME_LIMIT_REACHED,
|
||||
LIST_OBJECTS_LOCAL_READ_DIR_OUTCOME_ERROR, LIST_OBJECTS_LOCAL_READ_DIR_OUTCOME_OK, LIST_OBJECTS_SOURCE_WALKER,
|
||||
ListObjectsGatherObservation, ListObjectsIndexPageObservation, ListObjectsLocalReadDirObservation, init_list_objects_metrics,
|
||||
record_list_objects_gather, record_list_objects_index_attempt, record_list_objects_index_fallback,
|
||||
record_list_objects_index_live_verify_failure, record_list_objects_index_served, record_list_objects_local_read_dir,
|
||||
record_list_objects_merge,
|
||||
LIST_OBJECTS_GATHER_OUTCOME_INPUT_CLOSED, LIST_OBJECTS_GATHER_OUTCOME_LIMIT_REACHED, LIST_OBJECTS_SOURCE_WALKER,
|
||||
ListObjectsGatherObservation, ListObjectsIndexPageObservation, init_list_objects_metrics, record_list_objects_gather,
|
||||
record_list_objects_index_attempt, record_list_objects_index_fallback, record_list_objects_index_live_verify_failure,
|
||||
record_list_objects_index_served, record_list_objects_merge,
|
||||
};
|
||||
|
||||
// Backpressure metrics exports
|
||||
|
||||
@@ -42,13 +42,6 @@ const LIST_OBJECTS_INDEX_LIVE_VERIFY_FAILURE_TOTAL: &str = "rustfs_s3_list_objec
|
||||
const LIST_OBJECTS_INDEX_RETURNED_OBJECTS: &str = "rustfs_s3_list_objects_index_returned_objects";
|
||||
const LIST_OBJECTS_INDEX_RETURNED_PREFIXES: &str = "rustfs_s3_list_objects_index_returned_prefixes";
|
||||
const LIST_OBJECTS_INDEX_VERIFICATION_IO_AMPLIFICATION: &str = "rustfs_s3_list_objects_index_verification_io_amplification";
|
||||
const LIST_OBJECTS_LOCAL_READ_DIR_TOTAL: &str = "rustfs_s3_list_objects_local_read_dir_total";
|
||||
const LIST_OBJECTS_LOCAL_READ_DIR_DURATION_MS: &str = "rustfs_s3_list_objects_local_read_dir_duration_ms";
|
||||
const LIST_OBJECTS_LOCAL_READ_DIR_ENTRIES: &str = "rustfs_s3_list_objects_local_read_dir_entries";
|
||||
const LIST_OBJECTS_LOCAL_READ_DIR_LIMIT: &str = "rustfs_s3_list_objects_local_read_dir_limit";
|
||||
|
||||
pub const LIST_OBJECTS_LOCAL_READ_DIR_OUTCOME_OK: &str = "ok";
|
||||
pub const LIST_OBJECTS_LOCAL_READ_DIR_OUTCOME_ERROR: &str = "error";
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq)]
|
||||
pub struct ListObjectsGatherObservation {
|
||||
@@ -76,17 +69,6 @@ pub struct ListObjectsIndexPageObservation {
|
||||
pub is_truncated: bool,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq)]
|
||||
pub struct ListObjectsLocalReadDirObservation {
|
||||
pub outcome: &'static str,
|
||||
pub requested_count: i32,
|
||||
pub returned_entries: usize,
|
||||
pub duration_ms: f64,
|
||||
pub is_root: bool,
|
||||
pub has_filter_prefix: bool,
|
||||
pub has_forward: bool,
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
fn bool_label(value: bool) -> &'static str {
|
||||
if value { "true" } else { "false" }
|
||||
@@ -102,16 +84,6 @@ fn limit_as_f64(value: i32) -> f64 {
|
||||
value.max(0) as f64
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
fn count_mode_label(count: i32) -> &'static str {
|
||||
if count < 0 { "whole" } else { "bounded" }
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
fn read_dir_count_as_f64(value: i32) -> f64 {
|
||||
f64::from(value)
|
||||
}
|
||||
|
||||
pub fn init_list_objects_metrics() {
|
||||
static METRICS_DESC_INIT: OnceLock<()> = OnceLock::new();
|
||||
METRICS_DESC_INIT.get_or_init(|| {
|
||||
@@ -183,22 +155,6 @@ pub fn init_list_objects_metrics() {
|
||||
LIST_OBJECTS_INDEX_VERIFICATION_IO_AMPLIFICATION,
|
||||
"Ratio of live verification attempts to returned objects for opt-in ListObjects index serving."
|
||||
);
|
||||
describe_counter!(
|
||||
LIST_OBJECTS_LOCAL_READ_DIR_TOTAL,
|
||||
"Total number of local read_dir calls made while serving live-walker ListObjects pages."
|
||||
);
|
||||
describe_histogram!(
|
||||
LIST_OBJECTS_LOCAL_READ_DIR_DURATION_MS,
|
||||
"Duration in milliseconds of local read_dir calls made while serving live-walker ListObjects pages."
|
||||
);
|
||||
describe_histogram!(
|
||||
LIST_OBJECTS_LOCAL_READ_DIR_ENTRIES,
|
||||
"Number of immediate directory entries returned by local read_dir while serving live-walker ListObjects pages."
|
||||
);
|
||||
describe_histogram!(
|
||||
LIST_OBJECTS_LOCAL_READ_DIR_LIMIT,
|
||||
"Requested local read_dir count used while serving live-walker ListObjects pages; negative counts mean whole-directory enumeration."
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
@@ -368,38 +324,6 @@ pub fn record_list_objects_index_served(observation: ListObjectsIndexPageObserva
|
||||
.record(verification_io_amplification);
|
||||
}
|
||||
|
||||
pub fn record_list_objects_local_read_dir(observation: ListObjectsLocalReadDirObservation) {
|
||||
if !get_stage_metrics_enabled() {
|
||||
return;
|
||||
}
|
||||
|
||||
let count_mode = count_mode_label(observation.requested_count);
|
||||
|
||||
counter!(
|
||||
LIST_OBJECTS_LOCAL_READ_DIR_TOTAL,
|
||||
"outcome" => observation.outcome,
|
||||
"count_mode" => count_mode,
|
||||
"is_root" => bool_label(observation.is_root),
|
||||
"has_filter_prefix" => bool_label(observation.has_filter_prefix),
|
||||
"has_forward" => bool_label(observation.has_forward),
|
||||
)
|
||||
.increment(1);
|
||||
histogram!(
|
||||
LIST_OBJECTS_LOCAL_READ_DIR_DURATION_MS,
|
||||
"outcome" => observation.outcome,
|
||||
"count_mode" => count_mode,
|
||||
)
|
||||
.record(observation.duration_ms);
|
||||
histogram!(
|
||||
LIST_OBJECTS_LOCAL_READ_DIR_ENTRIES,
|
||||
"outcome" => observation.outcome,
|
||||
"count_mode" => count_mode,
|
||||
)
|
||||
.record(count_as_f64(observation.returned_entries));
|
||||
histogram!(LIST_OBJECTS_LOCAL_READ_DIR_LIMIT, "count_mode" => count_mode)
|
||||
.record(read_dir_count_as_f64(observation.requested_count));
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -449,27 +373,4 @@ mod tests {
|
||||
});
|
||||
record_list_objects_index_live_verify_failure("index_key_only", "read_error");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn record_local_read_dir_observation_accepts_whole_directory_counts() {
|
||||
init_list_objects_metrics();
|
||||
record_list_objects_local_read_dir(ListObjectsLocalReadDirObservation {
|
||||
outcome: LIST_OBJECTS_LOCAL_READ_DIR_OUTCOME_OK,
|
||||
requested_count: -1,
|
||||
returned_entries: 4096,
|
||||
duration_ms: 12.5,
|
||||
is_root: true,
|
||||
has_filter_prefix: false,
|
||||
has_forward: false,
|
||||
});
|
||||
record_list_objects_local_read_dir(ListObjectsLocalReadDirObservation {
|
||||
outcome: LIST_OBJECTS_LOCAL_READ_DIR_OUTCOME_ERROR,
|
||||
requested_count: -1,
|
||||
returned_entries: 0,
|
||||
duration_ms: 5000.0,
|
||||
is_root: true,
|
||||
has_filter_prefix: false,
|
||||
has_forward: true,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -27,7 +27,7 @@ authors.workspace = true
|
||||
|
||||
[dependencies]
|
||||
tokio = { workspace = true, features = ["rt", "sync"] }
|
||||
reqwest = { workspace = true, features = ["json"] }
|
||||
reqwest = { workspace = true, default-features = false, features = ["rustls", "http2", "system-proxy", "json"] }
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
serde_json = { workspace = true, features = ["raw_value"] }
|
||||
thiserror = { workspace = true }
|
||||
|
||||
@@ -60,7 +60,7 @@ rustfs-utils = { workspace = true }
|
||||
rustfs-security-governance = { workspace = true }
|
||||
|
||||
# HTTP client for Vault
|
||||
reqwest = { workspace = true }
|
||||
reqwest = { workspace = true, default-features = false, features = ["rustls", "http2", "system-proxy"] }
|
||||
vaultrs = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
|
||||
@@ -96,23 +96,6 @@ pub(super) fn rules() -> Vec<Rule> {
|
||||
"定位具体盘(结合 samples 的 fields),检查硬件。",
|
||||
)
|
||||
},
|
||||
Rule {
|
||||
evidence_fields: strings(["bucket", "path", "state", "timeout_ms", "drive"]),
|
||||
anchors: strings(["Metacache listing quorum failed", "Metacache reader peek timed out"]),
|
||||
min_count: 2,
|
||||
..base(
|
||||
"metacache-listing-timeout",
|
||||
P2Degraded,
|
||||
"disk",
|
||||
"metacache listing 多盘等待超时",
|
||||
any([
|
||||
all([contains("Metacache listing quorum failed"), field("state", "quorum_failed")]),
|
||||
all([contains("Metacache reader peek timed out"), field("state", "peek_timed_out")]),
|
||||
]),
|
||||
"对象 listing/scanner 期间 metacache reader 等待超时并可能导致 listing quorum failure;常见于宽前缀、高延迟盘或慢远端 reader。",
|
||||
"先确认是否存在宽前缀/慢盘/高延迟远端 reader;再参考 docs/operations/drive-timeout-tuning.md 调整 RUSTFS_DRIVE_WALKDIR_STALL_TIMEOUT_SECS、RUSTFS_DRIVE_WALKDIR_PEEK_TIMEOUT_SECS 或 RUSTFS_DRIVE_TIMEOUT_PROFILE=high_latency。",
|
||||
)
|
||||
},
|
||||
Rule {
|
||||
anchors: strings(["Unformatted disk"]),
|
||||
..base(
|
||||
|
||||
@@ -73,7 +73,7 @@ fn msg(message: &'static str) -> Sample {
|
||||
#[test]
|
||||
fn seed_rule_set_is_valid_and_complete() {
|
||||
let set = seed_rule_set();
|
||||
assert_eq!(set.rules().len(), 69);
|
||||
assert_eq!(set.rules().len(), 68);
|
||||
// Every message-matching rule carries at least one anchor for the CI
|
||||
// guard; the only anchor-less rules are the two structural matchers
|
||||
// (panic kind, scanner target+level).
|
||||
@@ -113,14 +113,6 @@ fn every_rule_has_a_positive_sample() {
|
||||
msg("reporting peer disks offline after consecutive storage_info failures"),
|
||||
),
|
||||
("drive-faulty-error", msg("remote drive is faulty")),
|
||||
(
|
||||
"metacache-listing-timeout",
|
||||
Sample {
|
||||
message: "Metacache listing quorum failed",
|
||||
fields: &[("state", "quorum_failed"), ("bucket", "photos"), ("path", "wide/")],
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
("unformatted-disk", msg("Unformatted disk found")),
|
||||
("disk-access-denied", msg("disk access denied: /data/disk1")),
|
||||
("inconsistent-drive", msg("inconsistent drive found")),
|
||||
@@ -316,23 +308,6 @@ fn smoke_samples_hit_exact_rule_sets() {
|
||||
&["disk-marked-faulty"],
|
||||
);
|
||||
exact(&msg("erasure write quorum (required=8, achieved=5)"), &["ec-write-quorum"]);
|
||||
exact(
|
||||
&Sample {
|
||||
message: "Metacache listing quorum failed",
|
||||
fields: &[("state", "quorum_failed"), ("bucket", "photos"), ("path", "wide/")],
|
||||
..Default::default()
|
||||
},
|
||||
&["metacache-listing-timeout"],
|
||||
);
|
||||
exact(
|
||||
&Sample {
|
||||
message: "S3Error InvalidAccessKeyId: Access Key Id you provided does not exist",
|
||||
fields: &[("code", "InvalidAccessKeyId")],
|
||||
..Default::default()
|
||||
},
|
||||
&["unknown-access-key"],
|
||||
);
|
||||
exact(&msg("HTTP transport failed: BrokenPipe while streaming response body"), &[]);
|
||||
// Internode auth failures legitimately hit both the internode rule and
|
||||
// the generic client signature rule.
|
||||
exact(
|
||||
|
||||
@@ -258,14 +258,6 @@ mod tests {
|
||||
assert!(GRAFANA_DASHBOARD.contains("rustfs_log_cleaner_active_file_size_bytes"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_grafana_dashboard_deduplicates_cluster_bucket_usage() {
|
||||
assert!(GRAFANA_DASHBOARD.contains("max by (job, bucket) (rustfs_cluster_usage_buckets_objects_count"));
|
||||
assert!(GRAFANA_DASHBOARD.contains("max by (job, bucket) (rustfs_cluster_usage_buckets_total_bytes"));
|
||||
assert!(!GRAFANA_DASHBOARD.contains("sum by (bucket) (rustfs_bucket_api_objects_total"));
|
||||
assert!(!GRAFANA_DASHBOARD.contains("sum by (bucket) (rustfs_bucket_api_usage_bytes"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_readme_mentions_deployed_dashboard_path() {
|
||||
assert!(README.contains(".docker/observability/grafana/dashboards/rustfs.json"));
|
||||
|
||||
@@ -97,7 +97,7 @@ pub static USAGE_VERSIONS_DISTRIBUTION_MD: LazyLock<MetricDescriptor> = LazyLock
|
||||
pub static USAGE_BUCKET_TOTAL_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::UsageBucketTotalBytes,
|
||||
"Cluster-wide logical bucket size in bytes",
|
||||
"Total bucket size in bytes",
|
||||
&[BUCKET_LABEL],
|
||||
subsystems::CLUSTER_USAGE_BUCKETS,
|
||||
)
|
||||
@@ -106,7 +106,7 @@ pub static USAGE_BUCKET_TOTAL_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::n
|
||||
pub static USAGE_BUCKET_OBJECTS_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::UsageBucketObjectsCount,
|
||||
"Cluster-wide logical object count in bucket",
|
||||
"Total objects count in bucket",
|
||||
&[BUCKET_LABEL],
|
||||
subsystems::CLUSTER_USAGE_BUCKETS,
|
||||
)
|
||||
@@ -156,22 +156,3 @@ pub static USAGE_BUCKET_OBJECT_VERSION_COUNT_DISTRIBUTION_MD: LazyLock<MetricDes
|
||||
subsystems::CLUSTER_USAGE_BUCKETS,
|
||||
)
|
||||
});
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn bucket_usage_descriptors_identify_cluster_wide_logical_values() {
|
||||
assert_eq!(
|
||||
USAGE_BUCKET_TOTAL_BYTES_MD.get_full_metric_name(),
|
||||
"rustfs_cluster_usage_buckets_total_bytes"
|
||||
);
|
||||
assert_eq!(
|
||||
USAGE_BUCKET_OBJECTS_TOTAL_MD.get_full_metric_name(),
|
||||
"rustfs_cluster_usage_buckets_objects_count"
|
||||
);
|
||||
assert!(USAGE_BUCKET_TOTAL_BYTES_MD.help.contains("Cluster-wide logical"));
|
||||
assert!(USAGE_BUCKET_OBJECTS_TOTAL_MD.help.contains("Cluster-wide logical"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -19,21 +19,21 @@ use std::sync::LazyLock;
|
||||
|
||||
const BUCKET_LABEL: &str = "bucket";
|
||||
|
||||
/// Cluster-wide logical bytes used by the bucket.
|
||||
/// Total bytes used by the bucket
|
||||
pub static BUCKET_USAGE_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::Custom("usage_bytes".to_string()),
|
||||
"Cluster-wide logical bytes used by the bucket",
|
||||
"Total bytes used by the bucket",
|
||||
&[BUCKET_LABEL],
|
||||
MetricSubsystem::new("/bucket/usage"),
|
||||
)
|
||||
});
|
||||
|
||||
/// Cluster-wide logical object count in the bucket.
|
||||
/// Total number of objects in the bucket
|
||||
pub static BUCKET_OBJECTS_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::Custom("objects_total".to_string()),
|
||||
"Cluster-wide logical object count in the bucket",
|
||||
"Total number of objects in the bucket",
|
||||
&[BUCKET_LABEL],
|
||||
MetricSubsystem::new("/bucket/usage"),
|
||||
)
|
||||
@@ -58,7 +58,5 @@ mod tests {
|
||||
assert_eq!(BUCKET_USAGE_BYTES_MD.subsystem.path(), "/bucket/usage");
|
||||
assert_eq!(BUCKET_OBJECTS_TOTAL_MD.subsystem.path(), "/bucket/usage");
|
||||
assert_eq!(BUCKET_QUOTA_BYTES_MD.subsystem.path(), "/bucket/usage");
|
||||
assert!(BUCKET_USAGE_BYTES_MD.help.contains("Cluster-wide logical"));
|
||||
assert!(BUCKET_OBJECTS_TOTAL_MD.help.contains("Cluster-wide logical"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -42,7 +42,7 @@ ipnetwork = { workspace = true, features = ["serde"] }
|
||||
base64-simd = { workspace = true }
|
||||
jsonwebtoken = { workspace = true, features = ["aws_lc_rs"] }
|
||||
regex = { workspace = true }
|
||||
reqwest = { workspace = true, features = ["json"] }
|
||||
reqwest = { workspace = true, default-features = false, features = ["rustls", "http2", "system-proxy", "json"] }
|
||||
chrono = { workspace = true, features = ["serde"] }
|
||||
tracing.workspace = true
|
||||
moka = { workspace = true, features = ["future"] }
|
||||
|
||||
@@ -83,8 +83,6 @@ pub struct GetBucketInfoResponse {
|
||||
pub struct DeleteBucketRequest {
|
||||
#[prost(string, tag = "1")]
|
||||
pub bucket: ::prost::alloc::string::String,
|
||||
#[prost(string, tag = "2")]
|
||||
pub options: ::prost::alloc::string::String,
|
||||
}
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct DeleteBucketResponse {
|
||||
@@ -1225,46 +1223,6 @@ pub struct LoadTransitionTierConfigResponse {
|
||||
#[prost(string, optional, tag = "2")]
|
||||
pub error_info: ::core::option::Option<::prost::alloc::string::String>,
|
||||
}
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct TierMutationPrepareRequest {
|
||||
#[prost(uint32, tag = "1")]
|
||||
pub version: u32,
|
||||
#[prost(string, tag = "2")]
|
||||
pub mutation_id: ::prost::alloc::string::String,
|
||||
#[prost(bytes = "bytes", tag = "3")]
|
||||
pub canonical_payload: ::prost::bytes::Bytes,
|
||||
}
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct TierMutationCommitRequest {
|
||||
#[prost(uint32, tag = "1")]
|
||||
pub version: u32,
|
||||
#[prost(string, tag = "2")]
|
||||
pub mutation_id: ::prost::alloc::string::String,
|
||||
#[prost(bytes = "bytes", tag = "3")]
|
||||
pub canonical_payload: ::prost::bytes::Bytes,
|
||||
}
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct TierMutationAbortRequest {
|
||||
#[prost(uint32, tag = "1")]
|
||||
pub version: u32,
|
||||
#[prost(string, tag = "2")]
|
||||
pub mutation_id: ::prost::alloc::string::String,
|
||||
#[prost(bytes = "bytes", tag = "3")]
|
||||
pub canonical_payload: ::prost::bytes::Bytes,
|
||||
}
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct TierMutationControlResponse {
|
||||
#[prost(bool, tag = "1")]
|
||||
pub success: bool,
|
||||
#[prost(enumeration = "TierMutationPeerState", tag = "2")]
|
||||
pub state: i32,
|
||||
#[prost(bool, tag = "3")]
|
||||
pub applied: bool,
|
||||
#[prost(string, optional, tag = "4")]
|
||||
pub error_info: ::core::option::Option<::prost::alloc::string::String>,
|
||||
#[prost(bytes = "bytes", tag = "5")]
|
||||
pub response_proof: ::prost::bytes::Bytes,
|
||||
}
|
||||
#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct GetLiveEventsRequest {
|
||||
#[prost(uint64, tag = "1")]
|
||||
@@ -1285,38 +1243,6 @@ pub struct GetLiveEventsResponse {
|
||||
#[prost(string, optional, tag = "5")]
|
||||
pub error_info: ::core::option::Option<::prost::alloc::string::String>,
|
||||
}
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)]
|
||||
#[repr(i32)]
|
||||
pub enum TierMutationPeerState {
|
||||
Unspecified = 0,
|
||||
Prepared = 1,
|
||||
Committed = 2,
|
||||
Aborted = 3,
|
||||
}
|
||||
impl TierMutationPeerState {
|
||||
/// String value of the enum field names used in the ProtoBuf definition.
|
||||
///
|
||||
/// The values are not transformed in any way and thus are considered stable
|
||||
/// (if the ProtoBuf definition does not change) and safe for programmatic use.
|
||||
pub fn as_str_name(&self) -> &'static str {
|
||||
match self {
|
||||
Self::Unspecified => "TIER_MUTATION_PEER_STATE_UNSPECIFIED",
|
||||
Self::Prepared => "TIER_MUTATION_PEER_STATE_PREPARED",
|
||||
Self::Committed => "TIER_MUTATION_PEER_STATE_COMMITTED",
|
||||
Self::Aborted => "TIER_MUTATION_PEER_STATE_ABORTED",
|
||||
}
|
||||
}
|
||||
/// Creates an enum from field names used in the ProtoBuf definition.
|
||||
pub fn from_str_name(value: &str) -> ::core::option::Option<Self> {
|
||||
match value {
|
||||
"TIER_MUTATION_PEER_STATE_UNSPECIFIED" => Some(Self::Unspecified),
|
||||
"TIER_MUTATION_PEER_STATE_PREPARED" => Some(Self::Prepared),
|
||||
"TIER_MUTATION_PEER_STATE_COMMITTED" => Some(Self::Committed),
|
||||
"TIER_MUTATION_PEER_STATE_ABORTED" => Some(Self::Aborted),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
/// Generated client implementations.
|
||||
pub mod node_service_client {
|
||||
#![allow(unused_variables, dead_code, missing_docs, clippy::wildcard_imports, clippy::let_unit_value)]
|
||||
@@ -5664,334 +5590,3 @@ pub mod heal_control_service_server {
|
||||
const NAME: &'static str = SERVICE_NAME;
|
||||
}
|
||||
}
|
||||
/// Generated client implementations.
|
||||
pub mod tier_mutation_control_service_client {
|
||||
#![allow(unused_variables, dead_code, missing_docs, clippy::wildcard_imports, clippy::let_unit_value)]
|
||||
use tonic::codegen::http::Uri;
|
||||
use tonic::codegen::*;
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct TierMutationControlServiceClient<T> {
|
||||
inner: tonic::client::Grpc<T>,
|
||||
}
|
||||
impl TierMutationControlServiceClient<tonic::transport::Channel> {
|
||||
/// Attempt to create a new client by connecting to a given endpoint.
|
||||
pub async fn connect<D>(dst: D) -> Result<Self, tonic::transport::Error>
|
||||
where
|
||||
D: TryInto<tonic::transport::Endpoint>,
|
||||
D::Error: Into<StdError>,
|
||||
{
|
||||
let conn = tonic::transport::Endpoint::new(dst)?.connect().await?;
|
||||
Ok(Self::new(conn))
|
||||
}
|
||||
}
|
||||
impl<T> TierMutationControlServiceClient<T>
|
||||
where
|
||||
T: tonic::client::GrpcService<tonic::body::Body>,
|
||||
T::Error: Into<StdError>,
|
||||
T::ResponseBody: Body<Data = Bytes> + std::marker::Send + 'static,
|
||||
<T::ResponseBody as Body>::Error: Into<StdError> + std::marker::Send,
|
||||
{
|
||||
pub fn new(inner: T) -> Self {
|
||||
let inner = tonic::client::Grpc::new(inner);
|
||||
Self { inner }
|
||||
}
|
||||
pub fn with_origin(inner: T, origin: Uri) -> Self {
|
||||
let inner = tonic::client::Grpc::with_origin(inner, origin);
|
||||
Self { inner }
|
||||
}
|
||||
pub fn with_interceptor<F>(inner: T, interceptor: F) -> TierMutationControlServiceClient<InterceptedService<T, F>>
|
||||
where
|
||||
F: tonic::service::Interceptor,
|
||||
T::ResponseBody: Default,
|
||||
T: tonic::codegen::Service<
|
||||
http::Request<tonic::body::Body>,
|
||||
Response = http::Response<<T as tonic::client::GrpcService<tonic::body::Body>>::ResponseBody>,
|
||||
>,
|
||||
<T as tonic::codegen::Service<http::Request<tonic::body::Body>>>::Error:
|
||||
Into<StdError> + std::marker::Send + std::marker::Sync,
|
||||
{
|
||||
TierMutationControlServiceClient::new(InterceptedService::new(inner, interceptor))
|
||||
}
|
||||
/// Compress requests with the given encoding.
|
||||
///
|
||||
/// This requires the server to support it otherwise it might respond with an
|
||||
/// error.
|
||||
#[must_use]
|
||||
pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self {
|
||||
self.inner = self.inner.send_compressed(encoding);
|
||||
self
|
||||
}
|
||||
/// Enable decompressing responses.
|
||||
#[must_use]
|
||||
pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self {
|
||||
self.inner = self.inner.accept_compressed(encoding);
|
||||
self
|
||||
}
|
||||
/// Limits the maximum size of a decoded message.
|
||||
///
|
||||
/// Default: `4MB`
|
||||
#[must_use]
|
||||
pub fn max_decoding_message_size(mut self, limit: usize) -> Self {
|
||||
self.inner = self.inner.max_decoding_message_size(limit);
|
||||
self
|
||||
}
|
||||
/// Limits the maximum size of an encoded message.
|
||||
///
|
||||
/// Default: `usize::MAX`
|
||||
#[must_use]
|
||||
pub fn max_encoding_message_size(mut self, limit: usize) -> Self {
|
||||
self.inner = self.inner.max_encoding_message_size(limit);
|
||||
self
|
||||
}
|
||||
pub async fn prepare_tier_mutation(
|
||||
&mut self,
|
||||
request: impl tonic::IntoRequest<super::TierMutationPrepareRequest>,
|
||||
) -> std::result::Result<tonic::Response<super::TierMutationControlResponse>, tonic::Status> {
|
||||
self.inner
|
||||
.ready()
|
||||
.await
|
||||
.map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?;
|
||||
let codec = tonic_prost::ProstCodec::default();
|
||||
let path = http::uri::PathAndQuery::from_static("/node_service.TierMutationControlService/PrepareTierMutation");
|
||||
let mut req = request.into_request();
|
||||
req.extensions_mut()
|
||||
.insert(GrpcMethod::new("node_service.TierMutationControlService", "PrepareTierMutation"));
|
||||
self.inner.unary(req, path, codec).await
|
||||
}
|
||||
pub async fn commit_tier_mutation(
|
||||
&mut self,
|
||||
request: impl tonic::IntoRequest<super::TierMutationCommitRequest>,
|
||||
) -> std::result::Result<tonic::Response<super::TierMutationControlResponse>, tonic::Status> {
|
||||
self.inner
|
||||
.ready()
|
||||
.await
|
||||
.map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?;
|
||||
let codec = tonic_prost::ProstCodec::default();
|
||||
let path = http::uri::PathAndQuery::from_static("/node_service.TierMutationControlService/CommitTierMutation");
|
||||
let mut req = request.into_request();
|
||||
req.extensions_mut()
|
||||
.insert(GrpcMethod::new("node_service.TierMutationControlService", "CommitTierMutation"));
|
||||
self.inner.unary(req, path, codec).await
|
||||
}
|
||||
pub async fn abort_tier_mutation(
|
||||
&mut self,
|
||||
request: impl tonic::IntoRequest<super::TierMutationAbortRequest>,
|
||||
) -> std::result::Result<tonic::Response<super::TierMutationControlResponse>, tonic::Status> {
|
||||
self.inner
|
||||
.ready()
|
||||
.await
|
||||
.map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?;
|
||||
let codec = tonic_prost::ProstCodec::default();
|
||||
let path = http::uri::PathAndQuery::from_static("/node_service.TierMutationControlService/AbortTierMutation");
|
||||
let mut req = request.into_request();
|
||||
req.extensions_mut()
|
||||
.insert(GrpcMethod::new("node_service.TierMutationControlService", "AbortTierMutation"));
|
||||
self.inner.unary(req, path, codec).await
|
||||
}
|
||||
}
|
||||
}
|
||||
/// Generated server implementations.
|
||||
pub mod tier_mutation_control_service_server {
|
||||
#![allow(unused_variables, dead_code, missing_docs, clippy::wildcard_imports, clippy::let_unit_value)]
|
||||
use tonic::codegen::*;
|
||||
/// Generated trait containing gRPC methods that should be implemented for use with TierMutationControlServiceServer.
|
||||
#[async_trait]
|
||||
pub trait TierMutationControlService: std::marker::Send + std::marker::Sync + 'static {
|
||||
async fn prepare_tier_mutation(
|
||||
&self,
|
||||
request: tonic::Request<super::TierMutationPrepareRequest>,
|
||||
) -> std::result::Result<tonic::Response<super::TierMutationControlResponse>, tonic::Status>;
|
||||
async fn commit_tier_mutation(
|
||||
&self,
|
||||
request: tonic::Request<super::TierMutationCommitRequest>,
|
||||
) -> std::result::Result<tonic::Response<super::TierMutationControlResponse>, tonic::Status>;
|
||||
async fn abort_tier_mutation(
|
||||
&self,
|
||||
request: tonic::Request<super::TierMutationAbortRequest>,
|
||||
) -> std::result::Result<tonic::Response<super::TierMutationControlResponse>, tonic::Status>;
|
||||
}
|
||||
#[derive(Debug)]
|
||||
pub struct TierMutationControlServiceServer<T> {
|
||||
inner: Arc<T>,
|
||||
accept_compression_encodings: EnabledCompressionEncodings,
|
||||
send_compression_encodings: EnabledCompressionEncodings,
|
||||
max_decoding_message_size: Option<usize>,
|
||||
max_encoding_message_size: Option<usize>,
|
||||
}
|
||||
impl<T> TierMutationControlServiceServer<T> {
|
||||
pub fn new(inner: T) -> Self {
|
||||
Self::from_arc(Arc::new(inner))
|
||||
}
|
||||
pub fn from_arc(inner: Arc<T>) -> Self {
|
||||
Self {
|
||||
inner,
|
||||
accept_compression_encodings: Default::default(),
|
||||
send_compression_encodings: Default::default(),
|
||||
max_decoding_message_size: None,
|
||||
max_encoding_message_size: None,
|
||||
}
|
||||
}
|
||||
pub fn with_interceptor<F>(inner: T, interceptor: F) -> InterceptedService<Self, F>
|
||||
where
|
||||
F: tonic::service::Interceptor,
|
||||
{
|
||||
InterceptedService::new(Self::new(inner), interceptor)
|
||||
}
|
||||
/// Enable decompressing requests with the given encoding.
|
||||
#[must_use]
|
||||
pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self {
|
||||
self.accept_compression_encodings.enable(encoding);
|
||||
self
|
||||
}
|
||||
/// Compress responses with the given encoding, if the client supports it.
|
||||
#[must_use]
|
||||
pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self {
|
||||
self.send_compression_encodings.enable(encoding);
|
||||
self
|
||||
}
|
||||
/// Limits the maximum size of a decoded message.
|
||||
///
|
||||
/// Default: `4MB`
|
||||
#[must_use]
|
||||
pub fn max_decoding_message_size(mut self, limit: usize) -> Self {
|
||||
self.max_decoding_message_size = Some(limit);
|
||||
self
|
||||
}
|
||||
/// Limits the maximum size of an encoded message.
|
||||
///
|
||||
/// Default: `usize::MAX`
|
||||
#[must_use]
|
||||
pub fn max_encoding_message_size(mut self, limit: usize) -> Self {
|
||||
self.max_encoding_message_size = Some(limit);
|
||||
self
|
||||
}
|
||||
}
|
||||
impl<T, B> tonic::codegen::Service<http::Request<B>> for TierMutationControlServiceServer<T>
|
||||
where
|
||||
T: TierMutationControlService,
|
||||
B: Body + std::marker::Send + 'static,
|
||||
B::Error: Into<StdError> + std::marker::Send + 'static,
|
||||
{
|
||||
type Response = http::Response<tonic::body::Body>;
|
||||
type Error = std::convert::Infallible;
|
||||
type Future = BoxFuture<Self::Response, Self::Error>;
|
||||
fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll<std::result::Result<(), Self::Error>> {
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
fn call(&mut self, req: http::Request<B>) -> Self::Future {
|
||||
match req.uri().path() {
|
||||
"/node_service.TierMutationControlService/PrepareTierMutation" => {
|
||||
#[allow(non_camel_case_types)]
|
||||
struct PrepareTierMutationSvc<T: TierMutationControlService>(pub Arc<T>);
|
||||
impl<T: TierMutationControlService> tonic::server::UnaryService<super::TierMutationPrepareRequest> for PrepareTierMutationSvc<T> {
|
||||
type Response = super::TierMutationControlResponse;
|
||||
type Future = BoxFuture<tonic::Response<Self::Response>, tonic::Status>;
|
||||
fn call(&mut self, request: tonic::Request<super::TierMutationPrepareRequest>) -> Self::Future {
|
||||
let inner = Arc::clone(&self.0);
|
||||
let fut =
|
||||
async move { <T as TierMutationControlService>::prepare_tier_mutation(&inner, request).await };
|
||||
Box::pin(fut)
|
||||
}
|
||||
}
|
||||
let accept_compression_encodings = self.accept_compression_encodings;
|
||||
let send_compression_encodings = self.send_compression_encodings;
|
||||
let max_decoding_message_size = self.max_decoding_message_size;
|
||||
let max_encoding_message_size = self.max_encoding_message_size;
|
||||
let inner = self.inner.clone();
|
||||
let fut = async move {
|
||||
let method = PrepareTierMutationSvc(inner);
|
||||
let codec = tonic_prost::ProstCodec::default();
|
||||
let mut grpc = tonic::server::Grpc::new(codec)
|
||||
.apply_compression_config(accept_compression_encodings, send_compression_encodings)
|
||||
.apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size);
|
||||
let res = grpc.unary(method, req).await;
|
||||
Ok(res)
|
||||
};
|
||||
Box::pin(fut)
|
||||
}
|
||||
"/node_service.TierMutationControlService/CommitTierMutation" => {
|
||||
#[allow(non_camel_case_types)]
|
||||
struct CommitTierMutationSvc<T: TierMutationControlService>(pub Arc<T>);
|
||||
impl<T: TierMutationControlService> tonic::server::UnaryService<super::TierMutationCommitRequest> for CommitTierMutationSvc<T> {
|
||||
type Response = super::TierMutationControlResponse;
|
||||
type Future = BoxFuture<tonic::Response<Self::Response>, tonic::Status>;
|
||||
fn call(&mut self, request: tonic::Request<super::TierMutationCommitRequest>) -> Self::Future {
|
||||
let inner = Arc::clone(&self.0);
|
||||
let fut =
|
||||
async move { <T as TierMutationControlService>::commit_tier_mutation(&inner, request).await };
|
||||
Box::pin(fut)
|
||||
}
|
||||
}
|
||||
let accept_compression_encodings = self.accept_compression_encodings;
|
||||
let send_compression_encodings = self.send_compression_encodings;
|
||||
let max_decoding_message_size = self.max_decoding_message_size;
|
||||
let max_encoding_message_size = self.max_encoding_message_size;
|
||||
let inner = self.inner.clone();
|
||||
let fut = async move {
|
||||
let method = CommitTierMutationSvc(inner);
|
||||
let codec = tonic_prost::ProstCodec::default();
|
||||
let mut grpc = tonic::server::Grpc::new(codec)
|
||||
.apply_compression_config(accept_compression_encodings, send_compression_encodings)
|
||||
.apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size);
|
||||
let res = grpc.unary(method, req).await;
|
||||
Ok(res)
|
||||
};
|
||||
Box::pin(fut)
|
||||
}
|
||||
"/node_service.TierMutationControlService/AbortTierMutation" => {
|
||||
#[allow(non_camel_case_types)]
|
||||
struct AbortTierMutationSvc<T: TierMutationControlService>(pub Arc<T>);
|
||||
impl<T: TierMutationControlService> tonic::server::UnaryService<super::TierMutationAbortRequest> for AbortTierMutationSvc<T> {
|
||||
type Response = super::TierMutationControlResponse;
|
||||
type Future = BoxFuture<tonic::Response<Self::Response>, tonic::Status>;
|
||||
fn call(&mut self, request: tonic::Request<super::TierMutationAbortRequest>) -> Self::Future {
|
||||
let inner = Arc::clone(&self.0);
|
||||
let fut =
|
||||
async move { <T as TierMutationControlService>::abort_tier_mutation(&inner, request).await };
|
||||
Box::pin(fut)
|
||||
}
|
||||
}
|
||||
let accept_compression_encodings = self.accept_compression_encodings;
|
||||
let send_compression_encodings = self.send_compression_encodings;
|
||||
let max_decoding_message_size = self.max_decoding_message_size;
|
||||
let max_encoding_message_size = self.max_encoding_message_size;
|
||||
let inner = self.inner.clone();
|
||||
let fut = async move {
|
||||
let method = AbortTierMutationSvc(inner);
|
||||
let codec = tonic_prost::ProstCodec::default();
|
||||
let mut grpc = tonic::server::Grpc::new(codec)
|
||||
.apply_compression_config(accept_compression_encodings, send_compression_encodings)
|
||||
.apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size);
|
||||
let res = grpc.unary(method, req).await;
|
||||
Ok(res)
|
||||
};
|
||||
Box::pin(fut)
|
||||
}
|
||||
_ => Box::pin(async move {
|
||||
let mut response = http::Response::new(tonic::body::Body::default());
|
||||
let headers = response.headers_mut();
|
||||
headers.insert(tonic::Status::GRPC_STATUS, (tonic::Code::Unimplemented as i32).into());
|
||||
headers.insert(http::header::CONTENT_TYPE, tonic::metadata::GRPC_CONTENT_TYPE);
|
||||
Ok(response)
|
||||
}),
|
||||
}
|
||||
}
|
||||
}
|
||||
impl<T> Clone for TierMutationControlServiceServer<T> {
|
||||
fn clone(&self) -> Self {
|
||||
let inner = self.inner.clone();
|
||||
Self {
|
||||
inner,
|
||||
accept_compression_encodings: self.accept_compression_encodings,
|
||||
send_compression_encodings: self.send_compression_encodings,
|
||||
max_decoding_message_size: self.max_decoding_message_size,
|
||||
max_encoding_message_size: self.max_encoding_message_size,
|
||||
}
|
||||
}
|
||||
}
|
||||
/// Generated gRPC service name
|
||||
pub const SERVICE_NAME: &str = "node_service.TierMutationControlService";
|
||||
impl<T> tonic::server::NamedService for TierMutationControlServiceServer<T> {
|
||||
const NAME: &'static str = SERVICE_NAME;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -35,7 +35,6 @@ use tonic::{
|
||||
transport::{Certificate, Channel, ClientTlsConfig, Endpoint},
|
||||
};
|
||||
use tracing::{debug, info, warn};
|
||||
use uuid::Uuid;
|
||||
|
||||
// Type alias for the complex client type
|
||||
pub type NodeServiceClientType = NodeServiceClient<
|
||||
@@ -166,9 +165,6 @@ pub fn internode_rpc_max_message_size() -> usize {
|
||||
pub const HEAL_CONTROL_RPC_MAX_MESSAGE_SIZE: usize = heal_control::RESULT_MAX_SIZE + 1024;
|
||||
pub const HEAL_CONTROL_PROTOCOL_VERSION: u32 = 2;
|
||||
pub const HEAL_CONTROL_CAPABILITY_PROBE_PREFIX: &[u8] = b"rustfs-heal-control-capability-v2\0";
|
||||
pub const TIER_MUTATION_RPC_MAX_PREPARE_PAYLOAD_SIZE: usize = 64 * 1024;
|
||||
pub const TIER_MUTATION_RPC_MAX_COMMIT_PAYLOAD_SIZE: usize = 1024;
|
||||
pub const TIER_MUTATION_RPC_MAX_MESSAGE_SIZE: usize = TIER_MUTATION_RPC_MAX_PREPARE_PAYLOAD_SIZE + 4096;
|
||||
|
||||
pub fn heal_control_coordinator_epoch(topology_fingerprint: &str) -> Result<u64, &'static str> {
|
||||
let prefix = topology_fingerprint
|
||||
@@ -255,100 +251,6 @@ pub fn canonical_heal_control_response_body(
|
||||
Ok(body)
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
#[non_exhaustive]
|
||||
pub enum TierMutationRpcPhase {
|
||||
Prepare,
|
||||
Commit,
|
||||
Abort,
|
||||
}
|
||||
|
||||
impl TierMutationRpcPhase {
|
||||
fn as_wire_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Prepare => "prepare",
|
||||
Self::Commit => "commit",
|
||||
Self::Abort => "abort",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub const TIER_MUTATION_RPC_PROTOCOL_VERSION: u32 = 1;
|
||||
|
||||
pub fn canonical_tier_mutation_rpc_body(
|
||||
version: u32,
|
||||
phase: TierMutationRpcPhase,
|
||||
mutation_id: Uuid,
|
||||
canonical_payload: &[u8],
|
||||
) -> Result<Vec<u8>, std::num::TryFromIntError> {
|
||||
const DOMAIN: &[u8] = b"rustfs-tier-mutation-rpc-v1\0";
|
||||
|
||||
let phase = phase.as_wire_str().as_bytes();
|
||||
let mutation_id = mutation_id.as_bytes();
|
||||
let mut body = Vec::with_capacity(DOMAIN.len() + 4 + 8 + phase.len() + mutation_id.len() + 8 + canonical_payload.len());
|
||||
body.extend_from_slice(DOMAIN);
|
||||
body.extend_from_slice(&version.to_be_bytes());
|
||||
body.extend_from_slice(&u64::try_from(phase.len())?.to_be_bytes());
|
||||
body.extend_from_slice(phase);
|
||||
body.extend_from_slice(mutation_id);
|
||||
body.extend_from_slice(&u64::try_from(canonical_payload.len())?.to_be_bytes());
|
||||
body.extend_from_slice(canonical_payload);
|
||||
Ok(body)
|
||||
}
|
||||
|
||||
pub struct TierMutationRpcResponseProofInput<'a> {
|
||||
pub version: u32,
|
||||
pub phase: TierMutationRpcPhase,
|
||||
pub mutation_id: Uuid,
|
||||
pub canonical_payload: &'a [u8],
|
||||
pub success: bool,
|
||||
pub state: i32,
|
||||
pub applied: bool,
|
||||
pub error_info: Option<&'a str>,
|
||||
}
|
||||
|
||||
pub fn canonical_tier_mutation_rpc_response_body(
|
||||
input: TierMutationRpcResponseProofInput<'_>,
|
||||
) -> Result<Vec<u8>, std::num::TryFromIntError> {
|
||||
const DOMAIN: &[u8] = b"rustfs-tier-mutation-rpc-response-v1\0";
|
||||
|
||||
let phase = input.phase.as_wire_str().as_bytes();
|
||||
let mutation_id = input.mutation_id.as_bytes();
|
||||
let error_info = input.error_info.map(str::as_bytes);
|
||||
let error_info_len = error_info.map_or(0, <[u8]>::len);
|
||||
let mut body = Vec::with_capacity(
|
||||
DOMAIN.len()
|
||||
+ 4
|
||||
+ 8
|
||||
+ phase.len()
|
||||
+ mutation_id.len()
|
||||
+ 8
|
||||
+ input.canonical_payload.len()
|
||||
+ 1
|
||||
+ 4
|
||||
+ 1
|
||||
+ 1
|
||||
+ 8
|
||||
+ error_info_len,
|
||||
);
|
||||
body.extend_from_slice(DOMAIN);
|
||||
body.extend_from_slice(&input.version.to_be_bytes());
|
||||
body.extend_from_slice(&u64::try_from(phase.len())?.to_be_bytes());
|
||||
body.extend_from_slice(phase);
|
||||
body.extend_from_slice(mutation_id);
|
||||
body.extend_from_slice(&u64::try_from(input.canonical_payload.len())?.to_be_bytes());
|
||||
body.extend_from_slice(input.canonical_payload);
|
||||
body.push(u8::from(input.success));
|
||||
body.extend_from_slice(&input.state.to_be_bytes());
|
||||
body.push(u8::from(input.applied));
|
||||
body.push(u8::from(error_info.is_some()));
|
||||
body.extend_from_slice(&u64::try_from(error_info_len)?.to_be_bytes());
|
||||
if let Some(error_info) = error_info {
|
||||
body.extend_from_slice(error_info);
|
||||
}
|
||||
Ok(body)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod heal_control_tests {
|
||||
use super::{
|
||||
@@ -461,179 +363,6 @@ mod heal_control_tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tier_mutation_rpc_tests {
|
||||
use super::{
|
||||
TIER_MUTATION_RPC_PROTOCOL_VERSION, TierMutationRpcPhase, TierMutationRpcResponseProofInput,
|
||||
canonical_tier_mutation_rpc_body, canonical_tier_mutation_rpc_response_body,
|
||||
};
|
||||
use crate::proto_gen::node_service::TierMutationPeerState;
|
||||
use uuid::uuid;
|
||||
|
||||
#[test]
|
||||
fn canonical_tier_mutation_body_binds_phase_id_and_payload() {
|
||||
let mutation_id = uuid!("12345678-1234-5678-9abc-def012345678");
|
||||
let payload = b"canonical-intent-record";
|
||||
let baseline = canonical_tier_mutation_rpc_body(
|
||||
TIER_MUTATION_RPC_PROTOCOL_VERSION,
|
||||
TierMutationRpcPhase::Prepare,
|
||||
mutation_id,
|
||||
payload,
|
||||
)
|
||||
.expect("small mutation body should encode");
|
||||
let mut golden = b"rustfs-tier-mutation-rpc-v1\0".to_vec();
|
||||
golden.extend_from_slice(&1_u32.to_be_bytes());
|
||||
golden.extend_from_slice(&7_u64.to_be_bytes());
|
||||
golden.extend_from_slice(b"prepare");
|
||||
golden.extend_from_slice(mutation_id.as_bytes());
|
||||
golden.extend_from_slice(&u64::try_from(payload.len()).expect("payload length should fit").to_be_bytes());
|
||||
golden.extend_from_slice(payload);
|
||||
assert_eq!(baseline, golden);
|
||||
|
||||
assert_ne!(
|
||||
baseline,
|
||||
canonical_tier_mutation_rpc_body(2, TierMutationRpcPhase::Prepare, mutation_id, payload)
|
||||
.expect("small mutation body should encode")
|
||||
);
|
||||
assert_ne!(
|
||||
baseline,
|
||||
canonical_tier_mutation_rpc_body(
|
||||
TIER_MUTATION_RPC_PROTOCOL_VERSION,
|
||||
TierMutationRpcPhase::Commit,
|
||||
mutation_id,
|
||||
payload,
|
||||
)
|
||||
.expect("small mutation body should encode")
|
||||
);
|
||||
assert_ne!(
|
||||
baseline,
|
||||
canonical_tier_mutation_rpc_body(
|
||||
TIER_MUTATION_RPC_PROTOCOL_VERSION,
|
||||
TierMutationRpcPhase::Prepare,
|
||||
uuid!("22345678-1234-5678-9abc-def012345678"),
|
||||
payload,
|
||||
)
|
||||
.expect("small mutation body should encode")
|
||||
);
|
||||
assert_ne!(
|
||||
baseline,
|
||||
canonical_tier_mutation_rpc_body(
|
||||
TIER_MUTATION_RPC_PROTOCOL_VERSION,
|
||||
TierMutationRpcPhase::Prepare,
|
||||
mutation_id,
|
||||
b"canonical-intent-record-tampered",
|
||||
)
|
||||
.expect("small mutation body should encode")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn canonical_tier_mutation_response_binds_request_state_and_error() {
|
||||
let mutation_id = uuid!("12345678-1234-5678-9abc-def012345678");
|
||||
let payload = b"canonical-intent-record";
|
||||
let baseline = canonical_tier_mutation_rpc_response_body(TierMutationRpcResponseProofInput {
|
||||
version: TIER_MUTATION_RPC_PROTOCOL_VERSION,
|
||||
phase: TierMutationRpcPhase::Prepare,
|
||||
mutation_id,
|
||||
canonical_payload: payload,
|
||||
success: true,
|
||||
state: TierMutationPeerState::Prepared as i32,
|
||||
applied: true,
|
||||
error_info: None,
|
||||
})
|
||||
.expect("small mutation response should encode");
|
||||
|
||||
let cases = [
|
||||
TierMutationRpcResponseProofInput {
|
||||
version: 2,
|
||||
phase: TierMutationRpcPhase::Prepare,
|
||||
mutation_id,
|
||||
canonical_payload: payload,
|
||||
success: true,
|
||||
state: TierMutationPeerState::Prepared as i32,
|
||||
applied: true,
|
||||
error_info: None,
|
||||
},
|
||||
TierMutationRpcResponseProofInput {
|
||||
version: TIER_MUTATION_RPC_PROTOCOL_VERSION,
|
||||
phase: TierMutationRpcPhase::Commit,
|
||||
mutation_id,
|
||||
canonical_payload: payload,
|
||||
success: true,
|
||||
state: TierMutationPeerState::Prepared as i32,
|
||||
applied: true,
|
||||
error_info: None,
|
||||
},
|
||||
TierMutationRpcResponseProofInput {
|
||||
version: TIER_MUTATION_RPC_PROTOCOL_VERSION,
|
||||
phase: TierMutationRpcPhase::Prepare,
|
||||
mutation_id: uuid!("22345678-1234-5678-9abc-def012345678"),
|
||||
canonical_payload: payload,
|
||||
success: true,
|
||||
state: TierMutationPeerState::Prepared as i32,
|
||||
applied: true,
|
||||
error_info: None,
|
||||
},
|
||||
TierMutationRpcResponseProofInput {
|
||||
version: TIER_MUTATION_RPC_PROTOCOL_VERSION,
|
||||
phase: TierMutationRpcPhase::Prepare,
|
||||
mutation_id,
|
||||
canonical_payload: b"tampered-intent-record",
|
||||
success: true,
|
||||
state: TierMutationPeerState::Prepared as i32,
|
||||
applied: true,
|
||||
error_info: None,
|
||||
},
|
||||
TierMutationRpcResponseProofInput {
|
||||
version: TIER_MUTATION_RPC_PROTOCOL_VERSION,
|
||||
phase: TierMutationRpcPhase::Prepare,
|
||||
mutation_id,
|
||||
canonical_payload: payload,
|
||||
success: false,
|
||||
state: TierMutationPeerState::Prepared as i32,
|
||||
applied: true,
|
||||
error_info: None,
|
||||
},
|
||||
TierMutationRpcResponseProofInput {
|
||||
version: TIER_MUTATION_RPC_PROTOCOL_VERSION,
|
||||
phase: TierMutationRpcPhase::Prepare,
|
||||
mutation_id,
|
||||
canonical_payload: payload,
|
||||
success: true,
|
||||
state: TierMutationPeerState::Committed as i32,
|
||||
applied: true,
|
||||
error_info: None,
|
||||
},
|
||||
TierMutationRpcResponseProofInput {
|
||||
version: TIER_MUTATION_RPC_PROTOCOL_VERSION,
|
||||
phase: TierMutationRpcPhase::Prepare,
|
||||
mutation_id,
|
||||
canonical_payload: payload,
|
||||
success: true,
|
||||
state: TierMutationPeerState::Prepared as i32,
|
||||
applied: false,
|
||||
error_info: None,
|
||||
},
|
||||
TierMutationRpcResponseProofInput {
|
||||
version: TIER_MUTATION_RPC_PROTOCOL_VERSION,
|
||||
phase: TierMutationRpcPhase::Prepare,
|
||||
mutation_id,
|
||||
canonical_payload: payload,
|
||||
success: true,
|
||||
state: TierMutationPeerState::Prepared as i32,
|
||||
applied: true,
|
||||
error_info: Some("error"),
|
||||
},
|
||||
];
|
||||
for case in cases {
|
||||
assert_ne!(
|
||||
baseline,
|
||||
canonical_tier_mutation_rpc_response_body(case).expect("small mutation response should encode")
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether internode metadata RPCs should send only the msgpack `_bin` payloads and leave the JSON
|
||||
/// compatibility strings empty (grpc-optimization P2-1). Shared by the client (`remote_disk`) and
|
||||
/// server (`node_service`) send paths. Defaults to `false` (dual-write); see
|
||||
|
||||
@@ -74,7 +74,6 @@ message GetBucketInfoResponse {
|
||||
|
||||
message DeleteBucketRequest {
|
||||
string bucket = 1;
|
||||
string options = 2;
|
||||
}
|
||||
|
||||
message DeleteBucketResponse {
|
||||
@@ -866,39 +865,6 @@ message LoadTransitionTierConfigResponse {
|
||||
optional string error_info = 2;
|
||||
}
|
||||
|
||||
message TierMutationPrepareRequest {
|
||||
uint32 version = 1;
|
||||
string mutation_id = 2;
|
||||
bytes canonical_payload = 3;
|
||||
}
|
||||
|
||||
message TierMutationCommitRequest {
|
||||
uint32 version = 1;
|
||||
string mutation_id = 2;
|
||||
bytes canonical_payload = 3;
|
||||
}
|
||||
|
||||
message TierMutationAbortRequest {
|
||||
uint32 version = 1;
|
||||
string mutation_id = 2;
|
||||
bytes canonical_payload = 3;
|
||||
}
|
||||
|
||||
enum TierMutationPeerState {
|
||||
TIER_MUTATION_PEER_STATE_UNSPECIFIED = 0;
|
||||
TIER_MUTATION_PEER_STATE_PREPARED = 1;
|
||||
TIER_MUTATION_PEER_STATE_COMMITTED = 2;
|
||||
TIER_MUTATION_PEER_STATE_ABORTED = 3;
|
||||
}
|
||||
|
||||
message TierMutationControlResponse {
|
||||
bool success = 1;
|
||||
TierMutationPeerState state = 2;
|
||||
bool applied = 3;
|
||||
optional string error_info = 4;
|
||||
bytes response_proof = 5;
|
||||
}
|
||||
|
||||
message GetLiveEventsRequest {
|
||||
uint64 after_sequence = 1;
|
||||
uint32 limit = 2;
|
||||
@@ -1017,9 +983,3 @@ service NodeService {
|
||||
service HealControlService {
|
||||
rpc HealControl(HealControlRequest) returns (HealControlResponse) {};
|
||||
}
|
||||
|
||||
service TierMutationControlService {
|
||||
rpc PrepareTierMutation(TierMutationPrepareRequest) returns (TierMutationControlResponse) {};
|
||||
rpc CommitTierMutation(TierMutationCommitRequest) returns (TierMutationControlResponse) {};
|
||||
rpc AbortTierMutation(TierMutationAbortRequest) returns (TierMutationControlResponse) {};
|
||||
}
|
||||
|
||||
@@ -43,7 +43,7 @@ crc-fast = { workspace = true }
|
||||
pin-project-lite.workspace = true
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
bytes = { workspace = true, features = ["serde"] }
|
||||
reqwest = { workspace = true, features = ["stream"] }
|
||||
reqwest = { workspace = true, default-features = false, features = ["rustls", "http2", "system-proxy", "stream"] }
|
||||
rustls-pki-types.workspace = true
|
||||
tokio-util = { workspace = true, features = ["io", "compat"] }
|
||||
faster-hex.workspace = true
|
||||
|
||||
@@ -1496,7 +1496,6 @@ mod tests {
|
||||
use axum::{Router, body::Body, extract::State, http::StatusCode, response::IntoResponse, routing::get};
|
||||
use futures::stream::{self, StreamExt as _};
|
||||
use http_body_util::BodyExt as _;
|
||||
use rustfs_io_metrics::internode_metrics::global_internode_metrics;
|
||||
use std::io::{self, IoSlice};
|
||||
use std::sync::{
|
||||
Arc,
|
||||
@@ -1742,7 +1741,6 @@ mod tests {
|
||||
let addr = listener.local_addr().expect("listener local address should be available");
|
||||
let app = Router::new()
|
||||
.route("/stream", get(get_stream).head(reject_head).put(accept_put))
|
||||
.route(WALK_DIR_PATH, get(get_stream))
|
||||
.route("/reject-put", get(get_stream).put(reject_put))
|
||||
.route("/stall", get(get_stalling_stream))
|
||||
.route("/delayed-first", get(get_delayed_first_chunk))
|
||||
@@ -1803,33 +1801,6 @@ mod tests {
|
||||
handle.abort();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn http_reader_records_walk_dir_recv_bytes() {
|
||||
let state = TestState::default();
|
||||
let Some((base_url, handle)) = start_test_server(state.clone()).await else {
|
||||
return;
|
||||
};
|
||||
let metrics = global_internode_metrics();
|
||||
let before = metrics.snapshot().recv_bytes_total;
|
||||
let url = base_url.replace("/stream", WALK_DIR_PATH);
|
||||
|
||||
let mut reader = HttpReader::new(url, Method::GET, HeaderMap::new(), None)
|
||||
.await
|
||||
.expect("walk_dir reader should open");
|
||||
let mut buf = Vec::new();
|
||||
reader.read_to_end(&mut buf).await.expect("walk_dir body should read to EOF");
|
||||
let after = metrics.snapshot().recv_bytes_total;
|
||||
|
||||
assert_eq!(buf, b"hello");
|
||||
assert_eq!(state.get_count.load(Ordering::SeqCst), 1);
|
||||
assert!(
|
||||
after >= before.saturating_add(5),
|
||||
"walk_dir HttpReader should record streamed bytes as internode recv bytes: before={before}, after={after}"
|
||||
);
|
||||
|
||||
handle.abort();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn http_reader_stall_timeout_triggers_after_progress_stops() {
|
||||
let state = TestState::default();
|
||||
|
||||
@@ -105,17 +105,17 @@ impl SessionCtxFactory {
|
||||
{\"id\":10,\"name\":\"Jack\",\"age\":38,\"department\":\"Finance\",\"salary\":7500}\n"
|
||||
.to_vec()
|
||||
} else {
|
||||
b"id,name,age,department,salary\n\
|
||||
1,Alice,25,HR,05000\n\
|
||||
2,Bob,30,IT,6000\n\
|
||||
3,Charlie,35,Finance,7000\n\
|
||||
4,Diana,22,Marketing,4500\n\
|
||||
5,Eve,28,IT,5500\n\
|
||||
6,Frank,40,Finance,8000\n\
|
||||
7,Grace,26,HR,5200\n\
|
||||
8,Henry,32,IT,6200\n\
|
||||
9,Ivy,24,Marketing,4800\n\
|
||||
10,Jack,38,Finance,7500"
|
||||
b"id,name,age,department,salary
|
||||
1,Alice,25,HR,5000
|
||||
2,Bob,30,IT,6000
|
||||
3,Charlie,35,Finance,7000
|
||||
4,Diana,22,Marketing,4500
|
||||
5,Eve,28,IT,5500
|
||||
6,Frank,40,Finance,8000
|
||||
7,Grace,26,HR,5200
|
||||
8,Henry,32,IT,6200
|
||||
9,Ivy,24,Marketing,4800
|
||||
10,Jack,38,Finance,7500"
|
||||
.to_vec()
|
||||
};
|
||||
|
||||
|
||||
@@ -180,7 +180,6 @@ impl SimpleQueryDispatcher {
|
||||
let mut need_rename_volume_name = false;
|
||||
let mut need_ignore_volume_name = false;
|
||||
let mut file_format = CsvFormat::default()
|
||||
.with_schema_infer_max_rec(0)
|
||||
.with_comment(
|
||||
csv.comments
|
||||
.clone()
|
||||
|
||||
@@ -224,20 +224,20 @@ mod tests {
|
||||
.to_vec();
|
||||
|
||||
let expected = [
|
||||
"+----+---------+-----+------------+--------+",
|
||||
"| id | name | age | department | salary |",
|
||||
"+----+---------+-----+------------+--------+",
|
||||
"| 1 | Alice | 25 | HR | 05000 |",
|
||||
"| 2 | Bob | 30 | IT | 6000 |",
|
||||
"| 3 | Charlie | 35 | Finance | 7000 |",
|
||||
"| 4 | Diana | 22 | Marketing | 4500 |",
|
||||
"| 5 | Eve | 28 | IT | 5500 |",
|
||||
"| 6 | Frank | 40 | Finance | 8000 |",
|
||||
"| 7 | Grace | 26 | HR | 5200 |",
|
||||
"| 8 | Henry | 32 | IT | 6200 |",
|
||||
"| 9 | Ivy | 24 | Marketing | 4800 |",
|
||||
"| 10 | Jack | 38 | Finance | 7500 |",
|
||||
"+----+---------+-----+------------+--------+",
|
||||
"+----------------+---------+-----+------------+--------+",
|
||||
"| id | name | age | department | salary |",
|
||||
"+----------------+---------+-----+------------+--------+",
|
||||
"| 1 | Alice | 25 | HR | 5000 |",
|
||||
"| 2 | Bob | 30 | IT | 6000 |",
|
||||
"| 3 | Charlie | 35 | Finance | 7000 |",
|
||||
"| 4 | Diana | 22 | Marketing | 4500 |",
|
||||
"| 5 | Eve | 28 | IT | 5500 |",
|
||||
"| 6 | Frank | 40 | Finance | 8000 |",
|
||||
"| 7 | Grace | 26 | HR | 5200 |",
|
||||
"| 8 | Henry | 32 | IT | 6200 |",
|
||||
"| 9 | Ivy | 24 | Marketing | 4800 |",
|
||||
"| 10 | Jack | 38 | Finance | 7500 |",
|
||||
"+----------------+---------+-----+------------+--------+",
|
||||
];
|
||||
|
||||
assert_batches_eq!(expected, &results);
|
||||
|
||||
@@ -15,7 +15,6 @@
|
||||
#[cfg(test)]
|
||||
mod integration_tests {
|
||||
use crate::{create_fresh_db, get_global_db, instance::make_rustfsms};
|
||||
use datafusion::arrow::array::{Array, StringArray};
|
||||
use rustfs_s3select_api::{
|
||||
QueryError,
|
||||
query::{Context, Query},
|
||||
@@ -171,123 +170,14 @@ mod integration_tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_csv_values_remain_strings() {
|
||||
let sql = "SELECT salary FROM S3Object LIMIT 1";
|
||||
async fn test_select_with_where_clause() {
|
||||
let sql = "SELECT name, age FROM S3Object WHERE age > 30";
|
||||
let input = create_test_input(sql);
|
||||
let db = get_global_db(input.clone(), true).await.expect("create CSV test database");
|
||||
let db = get_global_db(input.clone(), true).await.unwrap();
|
||||
let query = Query::new(Context { input: Arc::new(input) }, sql.to_string());
|
||||
|
||||
let batches = db
|
||||
.execute(&query)
|
||||
.await
|
||||
.expect("execute CSV query")
|
||||
.result()
|
||||
.chunk_result()
|
||||
.await
|
||||
.expect("collect CSV query output");
|
||||
let salaries = batches
|
||||
.first()
|
||||
.expect("CSV query should return one batch")
|
||||
.column(0)
|
||||
.as_any()
|
||||
.downcast_ref::<StringArray>()
|
||||
.expect("CSV column should use UTF-8 string values");
|
||||
|
||||
assert_eq!(salaries.value(0), "05000");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_csv_header_modes_keep_positional_string_columns() {
|
||||
for (header, expected) in [
|
||||
(FileHeaderInfo::IGNORE, ["05000", "6000"]),
|
||||
(FileHeaderInfo::NONE, ["salary", "05000"]),
|
||||
] {
|
||||
let sql = "SELECT _5 FROM S3Object LIMIT 2";
|
||||
let mut input = create_test_input(sql);
|
||||
input
|
||||
.request
|
||||
.input_serialization
|
||||
.csv
|
||||
.as_mut()
|
||||
.expect("CSV input should be configured")
|
||||
.file_header_info = Some(FileHeaderInfo::from_static(header));
|
||||
let db = get_global_db(input.clone(), true).await.expect("create CSV test database");
|
||||
let query = Query::new(Context { input: Arc::new(input) }, sql.to_string());
|
||||
let batches = db
|
||||
.execute(&query)
|
||||
.await
|
||||
.expect("execute positional CSV query")
|
||||
.result()
|
||||
.chunk_result()
|
||||
.await
|
||||
.expect("collect positional CSV query output");
|
||||
|
||||
let values = batches
|
||||
.iter()
|
||||
.flat_map(|batch| {
|
||||
let column = batch
|
||||
.column(0)
|
||||
.as_any()
|
||||
.downcast_ref::<StringArray>()
|
||||
.expect("positional CSV column should use UTF-8 strings");
|
||||
(0..column.len()).map(|row| column.value(row).to_string()).collect::<Vec<_>>()
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
assert_eq!(
|
||||
values,
|
||||
expected.map(|value| value.to_string()),
|
||||
"unexpected values for FileHeaderInfo={header}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_csv_numeric_comparison_with_and_without_cast() {
|
||||
for sql in [
|
||||
"SELECT name, age FROM S3Object WHERE age > 30",
|
||||
"SELECT name, age FROM S3Object WHERE CAST(age AS INT) > 30",
|
||||
] {
|
||||
let input = create_test_input(sql);
|
||||
let db = get_global_db(input.clone(), true).await.expect("create CSV test database");
|
||||
let query = Query::new(Context { input: Arc::new(input) }, sql.to_string());
|
||||
let batches = db
|
||||
.execute(&query)
|
||||
.await
|
||||
.expect("execute CSV numeric comparison")
|
||||
.result()
|
||||
.chunk_result()
|
||||
.await
|
||||
.expect("collect CSV numeric comparison output");
|
||||
|
||||
let mut rows = Vec::new();
|
||||
for batch in &batches {
|
||||
let names = batch
|
||||
.column(0)
|
||||
.as_any()
|
||||
.downcast_ref::<StringArray>()
|
||||
.expect("CSV name column should use UTF-8 strings");
|
||||
let ages = batch
|
||||
.column(1)
|
||||
.as_any()
|
||||
.downcast_ref::<StringArray>()
|
||||
.expect("CSV age column should use UTF-8 strings");
|
||||
for row in 0..batch.num_rows() {
|
||||
rows.push((names.value(row).to_string(), ages.value(row).to_string()));
|
||||
}
|
||||
}
|
||||
|
||||
assert_eq!(
|
||||
rows,
|
||||
[
|
||||
("Charlie".to_string(), "35".to_string()),
|
||||
("Frank".to_string(), "40".to_string()),
|
||||
("Henry".to_string(), "32".to_string()),
|
||||
("Jack".to_string(), "38".to_string()),
|
||||
],
|
||||
"unexpected rows for query: {sql}"
|
||||
);
|
||||
}
|
||||
let result = db.execute(&query).await;
|
||||
assert!(result.is_ok());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -41,11 +41,10 @@ use storage_api::lifecycle::{
|
||||
BUCKET_LIFECYCLE_CONFIG, BucketOperations, BucketOptions, BucketVersioningSys, CompletePart, DiskOption, ECStore,
|
||||
EcstoreError, Endpoint, EndpointServerPools, Endpoints, IlmAction, LcEvent, LcEventSrc, ListOperations as _,
|
||||
MakeBucketOptions, MockWarmBackend, MultipartOperations as _, ObjectIO as _, ObjectOperations as _, PoolEndpoints,
|
||||
STORAGE_FORMAT_FILE, TRANSITION_PENDING, TransitionCleanupStoreBarrier, TransitionOptions, assert_transition_meta_consistent,
|
||||
enqueue_transition_for_existing_objects, expire_transitioned_object, free_version_count, get_bucket_metadata,
|
||||
get_global_tier_config_mgr, init_background_expiry, init_bucket_metadata_sys, init_local_disks, is_err_object_not_found,
|
||||
is_err_version_not_found, new_disk, path2_bucket_object_with_base_path, recover_tier_delete_journal_entries,
|
||||
register_mock_tier_util, update_bucket_metadata, wait_for_free_version_absence,
|
||||
STORAGE_FORMAT_FILE, TransitionOptions, assert_transition_meta_consistent, enqueue_transition_for_existing_objects,
|
||||
expire_transitioned_object, free_version_count, get_bucket_metadata, get_global_tier_config_mgr, init_background_expiry,
|
||||
init_bucket_metadata_sys, init_local_disks, is_err_object_not_found, is_err_version_not_found, new_disk,
|
||||
path2_bucket_object_with_base_path, register_mock_tier_util, update_bucket_metadata, wait_for_free_version_absence,
|
||||
};
|
||||
|
||||
static GLOBAL_ENV: OnceLock<(Vec<PathBuf>, Arc<ECStore>)> = OnceLock::new();
|
||||
@@ -604,7 +603,6 @@ mod serial_tests {
|
||||
|
||||
let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
|
||||
let backend = register_mock_tier(&tier_name).await;
|
||||
backend.set_put_remote_version(Some(String::new())).await;
|
||||
|
||||
let bucket_name = format!("test-expire-get-race-{}", &Uuid::new_v4().simple().to_string()[..8]);
|
||||
let object_name = "test/race-object.bin";
|
||||
@@ -641,10 +639,6 @@ mod serial_tests {
|
||||
.get_object_info(bucket_name.as_str(), object_name, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("Failed to load transitioned object info");
|
||||
assert!(
|
||||
oi.transitioned_object.version_id.is_empty(),
|
||||
"the regression must exercise an unversioned remote tier"
|
||||
);
|
||||
|
||||
// Concurrent GET loop: hammer GET while the expiry runs. Every outcome
|
||||
// must be a full correct body or a clean not-found -- never a tier-fetch
|
||||
@@ -738,327 +732,6 @@ mod serial_tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||
#[serial]
|
||||
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial"]
|
||||
async fn rejected_transition_candidate_is_recovered_from_persisted_delete_journal() {
|
||||
let (_disk_paths, ecstore) = setup_isolated_test_env(false).await;
|
||||
|
||||
let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
|
||||
let backend = register_mock_tier(&tier_name).await;
|
||||
backend.set_put_read_limit(Some(4096)).await;
|
||||
backend.set_put_remote_version(Some(String::new())).await;
|
||||
backend.set_remove_failure(true);
|
||||
|
||||
let bucket_name = format!("test-transition-cleanup-journal-{}", &Uuid::new_v4().simple().to_string()[..8]);
|
||||
let object_name = "test/rejected-candidate.bin";
|
||||
let payload = b"rejected remote candidate must not replace the local source".repeat(1024);
|
||||
|
||||
create_test_bucket(&ecstore, bucket_name.as_str()).await;
|
||||
upload_test_object(&ecstore, bucket_name.as_str(), object_name, &payload).await;
|
||||
let original = ecstore
|
||||
.get_object_info(bucket_name.as_str(), object_name, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("source object metadata should resolve before transition");
|
||||
let opts = ObjectOptions {
|
||||
no_lock: true,
|
||||
transition: TransitionOptions {
|
||||
status: TRANSITION_PENDING.to_string(),
|
||||
tier: tier_name,
|
||||
etag: original.etag.clone().expect("uploaded source object should have an ETag"),
|
||||
..Default::default()
|
||||
},
|
||||
version_id: original.version_id.map(|version| version.to_string()),
|
||||
mod_time: original.mod_time,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
ecstore
|
||||
.transition_object(bucket_name.as_str(), object_name, &opts)
|
||||
.await
|
||||
.expect_err("a remotely accepted partial upload must not commit transition metadata");
|
||||
|
||||
let put_versions = backend.put_versions().await;
|
||||
assert_eq!(put_versions.len(), 1, "transition should create exactly one remote candidate");
|
||||
assert!(put_versions[0].1.is_empty());
|
||||
assert_eq!(
|
||||
backend.object_count().await,
|
||||
1,
|
||||
"failed cleanup must retain the remote candidate for recovery"
|
||||
);
|
||||
assert!(
|
||||
backend.remove_versions().await.is_empty(),
|
||||
"no cleanup path may delete the candidate while remove failures are enabled"
|
||||
);
|
||||
|
||||
let retained = recover_tier_delete_journal_entries(ecstore.clone(), 100, None)
|
||||
.await
|
||||
.expect("tier delete journal recovery should scan the persisted candidate");
|
||||
assert_eq!(retained.scanned, 1);
|
||||
assert_eq!(retained.deleted, 0);
|
||||
assert_eq!(retained.failed, 1);
|
||||
assert_eq!(backend.object_count().await, 1, "failed recovery must retain the remote candidate");
|
||||
|
||||
backend.set_remove_failure(false);
|
||||
let recovered = recover_tier_delete_journal_entries(ecstore.clone(), 100, None)
|
||||
.await
|
||||
.expect("tier delete journal recovery should delete the retained candidate");
|
||||
assert_eq!(recovered.scanned, 1);
|
||||
assert_eq!(recovered.deleted, 1);
|
||||
assert_eq!(recovered.failed, 0);
|
||||
let removed_versions = backend.remove_versions().await;
|
||||
assert!(!removed_versions.is_empty(), "recovery must issue at least one successful delete");
|
||||
assert!(
|
||||
removed_versions.iter().all(|removed| removed == &put_versions[0]),
|
||||
"every idempotent cleanup must delete the exact PUT object and version"
|
||||
);
|
||||
assert_eq!(backend.object_count().await, 0, "recovery should remove the rejected remote candidate");
|
||||
|
||||
let empty = recover_tier_delete_journal_entries(ecstore.clone(), 100, None)
|
||||
.await
|
||||
.expect("a removed tier delete journal entry should no longer be listed");
|
||||
assert_eq!(empty.scanned, 0, "successful recovery must remove the persisted journal entry");
|
||||
assert_eq!(
|
||||
read_object_fully(&ecstore, bucket_name.as_str(), object_name).await,
|
||||
payload,
|
||||
"rejected transition cleanup must leave the source object readable"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||
#[serial]
|
||||
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial"]
|
||||
async fn cancelled_before_cleanup_store_resolution_persists_journal() {
|
||||
let (_disk_paths, ecstore) = setup_isolated_test_env(false).await;
|
||||
let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
|
||||
let backend = register_mock_tier(&tier_name).await;
|
||||
backend.set_put_remote_version(Some(Uuid::new_v4().to_string())).await;
|
||||
backend.set_reject_non_empty_remote_versions(true);
|
||||
backend.set_remove_failure(true);
|
||||
let cleanup_store_barrier = TransitionCleanupStoreBarrier::install();
|
||||
|
||||
let bucket_name = format!("test-transition-cancel-cleanup-{}", &Uuid::new_v4().simple().to_string()[..8]);
|
||||
let object_name = "test/rejected-candidate.bin";
|
||||
let payload = b"cancelled rejected cleanup must retain durable recovery evidence".repeat(1024);
|
||||
create_test_bucket(&ecstore, bucket_name.as_str()).await;
|
||||
upload_test_object(&ecstore, bucket_name.as_str(), object_name, &payload).await;
|
||||
let original = ecstore
|
||||
.get_object_info(bucket_name.as_str(), object_name, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("source object metadata should resolve before transition");
|
||||
let opts = ObjectOptions {
|
||||
no_lock: true,
|
||||
transition: TransitionOptions {
|
||||
status: TRANSITION_PENDING.to_string(),
|
||||
tier: tier_name,
|
||||
etag: original.etag.clone().expect("uploaded source object should have an ETag"),
|
||||
..Default::default()
|
||||
},
|
||||
version_id: original.version_id.map(|version| version.to_string()),
|
||||
mod_time: original.mod_time,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let transition_store = ecstore.clone();
|
||||
let transition_bucket = bucket_name.clone();
|
||||
let transition = tokio::spawn(async move {
|
||||
transition_store
|
||||
.transition_object(transition_bucket.as_str(), object_name, &opts)
|
||||
.await
|
||||
});
|
||||
cleanup_store_barrier.wait_until_paused().await;
|
||||
transition.abort();
|
||||
assert!(
|
||||
transition
|
||||
.await
|
||||
.expect_err("aborted transition task should be cancelled")
|
||||
.is_cancelled()
|
||||
);
|
||||
|
||||
let retained = tokio::time::timeout(Duration::from_secs(30), async {
|
||||
loop {
|
||||
let recovery = recover_tier_delete_journal_entries(ecstore.clone(), 100, None)
|
||||
.await
|
||||
.expect("the cancelled transition journal should be readable");
|
||||
if recovery.scanned > 0 {
|
||||
break recovery;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("Drop should persist the rejected candidate through the saved instance context");
|
||||
assert_eq!((retained.scanned, retained.deleted, retained.failed), (1, 0, 1));
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
while backend.exact_remove_count() < 2 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("Drop cleanup and failed journal recovery must both preserve the exact version constraint");
|
||||
let failed_exact_attempts = backend.exact_remove_count();
|
||||
assert_eq!(backend.object_count().await, 1);
|
||||
assert!(backend.remove_versions().await.is_empty());
|
||||
|
||||
backend.set_remove_failure(false);
|
||||
let recovered = recover_tier_delete_journal_entries(ecstore.clone(), 100, None)
|
||||
.await
|
||||
.expect("recovery should delete the candidate retained by the cancelled transition");
|
||||
assert_eq!((recovered.scanned, recovered.deleted, recovered.failed), (1, 1, 0));
|
||||
assert_eq!(backend.remove_versions().await, backend.put_versions().await);
|
||||
assert_eq!(backend.exact_remove_count(), failed_exact_attempts + 1);
|
||||
assert_eq!(backend.object_count().await, 0);
|
||||
let empty = recover_tier_delete_journal_entries(ecstore.clone(), 100, None)
|
||||
.await
|
||||
.expect("successful recovery should remove the cancellation journal");
|
||||
assert_eq!(empty.scanned, 0);
|
||||
assert_eq!(
|
||||
read_object_fully(&ecstore, bucket_name.as_str(), object_name).await,
|
||||
payload,
|
||||
"cancelled transition cleanup must preserve the local source"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||
#[serial]
|
||||
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial"]
|
||||
async fn rejected_transition_cleanup_durability_matrix() {
|
||||
#[derive(Clone, Copy)]
|
||||
enum CleanupCase {
|
||||
Persisted,
|
||||
DeleteFallback,
|
||||
RetryPersisted,
|
||||
FullyFailed,
|
||||
}
|
||||
|
||||
let (_disk_paths, ecstore) = setup_isolated_test_env(false).await;
|
||||
|
||||
for case in [
|
||||
CleanupCase::Persisted,
|
||||
CleanupCase::DeleteFallback,
|
||||
CleanupCase::RetryPersisted,
|
||||
CleanupCase::FullyFailed,
|
||||
] {
|
||||
let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
|
||||
let backend = register_mock_tier(&tier_name).await;
|
||||
let remote_version = Uuid::new_v4().to_string();
|
||||
backend.set_put_remote_version(Some(remote_version.clone())).await;
|
||||
backend.set_reject_non_empty_remote_versions(true);
|
||||
backend.set_remove_failure(matches!(case, CleanupCase::FullyFailed));
|
||||
let put_barrier = backend.arm_put_barrier().await;
|
||||
let remove_barrier = if matches!(case, CleanupCase::RetryPersisted) {
|
||||
Some(backend.arm_failing_remove_barrier().await)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let bucket_name = format!("test-transition-journal-failure-{}", &Uuid::new_v4().simple().to_string()[..8]);
|
||||
let object_name = "test/rejected-candidate.bin";
|
||||
let payload = b"journal failure must either delete the exact candidate or report both failures".repeat(1024);
|
||||
create_test_bucket(&ecstore, bucket_name.as_str()).await;
|
||||
upload_test_object(&ecstore, bucket_name.as_str(), object_name, &payload).await;
|
||||
let original = ecstore
|
||||
.get_object_info(bucket_name.as_str(), object_name, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("source object metadata should resolve before transition");
|
||||
let opts = ObjectOptions {
|
||||
no_lock: true,
|
||||
transition: TransitionOptions {
|
||||
status: TRANSITION_PENDING.to_string(),
|
||||
tier: tier_name,
|
||||
etag: original.etag.clone().expect("uploaded source object should have an ETag"),
|
||||
..Default::default()
|
||||
},
|
||||
version_id: original.version_id.map(|version| version.to_string()),
|
||||
mod_time: original.mod_time,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let transition_store = ecstore.clone();
|
||||
let transition_bucket = bucket_name.clone();
|
||||
let transition = tokio::spawn(async move {
|
||||
transition_store
|
||||
.transition_object(transition_bucket.as_str(), object_name, &opts)
|
||||
.await
|
||||
});
|
||||
put_barrier.wait_until_paused().await;
|
||||
let set = ecstore.pools[0].get_disks(0);
|
||||
let mut saved_disks = if matches!(case, CleanupCase::Persisted) {
|
||||
None
|
||||
} else {
|
||||
let mut disks = set.disks.write().await;
|
||||
let saved = std::mem::take(&mut *disks);
|
||||
*disks = vec![None; saved.len()];
|
||||
Some(saved)
|
||||
};
|
||||
put_barrier.release();
|
||||
if let Some(remove_barrier) = remove_barrier.as_ref() {
|
||||
remove_barrier.wait_until_paused().await;
|
||||
*set.disks.write().await = saved_disks.take().expect("offline disks should be restorable");
|
||||
backend.set_remove_failure(true);
|
||||
remove_barrier.release();
|
||||
}
|
||||
let transition_result = tokio::time::timeout(Duration::from_secs(30), transition).await;
|
||||
if let Some(saved_disks) = saved_disks {
|
||||
*set.disks.write().await = saved_disks;
|
||||
}
|
||||
let err = transition_result
|
||||
.expect("transition should finish while validating cleanup durability")
|
||||
.expect("transition task should not panic")
|
||||
.expect_err("a versioned candidate must not commit to an unversioned tier");
|
||||
|
||||
match case {
|
||||
CleanupCase::Persisted | CleanupCase::DeleteFallback => {
|
||||
assert_eq!(backend.remove_versions().await, backend.put_versions().await);
|
||||
assert_eq!(backend.object_count().await, 0, "cleanup must remove the exact candidate");
|
||||
let recovery = recover_tier_delete_journal_entries(ecstore.clone(), 100, None)
|
||||
.await
|
||||
.expect("successful cleanup must not retain a journal entry");
|
||||
assert_eq!(recovery.scanned, 0);
|
||||
}
|
||||
CleanupCase::RetryPersisted => {
|
||||
assert!(
|
||||
!err.to_string().contains("journal retry error"),
|
||||
"a successful journal retry must preserve the original version-constraint error"
|
||||
);
|
||||
assert_eq!(backend.object_count().await, 1);
|
||||
let retained = recover_tier_delete_journal_entries(ecstore.clone(), 100, None)
|
||||
.await
|
||||
.expect("the retried journal should be recoverable");
|
||||
assert_eq!((retained.scanned, retained.deleted, retained.failed), (1, 0, 1));
|
||||
backend.set_remove_failure(false);
|
||||
let recovered = recover_tier_delete_journal_entries(ecstore.clone(), 100, None)
|
||||
.await
|
||||
.expect("recovery should delete the exact retried candidate");
|
||||
assert_eq!((recovered.scanned, recovered.deleted, recovered.failed), (1, 1, 0));
|
||||
assert_eq!(backend.remove_versions().await, backend.put_versions().await);
|
||||
assert_eq!(backend.object_count().await, 0);
|
||||
let empty = recover_tier_delete_journal_entries(ecstore.clone(), 100, None)
|
||||
.await
|
||||
.expect("successful recovery must remove the retried journal");
|
||||
assert_eq!(empty.scanned, 0);
|
||||
}
|
||||
CleanupCase::FullyFailed => {
|
||||
let message = err.to_string();
|
||||
assert!(message.contains("initial journal error"), "{message}");
|
||||
assert!(message.contains("cleanup error"), "{message}");
|
||||
assert!(message.contains("journal retry error"), "{message}");
|
||||
assert_eq!(backend.object_count().await, 1, "both failed safeguards must leave the candidate visible");
|
||||
assert!(backend.remove_versions().await.is_empty());
|
||||
let recovery = recover_tier_delete_journal_entries(ecstore.clone(), 100, None)
|
||||
.await
|
||||
.expect("failed journal writes must not create partial recovery entries");
|
||||
assert_eq!(recovery.scanned, 0);
|
||||
}
|
||||
}
|
||||
assert_eq!(
|
||||
read_object_fully(&ecstore, bucket_name.as_str(), object_name).await,
|
||||
payload,
|
||||
"rejected transition cleanup must preserve the local source"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||
#[serial]
|
||||
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
|
||||
|
||||
@@ -12,11 +12,10 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
pub(crate) use rustfs_ecstore::api::bucket::lifecycle::tier_delete_journal::recover_tier_delete_journal_entries;
|
||||
pub(crate) use rustfs_ecstore::api::bucket::lifecycle::{
|
||||
bucket_lifecycle_audit::LcEventSrc,
|
||||
bucket_lifecycle_ops::{enqueue_transition_for_existing_objects, expire_transitioned_object, init_background_expiry},
|
||||
lifecycle::{Event as LcEvent, IlmAction, TRANSITION_PENDING, TransitionOptions},
|
||||
lifecycle::{Event as LcEvent, IlmAction, TransitionOptions},
|
||||
};
|
||||
pub(crate) use rustfs_ecstore::api::bucket::metadata::BUCKET_LIFECYCLE_CONFIG;
|
||||
pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::{
|
||||
@@ -33,8 +32,8 @@ pub(crate) use rustfs_ecstore::api::storage::{ECStore, init_local_disks};
|
||||
// backend and xl.meta assertion helpers now live in ecstore behind the
|
||||
// `test-util` feature instead of being copied into this crate.
|
||||
pub(crate) use rustfs_ecstore::api::tier::test_util::{
|
||||
MockWarmBackend, TransitionCleanupStoreBarrier, assert_transition_meta_consistent, free_version_count,
|
||||
register_mock_tier as register_mock_tier_util, wait_for_free_version_absence,
|
||||
MockWarmBackend, assert_transition_meta_consistent, free_version_count, register_mock_tier as register_mock_tier_util,
|
||||
wait_for_free_version_absence,
|
||||
};
|
||||
use rustfs_storage_api as storage_contracts;
|
||||
|
||||
@@ -46,11 +45,10 @@ pub(crate) mod lifecycle {
|
||||
|
||||
pub(crate) use super::{
|
||||
BUCKET_LIFECYCLE_CONFIG, BucketVersioningSys, DiskOption, ECStore, EcstoreError, Endpoint, EndpointServerPools,
|
||||
Endpoints, IlmAction, LcEvent, LcEventSrc, MockWarmBackend, PoolEndpoints, STORAGE_FORMAT_FILE, TRANSITION_PENDING,
|
||||
TransitionCleanupStoreBarrier, TransitionOptions, assert_transition_meta_consistent,
|
||||
enqueue_transition_for_existing_objects, expire_transitioned_object, free_version_count, get_bucket_metadata,
|
||||
get_global_tier_config_mgr, init_background_expiry, init_bucket_metadata_sys, init_local_disks, is_err_object_not_found,
|
||||
is_err_version_not_found, new_disk, path2_bucket_object_with_base_path, recover_tier_delete_journal_entries,
|
||||
Endpoints, IlmAction, LcEvent, LcEventSrc, MockWarmBackend, PoolEndpoints, STORAGE_FORMAT_FILE, TransitionOptions,
|
||||
assert_transition_meta_consistent, enqueue_transition_for_existing_objects, expire_transitioned_object,
|
||||
free_version_count, get_bucket_metadata, get_global_tier_config_mgr, init_background_expiry, init_bucket_metadata_sys,
|
||||
init_local_disks, is_err_object_not_found, is_err_version_not_found, new_disk, path2_bucket_object_with_base_path,
|
||||
register_mock_tier_util, update_bucket_metadata, wait_for_free_version_absence,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -33,7 +33,7 @@ pub struct MakeBucketOptions {
|
||||
}
|
||||
|
||||
/// Operation to perform on a bucket during site replication delete.
|
||||
#[derive(Debug, Default, Clone, PartialEq, Serialize, Deserialize)]
|
||||
#[derive(Debug, Default, Clone, PartialEq)]
|
||||
pub enum SRBucketDeleteOp {
|
||||
/// No operation.
|
||||
#[default]
|
||||
@@ -45,7 +45,7 @@ pub enum SRBucketDeleteOp {
|
||||
}
|
||||
|
||||
/// Options for deleting a bucket.
|
||||
#[derive(Debug, Default, Clone, Serialize, Deserialize)]
|
||||
#[derive(Debug, Default, Clone)]
|
||||
pub struct DeleteBucketOptions {
|
||||
/// Skip acquiring namespace lock.
|
||||
pub no_lock: bool,
|
||||
@@ -53,8 +53,6 @@ pub struct DeleteBucketOptions {
|
||||
pub no_recreate: bool,
|
||||
/// Force deletion even if bucket is not empty.
|
||||
pub force: bool,
|
||||
/// Force deletion only after the local peer verifies the bucket is empty.
|
||||
pub force_if_empty: bool,
|
||||
/// Site replication delete operation.
|
||||
pub srdelete_op: SRBucketDeleteOp,
|
||||
}
|
||||
|
||||
@@ -26,7 +26,7 @@ lapin = { workspace = true, default-features = false, features = ["tokio", "rust
|
||||
libc = { workspace = true }
|
||||
pulsar = { workspace = true, default-features = false, features = ["tokio-rustls-runtime", "telemetry"] }
|
||||
regex = { workspace = true }
|
||||
reqwest = { workspace = true }
|
||||
reqwest = { workspace = true, default-features = false, features = ["rustls", "http2", "system-proxy"] }
|
||||
rumqttc = { workspace = true, features = ["websocket"] }
|
||||
redis = { workspace = true, features = ["connection-manager", "tokio-rustls-comp", "tls-rustls-insecure"] }
|
||||
rustls = { workspace = true, default-features = false, features = ["aws-lc-rs", "logging", "tls12", "prefer-post-quantum", "std"] }
|
||||
|
||||
@@ -896,63 +896,4 @@ mod tests {
|
||||
|
||||
handle.join().expect("mock server thread");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_webhook_client_reaches_https_origin_with_custom_ca() {
|
||||
use rustls::{
|
||||
ServerConfig, ServerConnection, StreamOwned,
|
||||
pki_types::{PrivateKeyDer, PrivatePkcs8KeyDer},
|
||||
};
|
||||
use std::io::{Read, Write};
|
||||
use std::net::TcpListener;
|
||||
use std::sync::{Arc, Once};
|
||||
|
||||
static INSTALL_CRYPTO_PROVIDER: Once = Once::new();
|
||||
INSTALL_CRYPTO_PROVIDER.call_once(|| {
|
||||
let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();
|
||||
});
|
||||
|
||||
let rcgen::CertifiedKey { cert, signing_key } =
|
||||
rcgen::generate_simple_self_signed(vec!["localhost".to_string()]).expect("cert should generate");
|
||||
let temp_dir = tempfile::tempdir().expect("tempdir");
|
||||
let ca_path = temp_dir.path().join("webhook-ca.pem");
|
||||
std::fs::write(&ca_path, cert.pem()).expect("write ca pem");
|
||||
|
||||
let cert_chain = vec![cert.der().clone()];
|
||||
let key_der = PrivateKeyDer::Pkcs8(PrivatePkcs8KeyDer::from(signing_key.serialize_der()));
|
||||
let server_config = Arc::new(
|
||||
ServerConfig::builder()
|
||||
.with_no_client_auth()
|
||||
.with_single_cert(cert_chain, key_der)
|
||||
.expect("server cert should be valid"),
|
||||
);
|
||||
|
||||
let listener = TcpListener::bind("127.0.0.1:0").expect("bind tls server");
|
||||
let addr = listener.local_addr().expect("local addr");
|
||||
let handle = std::thread::spawn(move || {
|
||||
let (stream, _) = listener.accept().expect("accept tls client");
|
||||
let connection = ServerConnection::new(server_config).expect("server connection");
|
||||
let mut tls_stream = StreamOwned::new(connection, stream);
|
||||
let mut buf = [0u8; 1024];
|
||||
let _ = tls_stream.read(&mut buf);
|
||||
let response = "HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok";
|
||||
tls_stream.write_all(response.as_bytes()).expect("write response");
|
||||
tls_stream.flush().expect("flush response");
|
||||
});
|
||||
|
||||
let args = WebhookArgs {
|
||||
client_ca: ca_path.to_string_lossy().into_owned(),
|
||||
..base_args()
|
||||
};
|
||||
let client = WebhookTarget::<serde_json::Value>::build_http_client(&args).expect("build https client");
|
||||
let resp = client
|
||||
.head(format!("https://localhost:{}/hook", addr.port()))
|
||||
.send()
|
||||
.await
|
||||
.expect("https webhook probe should trust configured ca");
|
||||
|
||||
assert_eq!(resp.status(), reqwest::StatusCode::OK);
|
||||
assert_eq!(resp.text_with_charset("utf-8").await.expect("read response body"), "");
|
||||
handle.join().expect("tls server thread");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -31,7 +31,7 @@ http = { workspace = true }
|
||||
ipnetwork = { workspace = true, features = ["serde"] }
|
||||
metrics = { workspace = true }
|
||||
moka = { workspace = true, features = ["future", "sync"] }
|
||||
reqwest = { workspace = true, features = ["json"] }
|
||||
reqwest = { workspace = true, default-features = false, features = ["rustls", "charset", "http2", "system-proxy", "json"] }
|
||||
rustfs-config = { workspace = true }
|
||||
rustfs-utils = { workspace = true, features = ["net"] }
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
|
||||
@@ -41,6 +41,12 @@ allow-git = [
|
||||
# owner: marshawcoco review: 2026-10
|
||||
"https://github.com/s3s-project/s3s.git",
|
||||
"https://github.com/apache/datafusion.git",
|
||||
# hyper pinned (via [patch.crates-io]) to a rev carrying the HTTP/1
|
||||
# flush-before-shutdown fix (hyperium/hyper#4018, commit 72046cc7) that is
|
||||
# not yet in a crates.io release. Fixes sporadic large-GET `unexpected EOF`
|
||||
# (rustfs/backlog#1232). Remove once a released hyper > 1.10.1 contains it.
|
||||
# owner: rustfs-maintainers review: 2026-07
|
||||
"https://github.com/hyperium/hyper.git",
|
||||
]
|
||||
|
||||
[bans]
|
||||
|
||||
@@ -56,7 +56,6 @@ Explicit per-operation overrides always win over the profile, so you can select
|
||||
| Environment variable | Default | `high_latency` default | Bounds |
|
||||
|---|---:|---:|---|
|
||||
| `RUSTFS_DRIVE_WALKDIR_STALL_TIMEOUT_SECS` | `5` | `60` | Max time a single walk filesystem call may go without answering during a listing walk. **The knob for listing failures on large prefixes.** |
|
||||
| `RUSTFS_DRIVE_WALKDIR_PEEK_TIMEOUT_SECS` | `10` | `120` | Max time the metacache merge consumer waits for the next visible entry from a walk reader. Values below the resolved stall timeout are clamped up to the stall timeout. |
|
||||
| `RUSTFS_DRIVE_WALKDIR_TIMEOUT_SECS` | `5` | `60` | Total wall-clock timeout for a `walk_dir`. Retained for non-foreground callers; the foreground listing path no longer uses it (see below). |
|
||||
| `RUSTFS_DRIVE_LIST_DIR_TIMEOUT_SECS` | `5` | `60` | Timeout for a standalone `list_dir` metadata listing. |
|
||||
| `RUSTFS_DRIVE_METADATA_TIMEOUT_SECS` | `5` | `60` | Timeout for metadata reads such as `read_metadata`. |
|
||||
@@ -119,10 +118,6 @@ Raise the walk stall budget, or select the high-latency profile:
|
||||
# widen just the listing walk budget
|
||||
-e RUSTFS_DRIVE_WALKDIR_STALL_TIMEOUT_SECS=60
|
||||
|
||||
# widen the metacache reader wait budget when drives keep progressing but do
|
||||
# not publish visible entries quickly enough for the merge consumer
|
||||
-e RUSTFS_DRIVE_WALKDIR_PEEK_TIMEOUT_SECS=120
|
||||
|
||||
# or raise every profile-aware drive default at once
|
||||
-e RUSTFS_DRIVE_TIMEOUT_PROFILE=high_latency
|
||||
```
|
||||
|
||||
+1
-1
@@ -118,7 +118,7 @@ hyper-util = { workspace = true, features = ["tokio", "server-auto", "server-gra
|
||||
http.workspace = true
|
||||
http-body.workspace = true
|
||||
http-body-util.workspace = true
|
||||
reqwest = { workspace = true, features = ["json", "stream"] }
|
||||
reqwest = { workspace = true, default-features = false, features = ["rustls", "charset", "http2", "system-proxy", "stream"] }
|
||||
socket2 = { workspace = true, features = ["all"] }
|
||||
tokio = { workspace = true, features = ["rt-multi-thread", "macros", "net", "signal", "process", "io-util", "fs"] }
|
||||
tokio-rustls = { workspace = true, default-features = false, features = ["logging", "tls12", "aws-lc-rs"] }
|
||||
|
||||
@@ -28,7 +28,7 @@ use crate::auth::{check_key_valid, get_session_token};
|
||||
use crate::cluster_snapshot::{
|
||||
ClusterReadOnlySnapshot, ClusterRuntimeReadinessState, ClusterRuntimeStatusSnapshot, cluster_has_actionable_pressure,
|
||||
};
|
||||
use crate::server::{ADMIN_PREFIX, ReadinessDegradedReason, RemoteAddr};
|
||||
use crate::server::{ADMIN_PREFIX, RemoteAddr};
|
||||
use http::{HeaderMap, HeaderValue, StatusCode};
|
||||
use hyper::Method;
|
||||
use matchit::Params;
|
||||
@@ -63,7 +63,6 @@ pub(crate) struct ClusterSnapshotDiscoveryResponse {
|
||||
pub rpc_boundary: Option<CapabilityStatus>,
|
||||
pub workload_admission: Option<CapabilityStatus>,
|
||||
pub runtime: Option<CapabilityStatus>,
|
||||
pub components: Option<ClusterComponentStatusView>,
|
||||
}
|
||||
|
||||
async fn authorize_cluster_snapshot_request(req: &S3Request<Body>) -> S3Result<()> {
|
||||
@@ -120,8 +119,7 @@ pub(crate) async fn build_cluster_snapshot_discovery_response() -> ClusterSnapsh
|
||||
|
||||
match snapshot {
|
||||
Some(snapshot) => {
|
||||
let components = ClusterComponentStatusView::from_snapshot(&snapshot);
|
||||
let summary = ClusterSnapshotSummary::from_snapshot_and_components(&snapshot, &components);
|
||||
let summary = ClusterSnapshotSummary::from(&snapshot);
|
||||
ClusterSnapshotDiscoveryResponse {
|
||||
path,
|
||||
summary: Some(summary.actionable_pressure.clone()),
|
||||
@@ -130,7 +128,6 @@ pub(crate) async fn build_cluster_snapshot_discovery_response() -> ClusterSnapsh
|
||||
rpc_boundary: Some(summary.rpc_boundary),
|
||||
workload_admission: Some(summary.workload_admission),
|
||||
runtime: Some(summary.runtime),
|
||||
components: Some(components),
|
||||
}
|
||||
}
|
||||
None => ClusterSnapshotDiscoveryResponse {
|
||||
@@ -141,7 +138,6 @@ pub(crate) async fn build_cluster_snapshot_discovery_response() -> ClusterSnapsh
|
||||
rpc_boundary: None,
|
||||
workload_admission: None,
|
||||
runtime: None,
|
||||
components: None,
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -151,7 +147,6 @@ pub(crate) struct ClusterSnapshotView {
|
||||
pub summary: ClusterSnapshotSummary,
|
||||
pub runtime_capabilities_path: String,
|
||||
pub extensions_catalog_path: String,
|
||||
pub components: ClusterComponentStatusView,
|
||||
pub topology: TopologySnapshot,
|
||||
pub membership: ClusterMembershipView,
|
||||
pub pool_state: ClusterPoolStateView,
|
||||
@@ -166,14 +161,11 @@ pub(crate) struct ClusterSnapshotView {
|
||||
|
||||
impl From<ClusterReadOnlySnapshot> for ClusterSnapshotView {
|
||||
fn from(snapshot: ClusterReadOnlySnapshot) -> Self {
|
||||
let components = ClusterComponentStatusView::from_snapshot(&snapshot);
|
||||
let summary = ClusterSnapshotSummary::from_snapshot_and_components(&snapshot, &components);
|
||||
let actionable_pressure = cluster_has_actionable_pressure(&snapshot);
|
||||
Self {
|
||||
summary,
|
||||
summary: ClusterSnapshotSummary::from(&snapshot),
|
||||
runtime_capabilities_path: format!("{}{}", ADMIN_PREFIX, system::RUNTIME_CAPABILITIES_ROUTE_SUFFIX),
|
||||
extensions_catalog_path: format!("{}{}", ADMIN_PREFIX, "/v4/extensions/catalog"),
|
||||
components,
|
||||
topology: snapshot.topology,
|
||||
membership: ClusterMembershipView::from(snapshot.membership),
|
||||
pool_state: ClusterPoolStateView::from(snapshot.pool_state),
|
||||
@@ -188,93 +180,12 @@ impl From<ClusterReadOnlySnapshot> for ClusterSnapshotView {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
pub(crate) struct ClusterComponentStatusView {
|
||||
pub storage: ClusterComponentStatus,
|
||||
pub peer_health: ClusterComponentStatus,
|
||||
pub listing: ClusterListingMetacacheStatus,
|
||||
pub usage: ClusterUsageFreshnessStatus,
|
||||
pub workload_admission: ClusterComponentStatus,
|
||||
}
|
||||
|
||||
impl ClusterComponentStatusView {
|
||||
fn from_snapshot(snapshot: &ClusterReadOnlySnapshot) -> Self {
|
||||
Self {
|
||||
storage: component_status_with_condition(
|
||||
"runtime_readiness",
|
||||
summarize_storage_readiness(snapshot),
|
||||
if snapshot.runtime_status.readiness.storage_ready {
|
||||
"healthy"
|
||||
} else {
|
||||
"degraded"
|
||||
},
|
||||
),
|
||||
peer_health: component_status("cluster_peer_health", summarize_peer_health(snapshot)),
|
||||
listing: summarize_listing_metacache(snapshot),
|
||||
usage: summarize_usage_freshness(snapshot),
|
||||
workload_admission: component_status("workload_admission", summarize_workload_admission(snapshot)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
pub(crate) struct ClusterComponentStatus {
|
||||
pub source: &'static str,
|
||||
pub condition: &'static str,
|
||||
pub status: CapabilityStatus,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
pub(crate) struct ClusterListingMetacacheStatus {
|
||||
pub source: &'static str,
|
||||
pub condition: &'static str,
|
||||
pub status: CapabilityStatus,
|
||||
pub internode_stall_timeouts_total: u64,
|
||||
pub hint: &'static str,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
pub(crate) struct ClusterUsageFreshnessStatus {
|
||||
pub source: &'static str,
|
||||
pub condition: &'static str,
|
||||
pub status: CapabilityStatus,
|
||||
pub dirty_pending_buckets: u64,
|
||||
pub last_dirty_mark_unix_secs: u64,
|
||||
pub last_dirty_clear_unix_secs: u64,
|
||||
pub last_cycle_dirty_buckets: u64,
|
||||
pub last_cycle_cleared_dirty_buckets: u64,
|
||||
pub last_usage_save_unix_secs: u64,
|
||||
pub last_usage_save_result: String,
|
||||
pub last_success_unix_secs: Option<u64>,
|
||||
pub last_error: Option<String>,
|
||||
}
|
||||
|
||||
fn component_status(source: &'static str, status: CapabilityStatus) -> ClusterComponentStatus {
|
||||
let condition = condition_from_capability(&status);
|
||||
component_status_with_condition(source, status, condition)
|
||||
}
|
||||
|
||||
fn component_status_with_condition(
|
||||
source: &'static str,
|
||||
status: CapabilityStatus,
|
||||
condition: &'static str,
|
||||
) -> ClusterComponentStatus {
|
||||
ClusterComponentStatus {
|
||||
source,
|
||||
condition,
|
||||
status,
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
pub(crate) struct ClusterSnapshotSummary {
|
||||
pub runtime: CapabilityStatus,
|
||||
pub topology: CapabilityStatus,
|
||||
pub membership: CapabilityStatus,
|
||||
pub storage: CapabilityStatus,
|
||||
pub peer_health: CapabilityStatus,
|
||||
pub listing: CapabilityStatus,
|
||||
pub usage: CapabilityStatus,
|
||||
pub rpc_boundary: CapabilityStatus,
|
||||
pub observability: CapabilityStatus,
|
||||
pub workload_admission: CapabilityStatus,
|
||||
@@ -283,17 +194,12 @@ pub(crate) struct ClusterSnapshotSummary {
|
||||
|
||||
impl From<&ClusterReadOnlySnapshot> for ClusterSnapshotSummary {
|
||||
fn from(snapshot: &ClusterReadOnlySnapshot) -> Self {
|
||||
let components = ClusterComponentStatusView::from_snapshot(snapshot);
|
||||
Self::from_snapshot_and_components(snapshot, &components)
|
||||
}
|
||||
}
|
||||
|
||||
impl ClusterSnapshotSummary {
|
||||
fn from_snapshot_and_components(snapshot: &ClusterReadOnlySnapshot, components: &ClusterComponentStatusView) -> Self {
|
||||
let topology = summarize_topology(snapshot);
|
||||
let membership = summarize_membership(snapshot);
|
||||
let peer_health = summarize_peer_health(snapshot);
|
||||
let rpc_boundary = summarize_rpc_boundary(snapshot);
|
||||
let observability = summarize_observability(snapshot);
|
||||
let workload_admission = summarize_workload_admission(snapshot);
|
||||
let actionable_pressure = if cluster_has_actionable_pressure(snapshot) {
|
||||
CapabilityStatus::supported().with_reason("cluster snapshot reports degraded runtime or non-open admission")
|
||||
} else {
|
||||
@@ -304,13 +210,10 @@ impl ClusterSnapshotSummary {
|
||||
runtime: summarize_runtime(snapshot),
|
||||
topology,
|
||||
membership,
|
||||
storage: components.storage.status.clone(),
|
||||
peer_health: components.peer_health.status.clone(),
|
||||
listing: components.listing.status.clone(),
|
||||
usage: components.usage.status.clone(),
|
||||
peer_health,
|
||||
rpc_boundary,
|
||||
observability,
|
||||
workload_admission: components.workload_admission.status.clone(),
|
||||
workload_admission,
|
||||
actionable_pressure,
|
||||
}
|
||||
}
|
||||
@@ -608,130 +511,6 @@ fn runtime_readiness_state_label(state: ClusterRuntimeReadinessState) -> &'stati
|
||||
}
|
||||
}
|
||||
|
||||
fn condition_from_capability(status: &CapabilityStatus) -> &'static str {
|
||||
match status.state {
|
||||
CapabilityState::Supported => "healthy",
|
||||
CapabilityState::Unsupported => "unsupported",
|
||||
CapabilityState::Disabled => "not_reported",
|
||||
CapabilityState::Unknown => "unknown",
|
||||
}
|
||||
}
|
||||
|
||||
fn summarize_storage_readiness(snapshot: &ClusterReadOnlySnapshot) -> CapabilityStatus {
|
||||
if snapshot.runtime_status.readiness.storage_ready {
|
||||
return CapabilityStatus::supported().with_reason("storage readiness reports ready");
|
||||
}
|
||||
|
||||
let storage_reasons = snapshot
|
||||
.runtime_status
|
||||
.degraded_reasons
|
||||
.iter()
|
||||
.filter_map(|reason| match reason {
|
||||
ReadinessDegradedReason::StorageQuorumUnavailable
|
||||
| ReadinessDegradedReason::StorageAndIamUnavailable
|
||||
| ReadinessDegradedReason::StorageAndLockUnavailable
|
||||
| ReadinessDegradedReason::StorageIamAndLockUnavailable => Some(reason.as_str()),
|
||||
_ => None,
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
if storage_reasons.is_empty() {
|
||||
CapabilityStatus::unknown().with_reason("storage readiness is false without a storage-specific degraded reason")
|
||||
} else {
|
||||
CapabilityStatus::unknown().with_reason(format!("storage readiness degraded: {}", storage_reasons.join(", ")))
|
||||
}
|
||||
}
|
||||
|
||||
fn summarize_listing_readiness(snapshot: &ClusterReadOnlySnapshot) -> CapabilityStatus {
|
||||
match snapshot
|
||||
.workload_admission
|
||||
.entries()
|
||||
.iter()
|
||||
.find(|entry| entry.class == WorkloadClass::ForegroundRead)
|
||||
{
|
||||
Some(entry) if entry.state == AdmissionState::Open => {
|
||||
CapabilityStatus::supported().with_reason("foreground read admission is open")
|
||||
}
|
||||
Some(entry) => {
|
||||
let mut reason = format!("foreground read admission is {}", admission_state_label(entry.state));
|
||||
if let Some(detail) = entry.reason.as_deref() {
|
||||
reason.push_str(": ");
|
||||
reason.push_str(detail);
|
||||
}
|
||||
CapabilityStatus::unknown().with_reason(reason)
|
||||
}
|
||||
None => CapabilityStatus::unknown().with_reason("foreground read admission is not reported"),
|
||||
}
|
||||
}
|
||||
|
||||
fn summarize_listing_metacache(snapshot: &ClusterReadOnlySnapshot) -> ClusterListingMetacacheStatus {
|
||||
const HINT: &str = "bucket and prefix attribution is emitted through metacache quorum logs; inspect operation-labelled walk_dir metrics for internode traffic";
|
||||
|
||||
let admission = summarize_listing_readiness(snapshot);
|
||||
let stalls = snapshot.listing_diagnostics.internode_stall_timeouts_total;
|
||||
let (condition, status) = if admission.state == CapabilityState::Supported {
|
||||
("healthy", admission)
|
||||
} else {
|
||||
(condition_from_capability(&admission), admission)
|
||||
};
|
||||
|
||||
ClusterListingMetacacheStatus {
|
||||
source: "workload_admission+internode_metrics",
|
||||
condition,
|
||||
status,
|
||||
internode_stall_timeouts_total: stalls,
|
||||
hint: HINT,
|
||||
}
|
||||
}
|
||||
|
||||
fn summarize_usage_freshness(snapshot: &ClusterReadOnlySnapshot) -> ClusterUsageFreshnessStatus {
|
||||
let freshness = &snapshot.usage_freshness;
|
||||
let (condition, status) = match freshness.last_usage_save_result.as_str() {
|
||||
"success" if freshness.dirty_pending_buckets == 0 => (
|
||||
"healthy",
|
||||
CapabilityStatus::supported().with_reason("usage cache was saved successfully and has no pending dirty buckets"),
|
||||
),
|
||||
"success" | "" if freshness.dirty_pending_buckets > 0 => (
|
||||
"stale",
|
||||
CapabilityStatus::unknown()
|
||||
.with_reason(format!("usage cache has {} pending dirty buckets", freshness.dirty_pending_buckets)),
|
||||
),
|
||||
"skipped_stale" => (
|
||||
"stale",
|
||||
CapabilityStatus::unknown().with_reason("last usage cache save was skipped because scanner data was stale"),
|
||||
),
|
||||
"failed" => ("degraded", CapabilityStatus::unknown().with_reason("last usage cache save failed")),
|
||||
"encode_failed" => (
|
||||
"degraded",
|
||||
CapabilityStatus::unknown().with_reason("last usage cache save failed during encoding"),
|
||||
),
|
||||
_ => (
|
||||
"unknown",
|
||||
CapabilityStatus::unknown().with_reason("no usage cache save result has been reported"),
|
||||
),
|
||||
};
|
||||
let last_success_unix_secs = (freshness.last_usage_save_result == "success" && freshness.last_usage_save_unix_secs > 0)
|
||||
.then_some(freshness.last_usage_save_unix_secs);
|
||||
let last_error = match freshness.last_usage_save_result.as_str() {
|
||||
"failed" | "skipped_stale" | "encode_failed" => Some(freshness.last_usage_save_result.clone()),
|
||||
_ => None,
|
||||
};
|
||||
|
||||
ClusterUsageFreshnessStatus {
|
||||
source: "scanner_metrics",
|
||||
condition,
|
||||
status,
|
||||
dirty_pending_buckets: freshness.dirty_pending_buckets,
|
||||
last_dirty_mark_unix_secs: freshness.last_dirty_mark_unix_secs,
|
||||
last_dirty_clear_unix_secs: freshness.last_dirty_clear_unix_secs,
|
||||
last_cycle_dirty_buckets: freshness.last_cycle_dirty_buckets,
|
||||
last_cycle_cleared_dirty_buckets: freshness.last_cycle_cleared_dirty_buckets,
|
||||
last_usage_save_unix_secs: freshness.last_usage_save_unix_secs,
|
||||
last_usage_save_result: freshness.last_usage_save_result.clone(),
|
||||
last_success_unix_secs,
|
||||
last_error,
|
||||
}
|
||||
}
|
||||
|
||||
fn summarize_runtime(snapshot: &ClusterReadOnlySnapshot) -> CapabilityStatus {
|
||||
match snapshot.runtime_status.state {
|
||||
ClusterRuntimeReadinessState::Ready => CapabilityStatus::supported().with_reason("runtime readiness reports ready"),
|
||||
@@ -785,17 +564,8 @@ fn summarize_peer_health(snapshot: &ClusterReadOnlySnapshot) -> CapabilityStatus
|
||||
.iter()
|
||||
.filter(|peer| peer.status.state == CapabilityState::Unknown)
|
||||
.count();
|
||||
let disabled = snapshot
|
||||
.peer_health
|
||||
.peers
|
||||
.iter()
|
||||
.filter(|peer| peer.status.state == CapabilityState::Disabled)
|
||||
.count();
|
||||
if unknown > 0 {
|
||||
CapabilityStatus::unknown().with_reason(format!("cluster peer health has {unknown} unresolved peers"))
|
||||
} else if disabled > 0 {
|
||||
let noun = if disabled == 1 { "peer" } else { "peers" };
|
||||
CapabilityStatus::disabled().with_reason(format!("cluster peer health is not reported by {disabled} {noun}"))
|
||||
} else {
|
||||
CapabilityStatus::supported().with_reason("cluster peer health resolved for all peers")
|
||||
}
|
||||
@@ -899,10 +669,7 @@ mod tests {
|
||||
ClusterMembershipSnapshot, ClusterNodeMembership, ClusterPeerHealth, ClusterPeerHealthSnapshot, ClusterPoolState,
|
||||
ClusterPoolStateSnapshot, ClusterRpcBoundarySnapshot, ClusterRpcChannelSnapshot, ClusterRpcPlane, ClusterRpcTransport,
|
||||
};
|
||||
use crate::cluster_snapshot::{
|
||||
ClusterListingDiagnosticsSnapshot, ClusterReadOnlySnapshot, ClusterRuntimeReadinessState, ClusterRuntimeStatusSnapshot,
|
||||
ClusterUsageFreshnessSnapshot,
|
||||
};
|
||||
use crate::cluster_snapshot::{ClusterReadOnlySnapshot, ClusterRuntimeReadinessState, ClusterRuntimeStatusSnapshot};
|
||||
use crate::server::{DependencyReadiness, ReadinessDegradedReason};
|
||||
use rustfs_concurrency::{AdmissionState, WorkloadAdmissionRegistrySnapshot, WorkloadAdmissionSnapshot, WorkloadClass};
|
||||
|
||||
@@ -940,7 +707,6 @@ mod tests {
|
||||
assert_eq!(response.rpc_boundary, None);
|
||||
assert_eq!(response.workload_admission, None);
|
||||
assert_eq!(response.runtime, None);
|
||||
assert_eq!(response.components, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1008,29 +774,11 @@ mod tests {
|
||||
state: ClusterRuntimeReadinessState::Degraded,
|
||||
degraded_reasons: vec![ReadinessDegradedReason::StorageAndLockUnavailable],
|
||||
},
|
||||
usage_freshness: ClusterUsageFreshnessSnapshot {
|
||||
dirty_pending_buckets: 3,
|
||||
last_usage_save_unix_secs: 123,
|
||||
last_usage_save_result: "skipped_stale".to_string(),
|
||||
last_usage_save_result_code: 3,
|
||||
..Default::default()
|
||||
},
|
||||
listing_diagnostics: ClusterListingDiagnosticsSnapshot {
|
||||
internode_stall_timeouts_total: 2,
|
||||
},
|
||||
};
|
||||
|
||||
let value = serde_json::to_value(ClusterSnapshotView::from(snapshot)).expect("serialize view");
|
||||
assert_eq!(value["runtime_capabilities_path"], "/rustfs/admin/v4/runtime/capabilities");
|
||||
assert_eq!(value["extensions_catalog_path"], "/rustfs/admin/v4/extensions/catalog");
|
||||
assert_eq!(value["components"]["storage"]["source"], "runtime_readiness");
|
||||
assert_eq!(value["components"]["storage"]["condition"], "degraded");
|
||||
assert_eq!(value["components"]["peer_health"]["source"], "cluster_peer_health");
|
||||
assert_eq!(value["components"]["listing"]["source"], "workload_admission+internode_metrics");
|
||||
assert_eq!(value["components"]["listing"]["condition"], "unknown");
|
||||
assert_eq!(value["components"]["listing"]["internode_stall_timeouts_total"], 2);
|
||||
assert_eq!(value["components"]["usage"]["source"], "scanner_metrics");
|
||||
assert_eq!(value["components"]["usage"]["condition"], "stale");
|
||||
assert_eq!(value["membership"]["drives"][0]["endpoint_type"], "url");
|
||||
assert_eq!(value["workload_admission"][0]["class"], "repair");
|
||||
assert_eq!(value["workload_admission"][0]["state"], "unknown");
|
||||
@@ -1068,255 +816,17 @@ mod tests {
|
||||
state: ClusterRuntimeReadinessState::Ready,
|
||||
degraded_reasons: Vec::new(),
|
||||
},
|
||||
usage_freshness: ClusterUsageFreshnessSnapshot::default(),
|
||||
listing_diagnostics: ClusterListingDiagnosticsSnapshot::default(),
|
||||
};
|
||||
|
||||
let summary = ClusterSnapshotSummary::from(&snapshot);
|
||||
assert_eq!(summary.runtime.state, CapabilityState::Supported);
|
||||
assert_eq!(summary.membership.state, CapabilityState::Unknown);
|
||||
assert_eq!(summary.storage.state, CapabilityState::Supported);
|
||||
assert_eq!(summary.storage.reason.as_deref(), Some("storage readiness reports ready"));
|
||||
assert_eq!(summary.peer_health.state, CapabilityState::Unknown);
|
||||
assert_eq!(summary.listing.state, CapabilityState::Supported);
|
||||
assert_eq!(summary.listing.reason.as_deref(), Some("foreground read admission is open"));
|
||||
assert_eq!(summary.usage.state, CapabilityState::Unknown);
|
||||
assert_eq!(summary.usage.reason.as_deref(), Some("no usage cache save result has been reported"));
|
||||
assert_eq!(summary.rpc_boundary.state, CapabilityState::Supported);
|
||||
assert_eq!(summary.workload_admission.state, CapabilityState::Supported);
|
||||
assert_eq!(summary.actionable_pressure.state, CapabilityState::Disabled);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cluster_snapshot_components_split_storage_peer_listing_and_usage_states() {
|
||||
let snapshot = ClusterReadOnlySnapshot {
|
||||
topology: TopologySnapshot::default(),
|
||||
membership: ClusterMembershipSnapshot {
|
||||
nodes: vec![ClusterNodeMembership {
|
||||
node_id: "node-a".to_string(),
|
||||
grid_host: "node-a:9000".to_string(),
|
||||
is_local: true,
|
||||
pools: vec![0],
|
||||
}],
|
||||
drives: Vec::new(),
|
||||
},
|
||||
pool_state: ClusterPoolStateSnapshot::default(),
|
||||
local_storage: ClusterLocalNodeStorageSnapshot::default(),
|
||||
peer_health: ClusterPeerHealthSnapshot {
|
||||
peers: vec![ClusterPeerHealth {
|
||||
node_id: "node-a".to_string(),
|
||||
is_local: true,
|
||||
status: CapabilityStatus::disabled().with_reason("peer health not reported by endpoints"),
|
||||
}],
|
||||
},
|
||||
rpc_boundary: sample_rpc_boundary_snapshot(),
|
||||
observability: ObservabilitySnapshot::default(),
|
||||
workload_admission: WorkloadAdmissionRegistrySnapshot::new(vec![
|
||||
WorkloadAdmissionSnapshot::new(WorkloadClass::ForegroundRead, AdmissionState::Throttled)
|
||||
.with_counts(Some(4), Some(8), Some(4))
|
||||
.with_reason("metacache listing backpressure"),
|
||||
]),
|
||||
runtime_status: ClusterRuntimeStatusSnapshot {
|
||||
readiness: DependencyReadiness {
|
||||
storage_ready: false,
|
||||
iam_ready: true,
|
||||
lock_quorum_ready: true,
|
||||
peer_health_ready: true,
|
||||
},
|
||||
state: ClusterRuntimeReadinessState::Degraded,
|
||||
degraded_reasons: vec![ReadinessDegradedReason::StorageQuorumUnavailable],
|
||||
},
|
||||
usage_freshness: ClusterUsageFreshnessSnapshot {
|
||||
dirty_pending_buckets: 3,
|
||||
last_usage_save_unix_secs: 123,
|
||||
last_usage_save_result: "skipped_stale".to_string(),
|
||||
last_usage_save_result_code: 3,
|
||||
..Default::default()
|
||||
},
|
||||
listing_diagnostics: ClusterListingDiagnosticsSnapshot {
|
||||
internode_stall_timeouts_total: 0,
|
||||
},
|
||||
};
|
||||
|
||||
let view = ClusterSnapshotView::from(snapshot);
|
||||
|
||||
assert_eq!(view.components.storage.status.state, CapabilityState::Unknown);
|
||||
assert_eq!(
|
||||
view.components.storage.status.reason.as_deref(),
|
||||
Some("storage readiness degraded: storage_quorum_unavailable")
|
||||
);
|
||||
assert_eq!(view.components.storage.condition, "degraded");
|
||||
assert_eq!(view.components.peer_health.status.state, CapabilityState::Disabled);
|
||||
assert_eq!(view.components.peer_health.condition, "not_reported");
|
||||
assert_eq!(view.components.listing.status.state, CapabilityState::Unknown);
|
||||
assert_eq!(
|
||||
view.components.listing.status.reason.as_deref(),
|
||||
Some("foreground read admission is throttled: metacache listing backpressure")
|
||||
);
|
||||
assert_eq!(view.components.listing.condition, "unknown");
|
||||
assert_eq!(view.components.usage.status.state, CapabilityState::Unknown);
|
||||
assert_eq!(view.components.usage.condition, "stale");
|
||||
assert_eq!(view.components.usage.last_usage_save_unix_secs, 123);
|
||||
assert_eq!(view.components.usage.last_usage_save_result, "skipped_stale");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cluster_snapshot_listing_component_keeps_historical_stalls_as_evidence() {
|
||||
let snapshot = ClusterReadOnlySnapshot {
|
||||
topology: TopologySnapshot::default(),
|
||||
membership: ClusterMembershipSnapshot::default(),
|
||||
pool_state: ClusterPoolStateSnapshot::default(),
|
||||
local_storage: ClusterLocalNodeStorageSnapshot::default(),
|
||||
peer_health: ClusterPeerHealthSnapshot::default(),
|
||||
rpc_boundary: ClusterRpcBoundarySnapshot::default(),
|
||||
observability: ObservabilitySnapshot::default(),
|
||||
workload_admission: WorkloadAdmissionRegistrySnapshot::new(vec![WorkloadAdmissionSnapshot::new(
|
||||
WorkloadClass::ForegroundRead,
|
||||
AdmissionState::Open,
|
||||
)]),
|
||||
runtime_status: ClusterRuntimeStatusSnapshot {
|
||||
readiness: DependencyReadiness::default(),
|
||||
state: ClusterRuntimeReadinessState::Unknown,
|
||||
degraded_reasons: Vec::new(),
|
||||
},
|
||||
usage_freshness: ClusterUsageFreshnessSnapshot::default(),
|
||||
listing_diagnostics: ClusterListingDiagnosticsSnapshot {
|
||||
internode_stall_timeouts_total: 2,
|
||||
},
|
||||
};
|
||||
|
||||
let component = super::summarize_listing_metacache(&snapshot);
|
||||
|
||||
assert_eq!(component.status.state, CapabilityState::Supported);
|
||||
assert_eq!(component.status.reason.as_deref(), Some("foreground read admission is open"));
|
||||
assert_eq!(component.condition, "healthy");
|
||||
assert_eq!(component.internode_stall_timeouts_total, 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cluster_component_condition_preserves_capability_states() {
|
||||
assert_eq!(super::condition_from_capability(&CapabilityStatus::supported()), "healthy");
|
||||
assert_eq!(super::condition_from_capability(&CapabilityStatus::unsupported()), "unsupported");
|
||||
assert_eq!(super::condition_from_capability(&CapabilityStatus::disabled()), "not_reported");
|
||||
assert_eq!(super::condition_from_capability(&CapabilityStatus::unknown()), "unknown");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cluster_snapshot_usage_component_reports_clean_success() {
|
||||
let snapshot = ClusterReadOnlySnapshot {
|
||||
topology: TopologySnapshot::default(),
|
||||
membership: ClusterMembershipSnapshot::default(),
|
||||
pool_state: ClusterPoolStateSnapshot::default(),
|
||||
local_storage: ClusterLocalNodeStorageSnapshot::default(),
|
||||
peer_health: ClusterPeerHealthSnapshot::default(),
|
||||
rpc_boundary: ClusterRpcBoundarySnapshot::default(),
|
||||
observability: ObservabilitySnapshot::default(),
|
||||
workload_admission: WorkloadAdmissionRegistrySnapshot::default(),
|
||||
runtime_status: ClusterRuntimeStatusSnapshot {
|
||||
readiness: DependencyReadiness::default(),
|
||||
state: ClusterRuntimeReadinessState::Unknown,
|
||||
degraded_reasons: Vec::new(),
|
||||
},
|
||||
usage_freshness: ClusterUsageFreshnessSnapshot {
|
||||
dirty_pending_buckets: 0,
|
||||
last_usage_save_unix_secs: 456,
|
||||
last_usage_save_result: "success".to_string(),
|
||||
last_usage_save_result_code: 1,
|
||||
..Default::default()
|
||||
},
|
||||
listing_diagnostics: ClusterListingDiagnosticsSnapshot::default(),
|
||||
};
|
||||
|
||||
let component = super::summarize_usage_freshness(&snapshot);
|
||||
|
||||
assert_eq!(component.status.state, CapabilityState::Supported);
|
||||
assert_eq!(component.condition, "healthy");
|
||||
assert_eq!(component.last_usage_save_unix_secs, 456);
|
||||
assert_eq!(component.last_usage_save_result, "success");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cluster_snapshot_usage_component_reports_dirty_without_save_result_as_stale() {
|
||||
let snapshot = ClusterReadOnlySnapshot {
|
||||
topology: TopologySnapshot::default(),
|
||||
membership: ClusterMembershipSnapshot::default(),
|
||||
pool_state: ClusterPoolStateSnapshot::default(),
|
||||
local_storage: ClusterLocalNodeStorageSnapshot::default(),
|
||||
peer_health: ClusterPeerHealthSnapshot::default(),
|
||||
rpc_boundary: ClusterRpcBoundarySnapshot::default(),
|
||||
observability: ObservabilitySnapshot::default(),
|
||||
workload_admission: WorkloadAdmissionRegistrySnapshot::default(),
|
||||
runtime_status: ClusterRuntimeStatusSnapshot {
|
||||
readiness: DependencyReadiness::default(),
|
||||
state: ClusterRuntimeReadinessState::Unknown,
|
||||
degraded_reasons: Vec::new(),
|
||||
},
|
||||
usage_freshness: ClusterUsageFreshnessSnapshot {
|
||||
dirty_pending_buckets: 2,
|
||||
..Default::default()
|
||||
},
|
||||
listing_diagnostics: ClusterListingDiagnosticsSnapshot::default(),
|
||||
};
|
||||
|
||||
let component = super::summarize_usage_freshness(&snapshot);
|
||||
|
||||
assert_eq!(component.status.state, CapabilityState::Unknown);
|
||||
assert_eq!(component.condition, "stale");
|
||||
assert_eq!(component.status.reason.as_deref(), Some("usage cache has 2 pending dirty buckets"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cluster_snapshot_summary_treats_not_reported_peer_health_as_disabled() {
|
||||
let snapshot = ClusterReadOnlySnapshot {
|
||||
topology: TopologySnapshot::default(),
|
||||
membership: ClusterMembershipSnapshot {
|
||||
nodes: vec![ClusterNodeMembership {
|
||||
node_id: "node-a".to_string(),
|
||||
grid_host: "node-a:9000".to_string(),
|
||||
is_local: true,
|
||||
pools: vec![0],
|
||||
}],
|
||||
drives: Vec::new(),
|
||||
},
|
||||
pool_state: ClusterPoolStateSnapshot::default(),
|
||||
local_storage: ClusterLocalNodeStorageSnapshot::default(),
|
||||
peer_health: ClusterPeerHealthSnapshot {
|
||||
peers: vec![ClusterPeerHealth {
|
||||
node_id: "node-a".to_string(),
|
||||
is_local: true,
|
||||
status: CapabilityStatus::disabled().with_reason("peer health not reported by endpoints"),
|
||||
}],
|
||||
},
|
||||
rpc_boundary: sample_rpc_boundary_snapshot(),
|
||||
observability: ObservabilitySnapshot::default(),
|
||||
workload_admission: WorkloadAdmissionRegistrySnapshot::new(vec![WorkloadAdmissionSnapshot::new(
|
||||
WorkloadClass::ForegroundRead,
|
||||
AdmissionState::Open,
|
||||
)]),
|
||||
runtime_status: ClusterRuntimeStatusSnapshot {
|
||||
readiness: DependencyReadiness {
|
||||
storage_ready: true,
|
||||
iam_ready: true,
|
||||
lock_quorum_ready: true,
|
||||
peer_health_ready: true,
|
||||
},
|
||||
state: ClusterRuntimeReadinessState::Ready,
|
||||
degraded_reasons: Vec::new(),
|
||||
},
|
||||
usage_freshness: ClusterUsageFreshnessSnapshot::default(),
|
||||
listing_diagnostics: ClusterListingDiagnosticsSnapshot::default(),
|
||||
};
|
||||
|
||||
let summary = ClusterSnapshotSummary::from(&snapshot);
|
||||
|
||||
assert_eq!(summary.peer_health.state, CapabilityState::Disabled);
|
||||
assert_eq!(
|
||||
summary.peer_health.reason.as_deref(),
|
||||
Some("cluster peer health is not reported by 1 peer")
|
||||
);
|
||||
assert_eq!(summary.actionable_pressure.state, CapabilityState::Disabled);
|
||||
}
|
||||
|
||||
fn sample_rpc_boundary_snapshot() -> ClusterRpcBoundarySnapshot {
|
||||
ClusterRpcBoundarySnapshot {
|
||||
control_channels: vec![ClusterRpcChannelSnapshot {
|
||||
|
||||
@@ -371,7 +371,7 @@ impl Operation for ListTargetsArns {
|
||||
log_notification_target_operation_blocked!("list_target_arns", None, None, &reason);
|
||||
return Err(s3_error!(InvalidRequest, "{reason}"));
|
||||
}
|
||||
let ns = get_notification_system().await?;
|
||||
let ns = get_notification_system()?;
|
||||
|
||||
let region = req.region.clone().ok_or_else(|| {
|
||||
warn!(
|
||||
|
||||
@@ -23,9 +23,7 @@ pub(crate) fn iam_error_to_s3_error(err: IamError) -> S3Error {
|
||||
| IamError::NoSuchTempAccount(_)
|
||||
| IamError::NoSuchGroup(_)
|
||||
| IamError::NoSuchPolicy => S3ErrorCode::NoSuchResource,
|
||||
IamError::InvalidAccessKeyLength | IamError::InvalidSecretKeyLength | IamError::AccessKeyAlreadyExists => {
|
||||
S3ErrorCode::InvalidArgument
|
||||
}
|
||||
IamError::InvalidAccessKeyLength | IamError::InvalidSecretKeyLength => S3ErrorCode::InvalidArgument,
|
||||
_ => S3ErrorCode::InternalError,
|
||||
};
|
||||
|
||||
@@ -66,15 +64,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn duplicate_access_key_maps_to_invalid_argument() {
|
||||
let s3_error = iam_error_to_s3_error(IamError::AccessKeyAlreadyExists);
|
||||
|
||||
assert_eq!(s3_error.code(), &S3ErrorCode::InvalidArgument);
|
||||
assert_eq!(s3_error.status_code(), Some(http::StatusCode::BAD_REQUEST));
|
||||
assert_eq!(s3_error.message(), Some("access key is already in use"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn non_validation_iam_errors_remain_internal_errors() {
|
||||
let s3_error = iam_error_to_s3_error(IamError::IamSysNotInitialized);
|
||||
|
||||
@@ -12,30 +12,16 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::server::init_event_notifier;
|
||||
use rustfs_config::server_config::Config;
|
||||
use s3s::{S3Result, s3_error};
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::Mutex;
|
||||
|
||||
static NOTIFICATION_SYSTEM_INIT_LOCK: Mutex<()> = Mutex::const_new(());
|
||||
|
||||
pub(crate) async fn get_notification_system() -> S3Result<Arc<rustfs_notify::NotificationSystem>> {
|
||||
if let Some(system) = rustfs_notify::notification_system() {
|
||||
return Ok(system);
|
||||
}
|
||||
|
||||
let _guard = NOTIFICATION_SYSTEM_INIT_LOCK.lock().await;
|
||||
if let Some(system) = rustfs_notify::notification_system() {
|
||||
return Ok(system);
|
||||
}
|
||||
|
||||
init_event_notifier().await;
|
||||
pub(crate) fn get_notification_system() -> S3Result<Arc<rustfs_notify::NotificationSystem>> {
|
||||
rustfs_notify::notification_system().ok_or_else(|| s3_error!(InternalError, "notification system not initialized"))
|
||||
}
|
||||
|
||||
pub(crate) async fn load_notification_config_snapshot() -> S3Result<(Arc<rustfs_notify::NotificationSystem>, Config)> {
|
||||
let system = get_notification_system().await?;
|
||||
let system = get_notification_system()?;
|
||||
let config = system.config.read().await.clone();
|
||||
Ok((system, config))
|
||||
}
|
||||
@@ -45,7 +31,7 @@ pub(crate) async fn set_notification_target_config(
|
||||
target_name: &str,
|
||||
kvs: rustfs_config::server_config::KVS,
|
||||
) -> S3Result<()> {
|
||||
let system = get_notification_system().await?;
|
||||
let system = get_notification_system()?;
|
||||
system
|
||||
.set_target_config(subsystem, target_name, kvs)
|
||||
.await
|
||||
@@ -53,7 +39,7 @@ pub(crate) async fn set_notification_target_config(
|
||||
}
|
||||
|
||||
pub(crate) async fn remove_notification_target_config(subsystem: &str, target_name: &str) -> S3Result<()> {
|
||||
let system = get_notification_system().await?;
|
||||
let system = get_notification_system()?;
|
||||
system
|
||||
.remove_target_config(subsystem, target_name)
|
||||
.await
|
||||
|
||||
@@ -398,9 +398,9 @@ impl Operation for AddServiceAccount {
|
||||
"admin service account state"
|
||||
);
|
||||
match e {
|
||||
rustfs_iam::error::Error::InvalidAccessKeyLength
|
||||
| rustfs_iam::error::Error::InvalidSecretKeyLength
|
||||
| rustfs_iam::error::Error::AccessKeyAlreadyExists => iam_error_to_s3_error(e),
|
||||
rustfs_iam::error::Error::InvalidAccessKeyLength | rustfs_iam::error::Error::InvalidSecretKeyLength => {
|
||||
iam_error_to_s3_error(e)
|
||||
}
|
||||
err => s3_error!(InternalError, "create service account failed, e: {:?}", err),
|
||||
}
|
||||
})?;
|
||||
@@ -1580,8 +1580,8 @@ mod tests {
|
||||
let create_block = &create_block[..create_end];
|
||||
|
||||
assert!(
|
||||
create_block.contains("AccessKeyAlreadyExists") && create_block.contains("iam_error_to_s3_error(e)"),
|
||||
"duplicate access keys must map to client S3 errors"
|
||||
create_block.contains("iam_error_to_s3_error(e)"),
|
||||
"service-account create validation errors must map to client S3 errors"
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -21,8 +21,8 @@ use crate::admin::runtime_sources::{
|
||||
current_token_signing_key, object_store_from_req,
|
||||
};
|
||||
use crate::admin::site_replication_identity::{
|
||||
canonical_endpoint, deployment_id_for_endpoint, is_https_endpoint, mark_unknown_peer_sync_enabled,
|
||||
normalize_peer_map_by_identity_with, same_identity_endpoint, site_identity_key,
|
||||
canonical_endpoint, deployment_id_for_endpoint, normalize_peer_map_by_identity_with, same_identity_endpoint,
|
||||
site_identity_key,
|
||||
};
|
||||
use crate::admin::storage_api::bucket::metadata::{
|
||||
BUCKET_CORS_CONFIG, BUCKET_LIFECYCLE_CONFIG, BUCKET_POLICY_CONFIG, BUCKET_QUOTA_CONFIG_FILE, BUCKET_REPLICATION_CONFIG,
|
||||
@@ -119,7 +119,6 @@ const SITE_REPL_MAX_NETPERF_DURATION: Duration = Duration::from_secs(30);
|
||||
const SITE_REPLICATION_PEER_REQUEST_TIMEOUT: Duration = Duration::from_secs(10);
|
||||
const SITE_REPLICATION_PEER_CONNECT_TIMEOUT: Duration = Duration::from_secs(3);
|
||||
const SITE_REPLICATION_PEER_ERROR_DETAIL_LIMIT: usize = 256;
|
||||
const SITE_REPLICATION_INITIAL_SYNC_ERROR_LIMIT: usize = 32;
|
||||
const MAX_PEER_CA_CERT_PEM_SIZE: usize = 256 * 1024;
|
||||
const ALLOW_LOOPBACK_REPLICATION_TARGET_ENV: &str = "RUSTFS_REPLICATION_ALLOW_LOOPBACK_TARGET";
|
||||
const SITE_REPLICATION_RETRY_QUEUE_LIMIT: usize = 256;
|
||||
@@ -428,8 +427,6 @@ struct SiteReplicationState {
|
||||
pending_endpoint_refresh: Option<PendingEndpointRefresh>,
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
retry_queue: Vec<SiteReplicationRetryEvent>,
|
||||
#[serde(default)]
|
||||
sync_state_initialized: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
|
||||
@@ -522,57 +519,6 @@ struct SiteReplicationBootstrapPlan {
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
|
||||
struct SRPeerJoinResponse {
|
||||
peer: PeerInfo,
|
||||
#[serde(rename = "initialSyncErrorMessage", default, skip_serializing_if = "String::is_empty")]
|
||||
initial_sync_error_message: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
|
||||
struct SRPeerJoinEnvelope {
|
||||
#[serde(flatten)]
|
||||
request: SRPeerJoinReq,
|
||||
#[serde(rename = "deferSyncStateEnable", default, skip_serializing_if = "std::ops::Not::not")]
|
||||
defer_sync_state_enable: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
struct SiteReplicationErrorSummary {
|
||||
entries: Vec<String>,
|
||||
total: usize,
|
||||
}
|
||||
|
||||
impl SiteReplicationErrorSummary {
|
||||
fn push(&mut self, error: impl AsRef<str>) {
|
||||
self.total = self.total.saturating_add(1);
|
||||
if self.entries.len() < SITE_REPLICATION_INITIAL_SYNC_ERROR_LIMIT {
|
||||
self.entries.push(summarize_peer_error_detail(error.as_ref()));
|
||||
}
|
||||
}
|
||||
|
||||
fn extend(&mut self, other: Self) {
|
||||
self.total = self.total.saturating_add(other.total);
|
||||
let remaining = SITE_REPLICATION_INITIAL_SYNC_ERROR_LIMIT.saturating_sub(self.entries.len());
|
||||
self.entries.extend(other.entries.into_iter().take(remaining));
|
||||
}
|
||||
|
||||
fn is_empty(&self) -> bool {
|
||||
self.total == 0
|
||||
}
|
||||
|
||||
fn reported(&self) -> usize {
|
||||
self.entries.len()
|
||||
}
|
||||
|
||||
fn render(&self) -> String {
|
||||
let mut message = self.entries.join("; ");
|
||||
let omitted = self.total.saturating_sub(self.entries.len());
|
||||
if omitted > 0 {
|
||||
if !message.is_empty() {
|
||||
message.push_str("; ");
|
||||
}
|
||||
message.push_str(&format!("{omitted} additional error(s) omitted"));
|
||||
}
|
||||
message
|
||||
}
|
||||
}
|
||||
|
||||
const GO_GOB_SITE_NETPERF_SCHEMA: &[u8] = &[
|
||||
@@ -892,12 +838,6 @@ async fn load_site_replication_state() -> S3Result<SiteReplicationState> {
|
||||
let mut state: SiteReplicationState = serde_json::from_slice(&data)
|
||||
.map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("invalid site replication state: {e}")))?;
|
||||
state.peers = normalize_peer_map_by_identity(state.peers);
|
||||
if !state.sync_state_initialized {
|
||||
if state.enabled() {
|
||||
mark_unknown_peer_sync_enabled(&mut state.peers);
|
||||
}
|
||||
state.sync_state_initialized = true;
|
||||
}
|
||||
Ok(state)
|
||||
}
|
||||
Err(StorageError::ConfigNotFound) => Ok(SiteReplicationState::default()),
|
||||
@@ -1717,32 +1657,6 @@ fn validate_add_sites(sites: &[PeerSite], local_peer: &PeerInfo) -> S3Result<()>
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// The web console's "Set Up Site Replication" flow sends only the remote peer(s) and omits the
|
||||
/// local deployment from the add payload. The add preflight requires the local deployment to be
|
||||
/// present (`validate_add_preflight_topology`), so inject the local site when the payload does not
|
||||
/// already include it. `mc admin replicate add` includes every site (matched here by endpoint
|
||||
/// identity), so this is a no-op for the CLI. The local site carries no credentials — they are not
|
||||
/// required for the local peer (`validate_add_sites` skips credential checks for it).
|
||||
fn ensure_local_site_present(sites: &mut Vec<PeerSite>, local_peer: &PeerInfo) {
|
||||
if sites
|
||||
.iter()
|
||||
.any(|site| same_identity_endpoint(&site.endpoint, &local_peer.endpoint))
|
||||
{
|
||||
return;
|
||||
}
|
||||
sites.insert(
|
||||
0,
|
||||
PeerSite {
|
||||
name: local_peer.name.clone(),
|
||||
endpoint: local_peer.endpoint.clone(),
|
||||
access_key: String::new(),
|
||||
secret_key: String::new(),
|
||||
skip_tls_verify: local_peer.skip_tls_verify,
|
||||
ca_cert_pem: local_peer.ca_cert_pem.clone(),
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
fn idp_settings_value(settings: &IDPSettings) -> S3Result<serde_json::Value> {
|
||||
serde_json::to_value(settings)
|
||||
.map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("serialize IDP settings failed: {e}")))
|
||||
@@ -2167,9 +2081,7 @@ fn build_join_peers(
|
||||
if let Some(local_site) = sites
|
||||
.iter()
|
||||
.find(|site| same_identity_endpoint(&site.endpoint, &normalized_local.endpoint))
|
||||
&& (is_https_endpoint(&local_site.endpoint) || !is_https_endpoint(&normalized_local.endpoint))
|
||||
{
|
||||
normalized_local.endpoint = local_site.endpoint.clone();
|
||||
normalized_local.skip_tls_verify = local_site.skip_tls_verify;
|
||||
normalized_local.ca_cert_pem = local_site.ca_cert_pem.clone();
|
||||
}
|
||||
@@ -2200,6 +2112,21 @@ fn build_join_peers(
|
||||
normalize_peer_map_by_identity(peers)
|
||||
}
|
||||
|
||||
/// Promote each peer's persisted `sync_state` from `Unknown` to `Enable` once site
|
||||
/// replication has been established for it. The persisted value is a static
|
||||
/// "configured & enabled" signal so that the `replicate info` endpoint (and the
|
||||
/// console, which read the persisted peer map verbatim) do not report `Unknown` for a
|
||||
/// healthy peer. `build_status_info` still refines this to `Disable`/`Unknown` from
|
||||
/// live health signals at query time for the `replicate status` endpoint. A peer that
|
||||
/// has been explicitly marked `Disable` is left untouched.
|
||||
fn mark_peers_sync_enabled(peers: &mut BTreeMap<String, PeerInfo>) {
|
||||
for peer in peers.values_mut() {
|
||||
if peer.sync_state == SyncStatus::Unknown {
|
||||
peer.sync_state = SyncStatus::Enable;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn normalize_join_peers_for_local(local_peer: &PeerInfo, peers: BTreeMap<String, PeerInfo>) -> BTreeMap<String, PeerInfo> {
|
||||
let mut normalized = BTreeMap::new();
|
||||
|
||||
@@ -2221,12 +2148,6 @@ fn normalize_join_peers_for_local(local_peer: &PeerInfo, peers: BTreeMap<String,
|
||||
normalize_peer_map_by_identity(normalized)
|
||||
}
|
||||
|
||||
fn initialize_join_peer_sync_state(peers: &mut BTreeMap<String, PeerInfo>, defer_sync_state_enable: bool) {
|
||||
if !defer_sync_state_enable {
|
||||
mark_unknown_peer_sync_enabled(peers);
|
||||
}
|
||||
}
|
||||
|
||||
fn reconcile_peer_with_actual_identity(mut state: SiteReplicationState, actual_peer: PeerInfo) -> SiteReplicationState {
|
||||
let mut actual_peer = normalize_peer_info(actual_peer);
|
||||
if let Some(requested_peer) = state
|
||||
@@ -2777,25 +2698,27 @@ async fn bootstrap_existing_metadata_after_add(
|
||||
state: &SiteReplicationState,
|
||||
local_peer: &PeerInfo,
|
||||
service_account_secret_key: &str,
|
||||
) -> SiteReplicationErrorSummary {
|
||||
) -> Vec<String> {
|
||||
let info = match build_sr_info(state, local_peer).await {
|
||||
Ok(info) => info,
|
||||
Err(err) => {
|
||||
let mut errors = SiteReplicationErrorSummary::default();
|
||||
errors.push(format!("local snapshot failed: {err}"));
|
||||
return errors;
|
||||
return vec![format!(
|
||||
"local snapshot failed: {}",
|
||||
summarize_peer_error_detail(&err.to_string())
|
||||
)];
|
||||
}
|
||||
};
|
||||
let plan = match site_replication_bootstrap_plan(&info) {
|
||||
Ok(plan) => plan,
|
||||
Err(err) => {
|
||||
let mut errors = SiteReplicationErrorSummary::default();
|
||||
errors.push(format!("bootstrap plan failed: {err}"));
|
||||
return errors;
|
||||
return vec![format!(
|
||||
"bootstrap plan failed: {}",
|
||||
summarize_peer_error_detail(&err.to_string())
|
||||
)];
|
||||
}
|
||||
};
|
||||
|
||||
let mut errors = SiteReplicationErrorSummary::default();
|
||||
let mut errors = Vec::new();
|
||||
for peer in state.peers.values() {
|
||||
if peer.deployment_id == local_peer.deployment_id || same_identity_endpoint(&peer.endpoint, &local_peer.endpoint) {
|
||||
continue;
|
||||
@@ -2820,6 +2743,17 @@ async fn bootstrap_existing_metadata_after_add(
|
||||
errors
|
||||
}
|
||||
|
||||
/// Combine the metadata-bootstrap and object-back-fill failure lists into the single
|
||||
/// `initial_sync_error_message` surfaced by `mc admin replicate add`, so an operator can
|
||||
/// see exactly which buckets did not propagate instead of receiving an unqualified success.
|
||||
fn compose_initial_sync_error_message(bootstrap_errors: Vec<String>, backfill_errors: Vec<String>) -> String {
|
||||
bootstrap_errors
|
||||
.into_iter()
|
||||
.chain(backfill_errors)
|
||||
.collect::<Vec<_>>()
|
||||
.join("; ")
|
||||
}
|
||||
|
||||
pub async fn site_replication_make_bucket_hook(bucket: &str, lock_enabled: bool) -> S3Result<()> {
|
||||
let _bucket_op_guard = SITE_REPLICATION_BUCKET_OP_LOCK.read().await;
|
||||
let runtime = {
|
||||
@@ -5471,10 +5405,10 @@ async fn backfill_existing_buckets_after_add(
|
||||
state: &SiteReplicationState,
|
||||
local_peer: &PeerInfo,
|
||||
bootstrap_token: Option<&str>,
|
||||
) -> SiteReplicationErrorSummary {
|
||||
let mut errors = SiteReplicationErrorSummary::default();
|
||||
) -> Vec<String> {
|
||||
let mut errors = Vec::new();
|
||||
let Some(store) = current_object_store_handle() else {
|
||||
errors.push("object store not initialized; pre-existing buckets were not backfilled");
|
||||
errors.push("object store not initialized; pre-existing buckets were not backfilled".to_string());
|
||||
return errors;
|
||||
};
|
||||
let buckets = match store.list_bucket(&BucketOptions::default()).await {
|
||||
@@ -5488,7 +5422,7 @@ async fn backfill_existing_buckets_after_add(
|
||||
error = ?err,
|
||||
"admin site replication state"
|
||||
);
|
||||
errors.push(format!("list buckets failed: {err}"));
|
||||
errors.push(format!("list buckets failed: {}", summarize_peer_error_detail(&err.to_string())));
|
||||
return errors;
|
||||
}
|
||||
};
|
||||
@@ -5507,7 +5441,10 @@ async fn backfill_existing_buckets_after_add(
|
||||
error = ?err,
|
||||
"admin site replication state"
|
||||
);
|
||||
errors.push(format!("{name}: versioning setup failed: {err}"));
|
||||
errors.push(format!(
|
||||
"{name}: versioning setup failed: {}",
|
||||
summarize_peer_error_detail(&err.to_string())
|
||||
));
|
||||
continue;
|
||||
}
|
||||
match ensure_site_replication_bucket_setup(name).await {
|
||||
@@ -5537,7 +5474,7 @@ async fn backfill_existing_buckets_after_add(
|
||||
error = ?err,
|
||||
"admin site replication state"
|
||||
);
|
||||
errors.push(format!("{name}: bucket setup failed: {err}"));
|
||||
errors.push(format!("{name}: bucket setup failed: {}", summarize_peer_error_detail(&err.to_string())));
|
||||
}
|
||||
}
|
||||
// Broadcast the bucket to peers so they create it too (idempotent on the peer side).
|
||||
@@ -5569,7 +5506,10 @@ async fn backfill_existing_buckets_after_add(
|
||||
error = ?err,
|
||||
"admin site replication state"
|
||||
);
|
||||
errors.push(format!("{name}: make-bucket broadcast failed: {err}"));
|
||||
errors.push(format!(
|
||||
"{name}: make-bucket broadcast failed: {}",
|
||||
summarize_peer_error_detail(&err.to_string())
|
||||
));
|
||||
}
|
||||
// Kick a resync toward every remote peer so existing objects travel across.
|
||||
for peer in state.peers.values() {
|
||||
@@ -5588,7 +5528,11 @@ async fn backfill_existing_buckets_after_add(
|
||||
detail = %result.err_detail,
|
||||
"admin site replication state"
|
||||
);
|
||||
errors.push(format!("{name} -> {}: resync kick failed: {}", peer.endpoint, result.err_detail));
|
||||
errors.push(format!(
|
||||
"{name} -> {}: resync kick failed: {}",
|
||||
peer.endpoint,
|
||||
summarize_peer_error_detail(&result.err_detail)
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -6236,10 +6180,7 @@ impl Operation for SiteReplicationAddHandler {
|
||||
return Err(s3_error!(InvalidRequest, "endpoint target refresh is pending"));
|
||||
}
|
||||
let local_peer = current_local_peer(&req, ¤t_state);
|
||||
let mut sites: Vec<PeerSite> = read_site_replication_json(req, &cred.secret_key, true).await?;
|
||||
// The web console's "Set Up Site Replication" omits the local deployment from the payload;
|
||||
// inject it so the add preflight (which requires the local deployment) succeeds. No-op for `mc`.
|
||||
ensure_local_site_present(&mut sites, &local_peer);
|
||||
let sites: Vec<PeerSite> = read_site_replication_json(req, &cred.secret_key, true).await?;
|
||||
validate_add_sites(&sites, &local_peer)?;
|
||||
let mut preflight_infos = Vec::with_capacity(sites.len());
|
||||
for site in &sites {
|
||||
@@ -6275,22 +6216,17 @@ impl Operation for SiteReplicationAddHandler {
|
||||
cred.access_key.clone(),
|
||||
replicate_ilm_expiry,
|
||||
);
|
||||
state.sync_state_initialized = true;
|
||||
let join_req = SRPeerJoinEnvelope {
|
||||
request: SRPeerJoinReq {
|
||||
svc_acct_access_key: service_account_access_key,
|
||||
svc_acct_secret_key: service_account_secret_key.clone(),
|
||||
svc_acct_parent: String::new(),
|
||||
peers: state.peers.clone(),
|
||||
updated_at: state.updated_at,
|
||||
},
|
||||
defer_sync_state_enable: true,
|
||||
let join_req = SRPeerJoinReq {
|
||||
svc_acct_access_key: service_account_access_key,
|
||||
svc_acct_secret_key: service_account_secret_key.clone(),
|
||||
svc_acct_parent: String::new(),
|
||||
peers: state.peers.clone(),
|
||||
updated_at: state.updated_at,
|
||||
};
|
||||
let peer_join_path =
|
||||
with_site_replication_bootstrap_token(SITE_REPLICATION_PEER_JOIN_PATH, &add_in_progress_guard.token.to_string());
|
||||
|
||||
let mut joined_endpoints = HashSet::new();
|
||||
let mut initial_sync_errors = SiteReplicationErrorSummary::default();
|
||||
for site in &sites {
|
||||
if same_identity_endpoint(&site.endpoint, &local_peer.endpoint)
|
||||
|| !joined_endpoints.insert(site_identity_key(&site.endpoint))
|
||||
@@ -6299,7 +6235,7 @@ impl Operation for SiteReplicationAddHandler {
|
||||
}
|
||||
|
||||
let mut peer_join_req = join_req.clone();
|
||||
peer_join_req.request.svc_acct_parent = site.access_key.clone();
|
||||
peer_join_req.svc_acct_parent = site.access_key.clone();
|
||||
let connection = PeerConnection::try_from(site)?;
|
||||
let body =
|
||||
send_peer_admin_request(&connection, &peer_join_path, &site.access_key, &site.secret_key, &peer_join_req).await?;
|
||||
@@ -6310,9 +6246,6 @@ impl Operation for SiteReplicationAddHandler {
|
||||
format!("parse peer join response from {} failed: {e}", site.endpoint),
|
||||
)
|
||||
})?;
|
||||
if !join_response.initial_sync_error_message.is_empty() {
|
||||
initial_sync_errors.push(format!("{}: {}", site.endpoint, join_response.initial_sync_error_message));
|
||||
}
|
||||
state = reconcile_peer_with_actual_identity(state, join_response.peer);
|
||||
let reconciled_peer = existing_peer_for_endpoint(&state, &site.endpoint).ok_or_else(|| {
|
||||
S3Error::with_message(
|
||||
@@ -6328,49 +6261,22 @@ impl Operation for SiteReplicationAddHandler {
|
||||
})?;
|
||||
}
|
||||
|
||||
mark_unknown_peer_sync_enabled(&mut state.peers);
|
||||
// BUG1: persist a real sync_state so `mc admin replicate info` and the console report
|
||||
// Enable for healthy peers instead of Unknown. build_status_info still refines this from
|
||||
// live health for the `replicate status` endpoint.
|
||||
mark_peers_sync_enabled(&mut state.peers);
|
||||
persist_site_replication_state(&state).await?;
|
||||
|
||||
for target in state.peers.values() {
|
||||
if target.deployment_id == local_peer.deployment_id || same_identity_endpoint(&target.endpoint, &local_peer.endpoint)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
let transport = match PeerTransport::for_runtime_peer(target).await {
|
||||
Ok(transport) => transport,
|
||||
Err(err) => {
|
||||
initial_sync_errors.push(format!("{}: finalize peer sync state failed: {err}", target.endpoint));
|
||||
continue;
|
||||
}
|
||||
};
|
||||
for peer in state.peers.values() {
|
||||
if let Err(err) = send_peer_admin_request_with_client(
|
||||
&transport.client,
|
||||
&transport.connection,
|
||||
SITE_REPLICATION_PEER_EDIT_PATH,
|
||||
&state.service_account_access_key,
|
||||
&service_account_secret_key,
|
||||
peer,
|
||||
)
|
||||
.await
|
||||
{
|
||||
initial_sync_errors
|
||||
.push(format!("{}: finalize sync state for {} failed: {err}", target.endpoint, peer.endpoint));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
initial_sync_errors.extend(bootstrap_existing_metadata_after_add(&state, &local_peer, &service_account_secret_key).await);
|
||||
let bootstrap_errors = bootstrap_existing_metadata_after_add(&state, &local_peer, &service_account_secret_key).await;
|
||||
|
||||
// Fix 1: back-fill pre-existing buckets so objects created before `replicate add`
|
||||
// are not silently left out of replication. Per-bucket failures are surfaced in the add
|
||||
// response below (BUG2) rather than swallowed; they do not abort the overall add.
|
||||
initial_sync_errors.extend(backfill_existing_buckets_after_add(&state, &local_peer, None).await);
|
||||
let backfill_errors = backfill_existing_buckets_after_add(&state, &local_peer, None).await;
|
||||
|
||||
json_response(&ReplicateAddStatus {
|
||||
success: true,
|
||||
status: SITE_REPL_ADD_SUCCESS.to_string(),
|
||||
initial_sync_error_message: initial_sync_errors.render(),
|
||||
initial_sync_error_message: compose_initial_sync_error_message(bootstrap_errors, backfill_errors),
|
||||
api_version: Some(SITE_REPL_API_VERSION.to_string()),
|
||||
..Default::default()
|
||||
})
|
||||
@@ -6598,22 +6504,18 @@ impl Operation for SRPeerJoinHandler {
|
||||
let _state_guard = SITE_REPLICATION_STATE_LOCK.lock().await;
|
||||
let mut state = load_site_replication_state().await?;
|
||||
let local_peer = current_local_peer(&req, &state);
|
||||
let join_envelope: SRPeerJoinEnvelope = read_site_replication_json(req, &cred.secret_key, true).await?;
|
||||
let defer_sync_state_enable = join_envelope.defer_sync_state_enable;
|
||||
let join_req = join_envelope.request;
|
||||
let join_req: SRPeerJoinReq = read_site_replication_json(req, &cred.secret_key, true).await?;
|
||||
validate_join_peer_snapshot(&join_req.peers)?;
|
||||
|
||||
if let Some(current_updated_at) = state.updated_at {
|
||||
let Some(incoming_updated_at) = join_req.updated_at else {
|
||||
return json_response(&SRPeerJoinResponse {
|
||||
peer: state.peers.get(&local_peer.deployment_id).cloned().unwrap_or(local_peer),
|
||||
..Default::default()
|
||||
});
|
||||
};
|
||||
if incoming_updated_at <= current_updated_at {
|
||||
return json_response(&SRPeerJoinResponse {
|
||||
peer: state.peers.get(&local_peer.deployment_id).cloned().unwrap_or(local_peer),
|
||||
..Default::default()
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -6672,8 +6574,9 @@ impl Operation for SRPeerJoinHandler {
|
||||
state.service_account_parent = join_req.svc_acct_parent;
|
||||
state.updated_at = join_req.updated_at.or_else(|| Some(OffsetDateTime::now_utc()));
|
||||
state.peers = normalize_join_peers_for_local(&local_peer, join_req.peers);
|
||||
initialize_join_peer_sync_state(&mut state.peers, defer_sync_state_enable);
|
||||
state.sync_state_initialized = true;
|
||||
// BUG1: persist Enable on the joining side too, so its `replicate info` endpoint reports a
|
||||
// real sync state rather than Unknown.
|
||||
mark_peers_sync_enabled(&mut state.peers);
|
||||
state.name = state
|
||||
.peers
|
||||
.get(&local_peer.deployment_id)
|
||||
@@ -6691,14 +6594,12 @@ impl Operation for SRPeerJoinHandler {
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_SITE_REPLICATION,
|
||||
result = "join_backfill_incomplete",
|
||||
error_count = backfill_errors.total,
|
||||
reported_error_count = backfill_errors.reported(),
|
||||
errors = %backfill_errors.join("; "),
|
||||
"admin site replication state"
|
||||
);
|
||||
}
|
||||
json_response(&SRPeerJoinResponse {
|
||||
peer: state.peers.get(&local_peer.deployment_id).cloned().unwrap_or(local_peer),
|
||||
initial_sync_error_message: backfill_errors.render(),
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -7397,7 +7298,7 @@ impl Operation for SiteReplicationRepairHandler {
|
||||
} else {
|
||||
"Partial".to_string()
|
||||
},
|
||||
err_detail: repair_errors.render(),
|
||||
err_detail: repair_errors.join("; "),
|
||||
api_version: Some(SITE_REPL_API_VERSION.to_string()),
|
||||
})
|
||||
}
|
||||
@@ -8156,66 +8057,6 @@ mod tests {
|
||||
assert_eq!(local.ca_cert_pem, "local-ca");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_build_join_peers_prefers_explicit_https_for_all_local_peer_tls_modes() {
|
||||
let local_peer = PeerInfo {
|
||||
deployment_id: "local-deployment".to_string(),
|
||||
..peer("local", "http://local.example.com:9000")
|
||||
};
|
||||
let custom_ca = valid_test_ca_pem("local.example.com");
|
||||
|
||||
for (skip_tls_verify, ca_cert_pem) in [(false, String::new()), (true, String::new()), (false, custom_ca)] {
|
||||
let peers = build_join_peers(
|
||||
&SiteReplicationState::default(),
|
||||
&local_peer,
|
||||
vec![PeerSite {
|
||||
name: "local".to_string(),
|
||||
endpoint: "https://local.example.com:9000".to_string(),
|
||||
skip_tls_verify,
|
||||
ca_cert_pem: ca_cert_pem.clone(),
|
||||
..PeerSite::default()
|
||||
}],
|
||||
false,
|
||||
);
|
||||
|
||||
let local = peers.get("local-deployment").expect("local peer should be present");
|
||||
assert_eq!(local.endpoint, "https://local.example.com:9000");
|
||||
assert_eq!(local.skip_tls_verify, skip_tls_verify);
|
||||
assert_eq!(local.ca_cert_pem, ca_cert_pem);
|
||||
assert!(validate_join_peer_snapshot(&peers).is_ok());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_build_join_peers_does_not_downgrade_local_https_tls_modes() {
|
||||
let custom_ca = valid_test_ca_pem("local.example.com");
|
||||
|
||||
for (skip_tls_verify, ca_cert_pem) in [(false, String::new()), (true, String::new()), (false, custom_ca)] {
|
||||
let local_peer = PeerInfo {
|
||||
deployment_id: "local-deployment".to_string(),
|
||||
skip_tls_verify,
|
||||
ca_cert_pem: ca_cert_pem.clone(),
|
||||
..peer("local", "https://local.example.com:9000")
|
||||
};
|
||||
let peers = build_join_peers(
|
||||
&SiteReplicationState::default(),
|
||||
&local_peer,
|
||||
vec![PeerSite {
|
||||
name: "local".to_string(),
|
||||
endpoint: "http://local.example.com:9000".to_string(),
|
||||
..PeerSite::default()
|
||||
}],
|
||||
false,
|
||||
);
|
||||
|
||||
let local = peers.get("local-deployment").expect("local peer should be present");
|
||||
assert_eq!(local.endpoint, "https://local.example.com:9000");
|
||||
assert_eq!(local.skip_tls_verify, skip_tls_verify);
|
||||
assert_eq!(local.ca_cert_pem, ca_cert_pem);
|
||||
assert!(validate_join_peer_snapshot(&peers).is_ok());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_build_join_peers_explicitly_disables_existing_remote_tls_settings() {
|
||||
let local_peer = PeerInfo {
|
||||
@@ -9168,63 +9009,6 @@ mod tests {
|
||||
assert!(err.to_string().contains("secretKey is required"));
|
||||
}
|
||||
|
||||
// Console fix: the web UI omits the local deployment from the add payload. ensure_local_site_present
|
||||
// injects it so the add preflight (which requires the local deployment) succeeds.
|
||||
#[test]
|
||||
fn test_ensure_local_site_present_injects_when_missing() {
|
||||
let local_peer = peer("local", "https://local.example.com");
|
||||
let mut sites = vec![PeerSite {
|
||||
name: "remote".to_string(),
|
||||
endpoint: "https://remote.example.com".to_string(),
|
||||
access_key: "remote-ak".to_string(),
|
||||
secret_key: "remote-sk".to_string(),
|
||||
..Default::default()
|
||||
}];
|
||||
|
||||
ensure_local_site_present(&mut sites, &local_peer);
|
||||
|
||||
assert_eq!(sites.len(), 2, "the local site must be injected when missing");
|
||||
assert!(
|
||||
sites
|
||||
.iter()
|
||||
.any(|s| same_identity_endpoint(&s.endpoint, &local_peer.endpoint)),
|
||||
"an injected site must match the local endpoint"
|
||||
);
|
||||
// The console payload (remote-only) now validates end-to-end at the add-sites stage.
|
||||
validate_add_sites(&sites, &local_peer).expect("add sites must validate after injecting the local site");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_ensure_local_site_present_noop_when_already_included() {
|
||||
let local_peer = peer("local", "https://local.example.com");
|
||||
let mut sites = vec![
|
||||
PeerSite {
|
||||
name: "local".to_string(),
|
||||
endpoint: "https://local.example.com".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
PeerSite {
|
||||
name: "remote".to_string(),
|
||||
endpoint: "https://remote.example.com".to_string(),
|
||||
access_key: "remote-ak".to_string(),
|
||||
secret_key: "remote-sk".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
];
|
||||
|
||||
ensure_local_site_present(&mut sites, &local_peer);
|
||||
|
||||
assert_eq!(sites.len(), 2, "the local site must not be duplicated when already present");
|
||||
assert_eq!(
|
||||
sites
|
||||
.iter()
|
||||
.filter(|s| same_identity_endpoint(&s.endpoint, &local_peer.endpoint))
|
||||
.count(),
|
||||
1,
|
||||
"exactly one local site entry"
|
||||
);
|
||||
}
|
||||
|
||||
fn preflight_site(name: &str, endpoint: &str, deployment_id: &str, bucket_count: usize) -> SiteReplicationAddPreflightInfo {
|
||||
SiteReplicationAddPreflightInfo {
|
||||
name: name.to_string(),
|
||||
@@ -11281,7 +11065,7 @@ mod tests {
|
||||
peers.values().all(|p| p.sync_state == SyncStatus::Unknown),
|
||||
"freshly constructed peers default to Unknown"
|
||||
);
|
||||
mark_unknown_peer_sync_enabled(&mut peers);
|
||||
mark_peers_sync_enabled(&mut peers);
|
||||
assert!(
|
||||
!peers.is_empty() && peers.values().all(|p| p.sync_state == SyncStatus::Enable),
|
||||
"add/join must persist Enable so the info endpoint reports a real sync state"
|
||||
@@ -11308,35 +11092,11 @@ mod tests {
|
||||
..peer("b", "https://b.example.com")
|
||||
},
|
||||
);
|
||||
mark_unknown_peer_sync_enabled(&mut peers);
|
||||
mark_peers_sync_enabled(&mut peers);
|
||||
assert_eq!(peers["a"].sync_state, SyncStatus::Enable, "Unknown must be promoted to Enable");
|
||||
assert_eq!(peers["b"].sync_state, SyncStatus::Disable, "explicit Disable must be preserved");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_join_peer_sync_state_waits_for_deferred_commit() {
|
||||
let mut peers = BTreeMap::from([("a".to_string(), peer("a", "https://a.example.com"))]);
|
||||
|
||||
initialize_join_peer_sync_state(&mut peers, true);
|
||||
assert_eq!(peers["a"].sync_state, SyncStatus::Unknown);
|
||||
|
||||
initialize_join_peer_sync_state(&mut peers, false);
|
||||
assert_eq!(peers["a"].sync_state, SyncStatus::Enable);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_join_deferred_sync_state_flag_is_wire_compatible() {
|
||||
let legacy: SRPeerJoinEnvelope = serde_json::from_value(serde_json::json!({})).expect("parse legacy peer join request");
|
||||
assert!(!legacy.defer_sync_state_enable);
|
||||
|
||||
let value = serde_json::to_value(SRPeerJoinEnvelope {
|
||||
defer_sync_state_enable: true,
|
||||
..Default::default()
|
||||
})
|
||||
.expect("serialize peer join request");
|
||||
assert_eq!(value.get("deferSyncStateEnable"), Some(&Value::Bool(true)));
|
||||
}
|
||||
|
||||
// BUG2: pre-existing-bucket back-fill failures must be surfaced in the add response's
|
||||
// initial_sync_error_message, not swallowed behind an unqualified success.
|
||||
#[test]
|
||||
@@ -11346,11 +11106,7 @@ mod tests {
|
||||
"test78787: replication setup skipped (site replication runtime unavailable)".to_string(),
|
||||
"test78787 -> https://peer.example.com: resync kick failed: timeout".to_string(),
|
||||
];
|
||||
let mut errors = SiteReplicationErrorSummary::default();
|
||||
for error in bootstrap_errors.into_iter().chain(backfill_errors) {
|
||||
errors.push(error);
|
||||
}
|
||||
let msg = errors.render();
|
||||
let msg = compose_initial_sync_error_message(bootstrap_errors, backfill_errors);
|
||||
assert!(msg.contains("peer-x: metadata sync failed"), "bootstrap errors must be surfaced");
|
||||
assert!(
|
||||
msg.contains("test78787: replication setup skipped"),
|
||||
@@ -11360,34 +11116,8 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_initial_sync_error_summary_is_bounded() {
|
||||
let mut errors = SiteReplicationErrorSummary::default();
|
||||
for index in 0..(SITE_REPLICATION_INITIAL_SYNC_ERROR_LIMIT + 5) {
|
||||
errors.push(format!("bucket-{index}: {}", "x".repeat(SITE_REPLICATION_PEER_ERROR_DETAIL_LIMIT + 32)));
|
||||
}
|
||||
|
||||
let message = errors.render();
|
||||
|
||||
assert_eq!(errors.reported(), SITE_REPLICATION_INITIAL_SYNC_ERROR_LIMIT);
|
||||
assert!(message.contains("5 additional error(s) omitted"));
|
||||
assert!(message.chars().count() <= SITE_REPLICATION_INITIAL_SYNC_ERROR_LIMIT * 258 + 64);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_peer_join_response_error_summary_is_wire_compatible() {
|
||||
let response: SRPeerJoinResponse = serde_json::from_value(serde_json::json!({
|
||||
"peer": peer("remote", "https://remote.example.com")
|
||||
}))
|
||||
.expect("parse legacy peer join response");
|
||||
|
||||
assert!(response.initial_sync_error_message.is_empty());
|
||||
|
||||
let value = serde_json::to_value(SRPeerJoinResponse {
|
||||
peer: peer("remote", "https://remote.example.com"),
|
||||
initial_sync_error_message: "bucket setup failed".to_string(),
|
||||
})
|
||||
.expect("serialize peer join response");
|
||||
assert_eq!(value.get("initialSyncErrorMessage").and_then(Value::as_str), Some("bucket setup failed"));
|
||||
fn test_compose_initial_sync_error_message_empty_on_success() {
|
||||
assert_eq!(compose_initial_sync_error_message(Vec::new(), Vec::new()), "");
|
||||
}
|
||||
|
||||
// Fix 5: remove --all must purge local state unconditionally even when peer errors occur
|
||||
|
||||
@@ -79,14 +79,6 @@ pub struct AddTierQuery {
|
||||
|
||||
pub struct AddTier {}
|
||||
|
||||
fn wasabi_payload_name(config: &TierConfig) -> S3Result<String> {
|
||||
config
|
||||
.wasabi
|
||||
.as_ref()
|
||||
.map(|wasabi| wasabi.name.clone())
|
||||
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing Wasabi configuration"))
|
||||
}
|
||||
|
||||
fn spawn_transition_tier_config_propagation(action: &'static str) {
|
||||
if let Some(notification_sys) = current_notification_system() {
|
||||
spawn_traced(async move {
|
||||
@@ -279,9 +271,6 @@ impl Operation for AddTier {
|
||||
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing S3 configuration"))?
|
||||
.name;
|
||||
}
|
||||
TierType::Wasabi => {
|
||||
args.name = wasabi_payload_name(&args)?;
|
||||
}
|
||||
TierType::RustFS => {
|
||||
args.name = args
|
||||
.rustfs
|
||||
@@ -955,20 +944,6 @@ mod tests {
|
||||
use http::Uri;
|
||||
use matchit::Router;
|
||||
|
||||
#[test]
|
||||
fn wasabi_payload_name_requires_nested_configuration() {
|
||||
let config: TierConfig = serde_json::from_slice(
|
||||
br#"{"type":"wasabi","wasabi":{"name":"WASABI-FIRST","accessKey":"ak","secretKey":"sk","bucket":"archive","region":"us-east-1"}}"#,
|
||||
)
|
||||
.expect("Wasabi AddTier payload should decode");
|
||||
assert_eq!(wasabi_payload_name(&config).expect("Wasabi payload name should exist"), "WASABI-FIRST");
|
||||
|
||||
let missing: TierConfig = serde_json::from_slice(br#"{"type":"wasabi"}"#).expect("type-only payload should decode");
|
||||
let err = wasabi_payload_name(&missing).expect_err("missing Wasabi payload must fail at the AddTier boundary");
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest);
|
||||
assert_eq!(err.message(), Some("missing Wasabi configuration"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_tier_name_prefers_path_parameter() {
|
||||
let uri: Uri = "/rustfs/admin/v3/tier/HOT?tier=COLD".parse().expect("uri should parse");
|
||||
|
||||
@@ -183,14 +183,6 @@ fn imported_service_account_access_key_failure(entry_access_key: &str, payload_a
|
||||
}
|
||||
|
||||
pub struct AddUser {}
|
||||
|
||||
fn map_add_user_create_error(err: rustfs_iam::error::Error) -> S3Error {
|
||||
match err {
|
||||
rustfs_iam::error::Error::AccessKeyAlreadyExists => iam_error_to_s3_error(err),
|
||||
err => S3Error::with_message(S3ErrorCode::InternalError, format!("failed to create user: {err}")),
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl Operation for AddUser {
|
||||
async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
|
||||
@@ -282,7 +274,10 @@ impl Operation for AddUser {
|
||||
)
|
||||
.await?;
|
||||
|
||||
let updated_at = iam_store.create_user(ak, &args).await.map_err(map_add_user_create_error)?;
|
||||
let updated_at = iam_store
|
||||
.create_user(ak, &args)
|
||||
.await
|
||||
.map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("failed to create user: {e}")))?;
|
||||
|
||||
if let Err(err) = site_replication_iam_change_hook(SRIAMItem {
|
||||
r#type: "iam-user".to_string(),
|
||||
@@ -1340,38 +1335,15 @@ mod tests {
|
||||
use super::{
|
||||
GROUP_POLICY_MAPPING_USER_TYPE, SERVICE_ACCOUNT_ACCESS_KEY_MISMATCH_ERROR, SERVICE_ACCOUNT_PARENT_SCOPE_ERROR,
|
||||
imported_service_account_access_key_failure, imported_service_account_parent_allowed,
|
||||
imported_service_account_parent_scope_failure, imported_service_account_status, map_add_user_create_error,
|
||||
should_check_deny_only, should_reject_group_import_name, should_restore_group_as_disabled,
|
||||
imported_service_account_parent_scope_failure, imported_service_account_status, should_check_deny_only,
|
||||
should_reject_group_import_name, should_restore_group_as_disabled,
|
||||
};
|
||||
use rustfs_credentials::{Credentials, IAM_POLICY_CLAIM_NAME_SA};
|
||||
use rustfs_iam::error::Error as IamError;
|
||||
use rustfs_madmin::user::SRSvcAccCreate;
|
||||
use s3s::S3ErrorCode;
|
||||
use serde_json::Value;
|
||||
use std::collections::HashMap;
|
||||
|
||||
#[test]
|
||||
fn add_user_maps_duplicate_access_keys_to_s3_errors() {
|
||||
let err = map_add_user_create_error(IamError::AccessKeyAlreadyExists);
|
||||
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
|
||||
assert_eq!(err.message(), Some("access key is already in use"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn add_user_preserves_internal_error_fallback() {
|
||||
let err = map_add_user_create_error(IamError::IamSysNotInitialized);
|
||||
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
assert_eq!(err.message(), Some("failed to create user: not initialized"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn add_user_operation_maps_create_errors() {
|
||||
let mapper_call = concat!("map_err(map_add_user_", "create_error)");
|
||||
assert!(include_str!("user.rs").contains(mapper_call));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_should_check_deny_only_for_regular_self_request() {
|
||||
let cred = Credentials {
|
||||
|
||||
@@ -13,9 +13,7 @@
|
||||
// limitations under the License.
|
||||
|
||||
use crate::admin::runtime_sources::{AppContext, current_app_context, current_object_store_handle_for_context};
|
||||
use crate::admin::site_replication_identity::{
|
||||
deployment_id_for_endpoint, mark_unknown_peer_sync_enabled, normalize_peer_map_by_identity_with,
|
||||
};
|
||||
use crate::admin::site_replication_identity::{deployment_id_for_endpoint, normalize_peer_map_by_identity_with};
|
||||
use crate::admin::storage_api::config::{read_admin_config, save_admin_config};
|
||||
use crate::admin::storage_api::error::Error as StorageError;
|
||||
use rustfs_madmin::PeerInfo;
|
||||
@@ -24,9 +22,8 @@ use serde_json::{Map, Value};
|
||||
use tracing::info;
|
||||
|
||||
const SITE_REPLICATION_STATE_PATH: &str = "config/site-replication/state.json";
|
||||
const SYNC_STATE_INITIALIZED_FIELD: &str = "sync_state_initialized";
|
||||
|
||||
fn normalize_peers_map(peers: &Map<String, Value>, initialize_sync_state: bool) -> Map<String, Value> {
|
||||
fn normalize_peers_map(peers: &Map<String, Value>) -> Map<String, Value> {
|
||||
let mut valid_peers = std::collections::BTreeMap::<String, PeerInfo>::new();
|
||||
let mut passthrough_invalid = Vec::<(String, Value)>::new();
|
||||
|
||||
@@ -49,10 +46,7 @@ fn normalize_peers_map(peers: &Map<String, Value>, initialize_sync_state: bool)
|
||||
}
|
||||
}
|
||||
|
||||
let mut deduped_by_deployment = normalize_peer_map_by_identity_with(valid_peers, |peer| peer);
|
||||
if initialize_sync_state && deduped_by_deployment.len() > 1 {
|
||||
mark_unknown_peer_sync_enabled(&mut deduped_by_deployment);
|
||||
}
|
||||
let deduped_by_deployment = normalize_peer_map_by_identity_with(valid_peers, |peer| peer);
|
||||
|
||||
let mut normalized = Map::new();
|
||||
for (_, peer) in deduped_by_deployment {
|
||||
@@ -75,32 +69,20 @@ fn normalize_site_replication_state_json(data: &[u8]) -> Result<Option<Vec<u8>>,
|
||||
|
||||
let before = obj.get("peers").and_then(|v| v.as_object()).map(|v| v.len()).unwrap_or(0);
|
||||
|
||||
let sync_state_initialized = obj
|
||||
.get(SYNC_STATE_INITIALIZED_FIELD)
|
||||
.and_then(Value::as_bool)
|
||||
.unwrap_or(false);
|
||||
let Some(peers_obj) = obj.get("peers").and_then(|v| v.as_object()) else {
|
||||
return Ok(None);
|
||||
};
|
||||
let normalized_peers = normalize_peers_map(peers_obj, !sync_state_initialized);
|
||||
if normalized_peers == *peers_obj && sync_state_initialized {
|
||||
|
||||
let normalized_peers = normalize_peers_map(peers_obj);
|
||||
if normalized_peers == *peers_obj {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let after = normalized_peers.len();
|
||||
obj.insert("peers".to_string(), Value::Object(normalized_peers));
|
||||
obj.insert(SYNC_STATE_INITIALIZED_FIELD.to_string(), Value::Bool(true));
|
||||
let normalized =
|
||||
serde_json::to_vec(&state).map_err(|e| format!("serialize normalized site replication state failed: {e}"))?;
|
||||
info!(
|
||||
event = "site_replication_state_normalized",
|
||||
component = "admin",
|
||||
subsystem = "site_replication",
|
||||
peers_before = before,
|
||||
peers_after = after,
|
||||
sync_state_migrated = !sync_state_initialized,
|
||||
"site replication state normalized"
|
||||
);
|
||||
info!("normalized site-replication peers during reload: {before} -> {after}");
|
||||
Ok(Some(normalized))
|
||||
}
|
||||
|
||||
@@ -147,12 +129,12 @@ mod tests {
|
||||
serde_json::json!({
|
||||
"name": name,
|
||||
"endpoint": endpoint,
|
||||
"deploymentID": deployment_id,
|
||||
"sync": "unknown",
|
||||
"defaultbandwidth": {},
|
||||
"replicate-ilm-expiry": false,
|
||||
"objectNamingMode": "",
|
||||
"apiVersion": "1"
|
||||
"deployment_id": deployment_id,
|
||||
"sync_state": "",
|
||||
"default_bandwidth": {},
|
||||
"replicate_ilm_expiry": false,
|
||||
"object_naming_mode": "",
|
||||
"api_version": "1"
|
||||
})
|
||||
}
|
||||
|
||||
@@ -238,55 +220,4 @@ mod tests {
|
||||
|
||||
assert_eq!(peers.len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_normalize_state_json_migrates_legacy_unknown_sync_state() {
|
||||
let data = serde_json::to_vec(&serde_json::json!({
|
||||
"name": "local",
|
||||
"peers": {
|
||||
"local": peer_value("local", "https://local.example.com", "local"),
|
||||
"remote": peer_value("remote", "https://remote.example.com", "remote")
|
||||
}
|
||||
}))
|
||||
.expect("serialize legacy state");
|
||||
|
||||
let normalized = normalize_site_replication_state_json(&data)
|
||||
.expect("normalize legacy state")
|
||||
.expect("legacy state should be migrated");
|
||||
let value: Value = serde_json::from_slice(&normalized).expect("parse normalized state");
|
||||
let peers = value.get("peers").and_then(Value::as_object).expect("normalized peers");
|
||||
|
||||
assert_eq!(value.get(SYNC_STATE_INITIALIZED_FIELD), Some(&Value::Bool(true)));
|
||||
assert!(
|
||||
peers
|
||||
.values()
|
||||
.all(|peer| peer.get("sync").and_then(Value::as_str) == Some("enable"))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_normalize_state_json_preserves_initialized_pending_sync_state() {
|
||||
let data = serde_json::to_vec(&serde_json::json!({
|
||||
"name": "local",
|
||||
(SYNC_STATE_INITIALIZED_FIELD): true,
|
||||
"peers": {
|
||||
"local": peer_value("local", "https://local.example.com", "local"),
|
||||
"remote": peer_value("remote", "https://remote.example.com", "remote")
|
||||
}
|
||||
}))
|
||||
.expect("serialize pending state");
|
||||
|
||||
let normalized = normalize_site_replication_state_json(&data)
|
||||
.expect("normalize pending state")
|
||||
.expect("pending state should receive canonical peer fields");
|
||||
let value: Value = serde_json::from_slice(&normalized).expect("parse normalized pending state");
|
||||
let peers = value.get("peers").and_then(Value::as_object).expect("normalized peers");
|
||||
|
||||
assert_eq!(value.get(SYNC_STATE_INITIALIZED_FIELD), Some(&Value::Bool(true)));
|
||||
assert!(
|
||||
peers
|
||||
.values()
|
||||
.all(|peer| peer.get("sync").and_then(Value::as_str) == Some("unknown"))
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use rustfs_madmin::{PeerInfo, SyncStatus};
|
||||
use rustfs_madmin::PeerInfo;
|
||||
use std::collections::{BTreeMap, hash_map::DefaultHasher};
|
||||
use std::hash::{Hash, Hasher};
|
||||
use url::Url;
|
||||
@@ -76,15 +76,7 @@ pub fn same_identity_endpoint(left: &str, right: &str) -> bool {
|
||||
site_identity_key(left) == site_identity_key(right)
|
||||
}
|
||||
|
||||
pub(crate) fn mark_unknown_peer_sync_enabled(peers: &mut BTreeMap<String, PeerInfo>) {
|
||||
for peer in peers.values_mut() {
|
||||
if peer.sync_state == SyncStatus::Unknown {
|
||||
peer.sync_state = SyncStatus::Enable;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn is_https_endpoint(endpoint: &str) -> bool {
|
||||
fn is_https_endpoint(endpoint: &str) -> bool {
|
||||
canonical_endpoint(endpoint).starts_with("https://")
|
||||
}
|
||||
|
||||
|
||||
@@ -330,6 +330,10 @@ impl DefaultAdminUsecase {
|
||||
return Err(Self::app_error_default(S3ErrorCode::NotImplemented));
|
||||
};
|
||||
|
||||
if endpoints.legacy() {
|
||||
return Err(Self::app_error_default(S3ErrorCode::NotImplemented));
|
||||
}
|
||||
|
||||
Self::refresh_pool_status_snapshot(store.as_ref()).await?;
|
||||
|
||||
let mut pool_statuses = Vec::new();
|
||||
@@ -396,12 +400,6 @@ impl DefaultAdminUsecase {
|
||||
let Some(store) = self.object_store() else {
|
||||
return Err(Self::app_error(S3ErrorCode::InternalError, "Not init"));
|
||||
};
|
||||
let Some(endpoints) = self.endpoints() else {
|
||||
return Err(Self::app_error_default(S3ErrorCode::NotImplemented));
|
||||
};
|
||||
if endpoints.legacy() {
|
||||
return Err(Self::app_error_default(S3ErrorCode::NotImplemented));
|
||||
}
|
||||
let pool_statuses = self.execute_list_pool_statuses().await?;
|
||||
Self::refresh_rebalance_status_snapshot(store.as_ref()).await?;
|
||||
let mut pools = Vec::with_capacity(pool_statuses.len());
|
||||
|
||||
@@ -153,11 +153,13 @@ async fn data_movement_put_object_marks_dirty_disks_for_capacity_manager() {
|
||||
|
||||
let actual_paths: HashSet<_> = dirty_disks
|
||||
.into_iter()
|
||||
.map(|disk| stdfs::canonicalize(&disk.drive_path).unwrap().to_string_lossy().into_owned())
|
||||
.filter_map(|disk| stdfs::canonicalize(&disk.drive_path).ok())
|
||||
.map(|p| p.to_string_lossy().into_owned())
|
||||
.collect();
|
||||
let expected_paths: HashSet<_> = disk_paths
|
||||
.iter()
|
||||
.map(|path| stdfs::canonicalize(path).unwrap().to_string_lossy().into_owned())
|
||||
.filter_map(|path| stdfs::canonicalize(path).ok())
|
||||
.map(|p| p.to_string_lossy().into_owned())
|
||||
.collect();
|
||||
// The global dirty scope registry is process-wide; concurrent tests may
|
||||
// add extra entries, so we only verify that our expected paths are present.
|
||||
@@ -217,9 +219,12 @@ async fn heal_object_marks_missing_shard_disk_dirty_for_capacity_manager() {
|
||||
let dirty_disks = manager.get_dirty_disks().await;
|
||||
let actual_paths: HashSet<_> = dirty_disks
|
||||
.into_iter()
|
||||
.map(|disk| stdfs::canonicalize(&disk.drive_path).unwrap().to_string_lossy().into_owned())
|
||||
.filter_map(|disk| stdfs::canonicalize(&disk.drive_path).ok())
|
||||
.map(|p| p.to_string_lossy().into_owned())
|
||||
.collect();
|
||||
let expected_missing_disk = stdfs::canonicalize(&disk_paths[0]).unwrap().to_string_lossy().into_owned();
|
||||
let expected_missing_disk = stdfs::canonicalize(&disk_paths[0])
|
||||
.map(|p| p.to_string_lossy().into_owned())
|
||||
.unwrap_or_else(|_| disk_paths[0].to_string_lossy().into_owned());
|
||||
|
||||
assert!(
|
||||
error.is_none() || actual_paths.contains(&expected_missing_disk),
|
||||
|
||||
@@ -836,13 +836,9 @@ async fn complete_multipart_upload_transitions_immediately_via_usecase() {
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let complete_output = Box::pin(usecase.execute_complete_multipart_upload(build_request(complete_input, Method::POST)))
|
||||
Box::pin(usecase.execute_complete_multipart_upload(build_request(complete_input, Method::POST)))
|
||||
.await
|
||||
.expect("Failed to complete multipart upload through usecase");
|
||||
assert!(
|
||||
complete_output.output.version_id.is_some(),
|
||||
"versioned CompleteMultipartUpload should return the created version ID"
|
||||
);
|
||||
|
||||
let info = wait_for_transition(&ecstore, bucket.as_str(), object, TRANSITION_WAIT_TIMEOUT)
|
||||
.await
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user