mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-21 20:06:37 +00:00
Compare commits
3 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 3f994c59eb | |||
| 7c1d9dec8f | |||
| 4a8759239d |
@@ -1,2 +0,0 @@
|
|||||||
sha256-darwin=b4ae71aa894e5c7795ae3eb8116f1777a7601d0f5db3898be2e48faf3329bd9b
|
|
||||||
sha256-linux=433debd9d9defa832986269abdf0f1d131597b2d7a417ce930e17c1fd47d85ba
|
|
||||||
@@ -1 +0,0 @@
|
|||||||
sha256=9b9bc336b43b70d0e06e0adb5455bf035bb18945d85d60936eb6fe4d48e0e680
|
|
||||||
@@ -1,2 +0,0 @@
|
|||||||
sha256-darwin=55534a97fbd376f64c8f6c341d319017d11ff77cad6da8629a1a7f6a874e0315
|
|
||||||
sha256-linux=c06fb8c19aed6f388b9dc61cb8251b7a44f8561a9bf764ad2b9e635598f8dc17
|
|
||||||
@@ -1 +0,0 @@
|
|||||||
sha256=655a3f3c1d042e694339d15caba7580518320322d1bac0f09450b37e6c09e2e7
|
|
||||||
@@ -1 +0,0 @@
|
|||||||
sha256=ec27cde6ce6400723c4b372bfbd2ac61709c744294e4810af765e8a808d8e31d
|
|
||||||
@@ -66,20 +66,10 @@ s3s-footprint-check: ## Check the s3s dependency footprint ratchet stays frozen
|
|||||||
./scripts/check_s3s_footprint.sh
|
./scripts/check_s3s_footprint.sh
|
||||||
|
|
||||||
.PHONY: fips-wording-check
|
.PHONY: fips-wording-check
|
||||||
fips-wording-check: ## Check docs and crates/kms do not over-claim crypto capabilities
|
fips-wording-check: ## Check outward docs do not make unsupported FIPS claims
|
||||||
@echo "📣 Checking cryptographic capability wording guard..."
|
@echo "📣 Checking FIPS wording guard..."
|
||||||
./scripts/check_fips_wording.sh
|
./scripts/check_fips_wording.sh
|
||||||
|
|
||||||
.PHONY: embedded-secrets-check
|
|
||||||
embedded-secrets-check: ## Check no private key material or credential literal is committed
|
|
||||||
@echo "🔑 Checking embedded secret material guard..."
|
|
||||||
./scripts/check_embedded_secrets.sh
|
|
||||||
|
|
||||||
.PHONY: test-wiring-check
|
|
||||||
test-wiring-check: ## Check tests stay registered and selected by their intended runners
|
|
||||||
@echo "🧪 Checking test wiring..."
|
|
||||||
python3 ./scripts/check_test_wiring.py
|
|
||||||
|
|
||||||
.PHONY: log-analyzer-rules-check
|
.PHONY: log-analyzer-rules-check
|
||||||
log-analyzer-rules-check: core-deps ## Check log-analyzer rule anchors still exist verbatim in source
|
log-analyzer-rules-check: core-deps ## Check log-analyzer rule anchors still exist verbatim in source
|
||||||
@echo "🩺 Checking log-analyzer rule anchors..."
|
@echo "🩺 Checking log-analyzer rule anchors..."
|
||||||
|
|||||||
@@ -19,13 +19,13 @@ planning-docs-check: ## Check that no planning-type documents are committed
|
|||||||
./scripts/check_no_planning_docs.sh
|
./scripts/check_no_planning_docs.sh
|
||||||
|
|
||||||
.PHONY: pre-commit
|
.PHONY: pre-commit
|
||||||
pre-commit: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check quick-check ## Run fast pre-commit checks without clippy/full tests
|
pre-commit: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check fips-wording-check doc-paths-check planning-docs-check quick-check ## Run fast pre-commit checks without clippy/full tests
|
||||||
@echo "✅ All pre-commit checks passed!"
|
@echo "✅ All pre-commit checks passed!"
|
||||||
|
|
||||||
.PHONY: pre-pr
|
.PHONY: pre-pr
|
||||||
pre-pr: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check log-analyzer-rules-check clippy-check test ## Run full pre-PR checks with clippy and tests
|
pre-pr: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check fips-wording-check doc-paths-check planning-docs-check log-analyzer-rules-check clippy-check test ## Run full pre-PR checks with clippy and tests
|
||||||
@echo "✅ All pre-PR checks passed!"
|
@echo "✅ All pre-PR checks passed!"
|
||||||
|
|
||||||
.PHONY: dev-check
|
.PHONY: dev-check
|
||||||
dev-check: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check quick-check ## Run fast local development checks
|
dev-check: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check fips-wording-check doc-paths-check planning-docs-check quick-check ## Run fast local development checks
|
||||||
@echo "✅ Fast development checks passed!"
|
@echo "✅ Fast development checks passed!"
|
||||||
|
|||||||
@@ -34,9 +34,6 @@ script-tests: ## Run shell script tests
|
|||||||
./scripts/test_exact_1mib_handoff_abba.sh
|
./scripts/test_exact_1mib_handoff_abba.sh
|
||||||
./scripts/test_pinned_paired_abba_bench.sh
|
./scripts/test_pinned_paired_abba_bench.sh
|
||||||
./scripts/test_manual_transition_runbooks.sh
|
./scripts/test_manual_transition_runbooks.sh
|
||||||
./scripts/check_embedded_secrets.sh --self-test
|
|
||||||
python3 ./scripts/check_test_wiring.py --self-test
|
|
||||||
python3 ./scripts/s3-tests/test_report_compat.py
|
|
||||||
bash -n ./scripts/validate_object_data_cache_cold_stampede.sh
|
bash -n ./scripts/validate_object_data_cache_cold_stampede.sh
|
||||||
python3 ./scripts/check_object_data_cache_follower_samples.py --self-test
|
python3 ./scripts/check_object_data_cache_follower_samples.py --self-test
|
||||||
./scripts/validate_object_data_cache_cold_stampede.sh --self-test
|
./scripts/validate_object_data_cache_cold_stampede.sh --self-test
|
||||||
|
|||||||
+19
-71
@@ -34,15 +34,13 @@ e2e-vault = { max-threads = 1 }
|
|||||||
|
|
||||||
# Reliability / fault-injection e2e tests each spawn a single-node 4-disk RustFS
|
# Reliability / fault-injection e2e tests each spawn a single-node 4-disk RustFS
|
||||||
# server and manipulate its disk directories at runtime (crates/e2e_test:
|
# server and manipulate its disk directories at runtime (crates/e2e_test:
|
||||||
# reliability_disk_fault_test, degraded_read_eof_regression_test / dist-13, and
|
# reliability_disk_fault_test, degraded_read_eof_regression_test / dist-13). They
|
||||||
# replacement_privileged_e2e_test when explicitly run as root on Linux). They
|
|
||||||
# are correct in isolation but resource-heavy; serialize them under nextest's
|
# are correct in isolation but resource-heavy; serialize them under nextest's
|
||||||
# process boundary (serial_test's #[serial] does not cross it) so several 4-disk
|
# process boundary (serial_test's #[serial] does not cross it) so several 4-disk
|
||||||
# servers never run at once. The e2e-full merge/main lane picks these up;
|
# servers never run at once. ci-7's nightly picks these up via the e2e suite;
|
||||||
# they are deliberately NOT in the fast PR `e2e-smoke` filter.
|
# they are deliberately NOT in the fast PR `e2e-smoke` filter.
|
||||||
e2e-reliability = { max-threads = 1 }
|
e2e-reliability = { max-threads = 1 }
|
||||||
e2e-inline-boundaries = { max-threads = 1 }
|
e2e-inline-boundaries = { max-threads = 1 }
|
||||||
e2e-cluster-nightly = { max-threads = 1 }
|
|
||||||
|
|
||||||
# --- default profile (local): serialize the flaky groups, never retry --------
|
# --- default profile (local): serialize the flaky groups, never retry --------
|
||||||
[[profile.default.overrides]]
|
[[profile.default.overrides]]
|
||||||
@@ -88,18 +86,11 @@ test-group = 'ecstore-serial-flaky'
|
|||||||
filter = 'package(rustfs-ecstore) & (test(bucket_delete_waits_for_config_mutation_fence) | test(stale_config_request_cannot_mutate_a_recreated_bucket) | test(disk_incarnation_read_detects_stale_cache_until_peer_reload) | test(lifecycle_expiry_fails_closed_on_corrupt_object_lock_metadata) | test(expiry_configs_are_resolved_from_the_owning_store))'
|
filter = 'package(rustfs-ecstore) & (test(bucket_delete_waits_for_config_mutation_fence) | test(stale_config_request_cannot_mutate_a_recreated_bucket) | test(disk_incarnation_read_detects_stale_cache_until_peer_reload) | test(lifecycle_expiry_fails_closed_on_corrupt_object_lock_metadata) | test(expiry_configs_are_resolved_from_the_owning_store))'
|
||||||
test-group = 'ecstore-serial-flaky'
|
test-group = 'ecstore-serial-flaky'
|
||||||
|
|
||||||
# Serialize the default-off dst-dir fsync group-commit tests. They use
|
|
||||||
# process-global test hooks/registry to deterministically freeze fsync batches;
|
|
||||||
# no retries, just one at a time under nextest too.
|
|
||||||
[[profile.default.overrides]]
|
|
||||||
filter = 'package(rustfs-ecstore) & test(dst_dir_fsync_group_commit)'
|
|
||||||
test-group = 'ecstore-serial-flaky'
|
|
||||||
|
|
||||||
# Serialize the 4-disk reliability / degraded-read e2e tests (see the
|
# Serialize the 4-disk reliability / degraded-read e2e tests (see the
|
||||||
# e2e-reliability test-group note above). The matching ci-profile override is at
|
# e2e-reliability test-group note above). The matching ci-profile override is at
|
||||||
# the end of the file, after [profile.ci] is declared.
|
# the end of the file, after [profile.ci] is declared.
|
||||||
[[profile.default.overrides]]
|
[[profile.default.overrides]]
|
||||||
filter = 'package(e2e_test) & test(/^(reliability_disk_fault|degraded_read_eof_regression|replacement_privileged_e2e)_test::/)'
|
filter = 'package(e2e_test) & test(/^(reliability_disk_fault|degraded_read_eof_regression)_test::/)'
|
||||||
test-group = 'e2e-reliability'
|
test-group = 'e2e-reliability'
|
||||||
|
|
||||||
[[profile.default.overrides]]
|
[[profile.default.overrides]]
|
||||||
@@ -162,9 +153,9 @@ retries = 2
|
|||||||
# Serialize the 4-disk reliability / degraded-read e2e tests under the ci
|
# Serialize the 4-disk reliability / degraded-read e2e tests under the ci
|
||||||
# profile too (see the e2e-reliability test-group note near the top). Not a
|
# profile too (see the e2e-reliability test-group note near the top). Not a
|
||||||
# quarantine: no retries, just single-threaded so several 4-disk servers never
|
# quarantine: no retries, just single-threaded so several 4-disk servers never
|
||||||
# run concurrently when e2e-full runs the suite.
|
# run concurrently when ci-7's nightly runs the full e2e suite.
|
||||||
[[profile.ci.overrides]]
|
[[profile.ci.overrides]]
|
||||||
filter = 'package(e2e_test) & test(/^(reliability_disk_fault|degraded_read_eof_regression|replacement_privileged_e2e)_test::/)'
|
filter = 'package(e2e_test) & test(/^(reliability_disk_fault|degraded_read_eof_regression)_test::/)'
|
||||||
test-group = 'e2e-reliability'
|
test-group = 'e2e-reliability'
|
||||||
|
|
||||||
# Serialize the multipart crash-consistency scenarios under the ci profile too
|
# Serialize the multipart crash-consistency scenarios under the ci profile too
|
||||||
@@ -196,10 +187,6 @@ test-group = 'ecstore-serial-flaky'
|
|||||||
filter = 'package(rustfs-ecstore) & (test(bucket_delete_waits_for_config_mutation_fence) | test(stale_config_request_cannot_mutate_a_recreated_bucket) | test(disk_incarnation_read_detects_stale_cache_until_peer_reload) | test(lifecycle_expiry_fails_closed_on_corrupt_object_lock_metadata) | test(expiry_configs_are_resolved_from_the_owning_store))'
|
filter = 'package(rustfs-ecstore) & (test(bucket_delete_waits_for_config_mutation_fence) | test(stale_config_request_cannot_mutate_a_recreated_bucket) | test(disk_incarnation_read_detects_stale_cache_until_peer_reload) | test(lifecycle_expiry_fails_closed_on_corrupt_object_lock_metadata) | test(expiry_configs_are_resolved_from_the_owning_store))'
|
||||||
test-group = 'ecstore-serial-flaky'
|
test-group = 'ecstore-serial-flaky'
|
||||||
|
|
||||||
[[profile.ci.overrides]]
|
|
||||||
filter = 'package(rustfs-ecstore) & test(dst_dir_fsync_group_commit)'
|
|
||||||
test-group = 'ecstore-serial-flaky'
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# e2e-smoke profile — PR smoke subset of the e2e_test crate (backlog#1149 ci-4)
|
# e2e-smoke profile — PR smoke subset of the e2e_test crate (backlog#1149 ci-4)
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
@@ -231,8 +218,8 @@ test-group = 'ecstore-serial-flaky'
|
|||||||
# the nightly profile derives its set as "the replication module MINUS this
|
# the nightly profile derives its set as "the replication module MINUS this
|
||||||
# allowlist", so any new replication test lands in nightly by default (never
|
# allowlist", so any new replication test lands in nightly by default (never
|
||||||
# silently unrun) until it is explicitly blessed as fast here. Keep the two
|
# silently unrun) until it is explicitly blessed as fast here. Keep the two
|
||||||
# regexes byte-identical. The committed profile selection digests make changes
|
# regexes byte-identical. Count invariant: 20 here + 49 nightly = 69 total
|
||||||
# visible in CI; current counts live in docs/testing/e2e-suite-inventory.md.
|
# (authority: `cargo nextest list`; docs/testing/e2e-suite-inventory.md).
|
||||||
# HISTORY (2026-07-11): the 20 fast tests were briefly pulled out of this lane
|
# HISTORY (2026-07-11): the 20 fast tests were briefly pulled out of this lane
|
||||||
# (#4724) because they set a loopback (127.0.0.1) replication target that the
|
# (#4724) because they set a loopback (127.0.0.1) replication target that the
|
||||||
# SSRF egress guard rejected on every PR after repl-1 (#4712). That is fixed —
|
# SSRF egress guard rejected on every PR after repl-1 (#4712). That is fixed —
|
||||||
@@ -264,16 +251,10 @@ test-group = 'ecstore-serial-flaky'
|
|||||||
# cluster, so it keeps the lane's parallel-safe / no-external-dependency
|
# cluster, so it keeps the lane's parallel-safe / no-external-dependency
|
||||||
# properties. The RustFS warm backend has no loopback guard (that guard is
|
# properties. The RustFS warm backend has no loopback guard (that guard is
|
||||||
# replication-only), so it needs no opt-in env for its 127.0.0.1 tier target.
|
# replication-only), so it needs no opt-in env for its 127.0.0.1 tier target.
|
||||||
#
|
|
||||||
# Disk compression (backlog#1848): the `compression` module joins the smoke
|
|
||||||
# lane so the multipart disk-compression roundtrips (restored after
|
|
||||||
# rustfs/rustfs#5169 disabled them) have PR-lane signal, not just merge-gate.
|
|
||||||
# Single-node servers on random ports with isolated temp dirs — meets the
|
|
||||||
# admission criteria unchanged.
|
|
||||||
[profile.e2e-smoke]
|
[profile.e2e-smoke]
|
||||||
default-filter = """
|
default-filter = """
|
||||||
package(e2e_test) & (
|
package(e2e_test) & (
|
||||||
test(/^(delete_marker_migration_semantics|version_id_regression|list_objects_v2_pagination|list_object_versions_regression|list_objects_duplicates|list_buckets_double_slash|list_buckets_auth|list_buckets_iam_filter|leading_slash_key|special_chars|create_bucket_region|delete_objects_versioning|head_object_consistency|head_object_range|copy_object_metadata|copy_object_tagging|copy_source_invalid_date|content_encoding|compression|multipart_storage_class|storage_class_capability|ssec_copy|anonymous_access|bucket_policy_check|presigned_negative|negative_sigv4|admin_auth|notification_webhook|tls_hot_reload|console_smoke|admin_iam_crud|admin_pools|sts_query_compat)_test::|^fake_s3_target::/)
|
test(/^(delete_marker_migration_semantics|version_id_regression|list_objects_v2_pagination|list_object_versions_regression|list_objects_duplicates|list_buckets_double_slash|list_buckets_auth|list_buckets_iam_filter|leading_slash_key|special_chars|create_bucket_region|delete_objects_versioning|head_object_consistency|head_object_range|copy_object_metadata|copy_object_tagging|copy_source_invalid_date|content_encoding|multipart_storage_class|storage_class_capability|ssec_copy|anonymous_access|bucket_policy_check|presigned_negative|negative_sigv4|admin_auth|notification_webhook|tls_hot_reload|console_smoke|admin_iam_crud|admin_pools|sts_query_compat)_test::|^fake_s3_target::/)
|
||||||
| test(/^replication_extension_test::(test_replication_check_succeeds_with_remote_target|test_replication_check_rejects_target_without_object_lock|test_set_remote_target_rejects_unversioned_source_bucket|test_replication_check_rejects_unversioned_source_bucket|test_replication_check_rejects_missing_replication_config|test_replication_check_rejects_invalid_bucket|test_set_remote_target_rejects_same_bucket_on_same_deployment|test_set_remote_target_rejects_unversioned_target_bucket|test_set_remote_target_update_requires_arn|test_set_remote_target_update_rejects_missing_target|test_set_remote_target_rejects_invalid_target_url|test_set_remote_target_rejects_self_signed_https_target_without_skip_tls_verify|test_set_remote_target_rejects_private_ca_https_target_without_ca_cert_pem|test_list_remote_targets_rejects_empty_bucket|test_list_remote_targets_rejects_invalid_bucket|test_remove_remote_target_rejects_missing_target|test_remove_remote_target_rejects_missing_arn|test_remove_remote_target_rejects_invalid_bucket|test_remove_remote_target_rejects_target_used_by_replication|test_delete_bucket_replication_removes_remote_target)$/)
|
| test(/^replication_extension_test::(test_replication_check_succeeds_with_remote_target|test_replication_check_rejects_target_without_object_lock|test_set_remote_target_rejects_unversioned_source_bucket|test_replication_check_rejects_unversioned_source_bucket|test_replication_check_rejects_missing_replication_config|test_replication_check_rejects_invalid_bucket|test_set_remote_target_rejects_same_bucket_on_same_deployment|test_set_remote_target_rejects_unversioned_target_bucket|test_set_remote_target_update_requires_arn|test_set_remote_target_update_rejects_missing_target|test_set_remote_target_rejects_invalid_target_url|test_set_remote_target_rejects_self_signed_https_target_without_skip_tls_verify|test_set_remote_target_rejects_private_ca_https_target_without_ca_cert_pem|test_list_remote_targets_rejects_empty_bucket|test_list_remote_targets_rejects_invalid_bucket|test_remove_remote_target_rejects_missing_target|test_remove_remote_target_rejects_missing_arn|test_remove_remote_target_rejects_invalid_bucket|test_remove_remote_target_rejects_target_used_by_replication|test_delete_bucket_replication_removes_remote_target)$/)
|
||||||
| test(/^reliant::lifecycle::/)
|
| test(/^reliant::lifecycle::/)
|
||||||
| test(/^reliant::tiering::/)
|
| test(/^reliant::tiering::/)
|
||||||
@@ -328,8 +309,9 @@ slow-timeout = { period = "60s", terminate-after = 2, grace-period = "10s" }
|
|||||||
# the STS dual-node test actually exercises its path (it skips gracefully with
|
# the STS dual-node test actually exercises its path (it skips gracefully with
|
||||||
# a visible log line when awscurl is absent), and routes scheduled failures
|
# a visible log line when awscurl is absent), and routes scheduled failures
|
||||||
# through .github/actions/schedule-failure-issue (ci-8). Explicit division of
|
# through .github/actions/schedule-failure-issue (ci-8). Explicit division of
|
||||||
# labor with e2e-full: these tests run only in the consolidated nightly
|
# labor with ci-5's future e2e-full merge gate: these tests run ONLY here, not
|
||||||
# workflow, not in the merge/main lane.
|
# double-run there. TODO(ci-7): fold this interim repl-owned lane into the ci
|
||||||
|
# domain's consolidated scheduled e2e workflow once it exists.
|
||||||
[profile.e2e-repl-nightly]
|
[profile.e2e-repl-nightly]
|
||||||
default-filter = """
|
default-filter = """
|
||||||
package(e2e_test)
|
package(e2e_test)
|
||||||
@@ -343,60 +325,26 @@ fail-fast = false
|
|||||||
# workflow as the failure-triage artifact.
|
# workflow as the failure-triage artifact.
|
||||||
path = "junit.xml"
|
path = "junit.xml"
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# e2e-nightly profile — destructive multi-process cluster fault domains
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# These seven modules are deliberately outside e2e-full's merge budget. Each
|
|
||||||
# starts a real multi-process or multi-disk topology and exercises node/disk
|
|
||||||
# loss, quorum, cleanup, notification fan-in, or admin-timeout behavior. The
|
|
||||||
# consolidated nightly workflow runs them serially to avoid resource
|
|
||||||
# starvation; failures are never retried.
|
|
||||||
[profile.e2e-nightly]
|
|
||||||
default-filter = """
|
|
||||||
package(e2e_test)
|
|
||||||
& test(/^(admin_timeout_regression_test|cluster_concurrency_test|cluster_multidrive_pool_test|heal_erasure_disk_rebuild_test|namespace_lock_quorum_test|object_lambda_test|stale_multipart_cleanup_cluster_test)::/)
|
|
||||||
"""
|
|
||||||
fail-fast = false
|
|
||||||
|
|
||||||
[profile.e2e-nightly.junit]
|
|
||||||
path = "junit.xml"
|
|
||||||
|
|
||||||
[[profile.e2e-nightly.overrides]]
|
|
||||||
filter = 'package(e2e_test)'
|
|
||||||
test-group = 'e2e-cluster-nightly'
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# e2e-protocols profile — serial protocol lane
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# The suite owns fixed ports, so the nightly workflow runs this exact profile
|
|
||||||
# with one nextest worker.
|
|
||||||
[profile.e2e-protocols]
|
|
||||||
default-filter = 'package(e2e_test) & test(/^protocols::/)'
|
|
||||||
fail-fast = false
|
|
||||||
|
|
||||||
[profile.e2e-protocols.junit]
|
|
||||||
path = "junit.xml"
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# e2e-full profile — merge-gate full single-node e2e lane (backlog#1149 ci-5)
|
# e2e-full profile — merge-gate full single-node e2e lane (backlog#1149 ci-5)
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# The merge gate (ci.yml `e2e-full` job: push main + merge_group +
|
# The merge gate (ci.yml `e2e-full` job: push main + merge_group +
|
||||||
# workflow_dispatch). Runs the user-visible KMS, object-lock, multipart-auth,
|
# workflow_dispatch). Runs the never-automated user-visible suites — KMS (40),
|
||||||
# quota, checksum, encryption,
|
# object_lock (33), multipart_auth (109), quota, checksum, encryption,
|
||||||
# security-boundary, ... — that the fast PR `e2e-smoke` subset deliberately
|
# security-boundary, ... — that the fast PR `e2e-smoke` subset deliberately
|
||||||
# skips. Budget <= 45 min; authority for the suite count is `cargo nextest list
|
# skips. Budget <= 45 min; authority for the suite count is `cargo nextest list
|
||||||
# --profile e2e-full` (see docs/testing/e2e-suite-inventory.md).
|
# --profile e2e-full` (see docs/testing/e2e-suite-inventory.md).
|
||||||
#
|
#
|
||||||
# The filter is "the whole e2e_test crate MINUS the sets owned by other lanes":
|
# The filter is "the whole e2e_test crate MINUS the sets owned by other lanes":
|
||||||
# * protocols:: — FTPS/SFTP/WebDAV, run from the dedicated protocol profile
|
# * protocols:: — FTPS/SFTP/WebDAV, still pinned to --test-threads=1 by fixed
|
||||||
# with one worker because the suite owns fixed ports.
|
# ports; they join a scheduled lane once ci-6 randomises the ports (ci-7).
|
||||||
# * the 7 cluster suites that spin up a RustFSTestClusterEnvironment
|
# * the 7 cluster suites that spin up a RustFSTestClusterEnvironment
|
||||||
# (cluster_concurrency, cluster_multidrive_pool, stale_multipart_cleanup_cluster,
|
# (cluster_concurrency, cluster_multidrive_pool, stale_multipart_cleanup_cluster,
|
||||||
# namespace_lock_quorum, heal_erasure_disk_rebuild, admin_timeout_regression,
|
# namespace_lock_quorum, heal_erasure_disk_rebuild, admin_timeout_regression,
|
||||||
# object_lambda) — too heavy for the merge budget; they run in the
|
# object_lambda) — too heavy for the merge budget; they run in ci-7's
|
||||||
# e2e-nightly serial cluster-fault lane.
|
# nightly 4-node lane.
|
||||||
# * replication_extension_test — repl-1 already splits it into the PR
|
# * replication_extension_test — repl-1 already splits it into the PR
|
||||||
# `e2e-smoke` (20 fast) and `e2e-repl-nightly` (55 slow) lanes and reserves
|
# `e2e-smoke` (20 fast) and `e2e-repl-nightly` (49 slow) lanes and reserves
|
||||||
# it for those, so e2e-full does not double-run it.
|
# it for those, so e2e-full does not double-run it.
|
||||||
# * #[ignore]d tests — nextest skips them by default (no --run-ignored); the
|
# * #[ignore]d tests — nextest skips them by default (no --run-ignored); the
|
||||||
# manual-localhost:9000 reliant/policy tests are ci-13's migration.
|
# manual-localhost:9000 reliant/policy tests are ci-13's migration.
|
||||||
@@ -435,7 +383,7 @@ path = "junit.xml"
|
|||||||
# quarantine: no retries, just single-threaded so several 4-disk servers never
|
# quarantine: no retries, just single-threaded so several 4-disk servers never
|
||||||
# run concurrently.
|
# run concurrently.
|
||||||
[[profile.e2e-full.overrides]]
|
[[profile.e2e-full.overrides]]
|
||||||
filter = 'package(e2e_test) & test(/^(reliability_disk_fault|degraded_read_eof_regression|replacement_privileged_e2e)_test::/)'
|
filter = 'package(e2e_test) & test(/^(reliability_disk_fault|degraded_read_eof_regression)_test::/)'
|
||||||
test-group = 'e2e-reliability'
|
test-group = 'e2e-reliability'
|
||||||
|
|
||||||
[[profile.e2e-full.overrides]]
|
[[profile.e2e-full.overrides]]
|
||||||
|
|||||||
@@ -17,11 +17,9 @@
|
|||||||
# =============================================================================
|
# =============================================================================
|
||||||
#
|
#
|
||||||
# Metric source: the KMS operation-policy choke point in
|
# Metric source: the KMS operation-policy choke point in
|
||||||
# crates/kms/src/policy.rs, except KmsKeyRotationOverdue, which reads the
|
# crates/kms/src/policy.rs. All label values are bounded static strings
|
||||||
# label-less key-lifecycle gauge published by the deletion worker's sweep
|
# (operation, op_class, outcome, error_class, backend, scope); key identifiers,
|
||||||
# (crates/kms/src/deletion_worker.rs). All label values are bounded static
|
# key material, and tokens never appear in labels.
|
||||||
# strings (operation, op_class, outcome, error_class, backend, scope); key
|
|
||||||
# identifiers, key material, and tokens never appear in labels.
|
|
||||||
#
|
#
|
||||||
# Response procedures: docs/operations/kms-observability-runbook.md
|
# Response procedures: docs/operations/kms-observability-runbook.md
|
||||||
#
|
#
|
||||||
@@ -214,38 +212,3 @@ groups:
|
|||||||
circuit_open until the half-open probe succeeds or returns
|
circuit_open until the half-open probe succeeds or returns
|
||||||
a non-retryable failure.
|
a non-retryable failure.
|
||||||
runbook_url: "https://github.com/rustfs/rustfs/blob/main/docs/operations/kms-observability-runbook.md#kmsbackendcircuitopen"
|
runbook_url: "https://github.com/rustfs/rustfs/blob/main/docs/operations/kms-observability-runbook.md#kmsbackendcircuitopen"
|
||||||
|
|
||||||
# ------------------------------------------------------------------
|
|
||||||
# 7. KmsKeyRotationOverdue
|
|
||||||
# The least recently rotated usable key has gone more than 400
|
|
||||||
# days without a rotation (measured from creation for keys with
|
|
||||||
# no recorded rotation). Direct gauge state published by the
|
|
||||||
# deletion worker's sweep, so no traffic guard applies; the
|
|
||||||
# one-hour hold only bridges scrape gaps. The worker runs only
|
|
||||||
# on backends with the schedule_deletion capability, so on the
|
|
||||||
# Static backend the series never exists and this alert cannot
|
|
||||||
# fire — that backend cannot rotate either; see the rotation
|
|
||||||
# driver matrix in docs/operations/kms-backend-security.md.
|
|
||||||
# Threshold: 400 days — conservative default sitting above a
|
|
||||||
# one-year rotation policy. Align it with the rotation period
|
|
||||||
# your compliance policy requires, and with
|
|
||||||
# RUSTFS_KMS_ROTATION_MAX_AGE_SECS so the per-key rotation_due
|
|
||||||
# verdict and this aggregate alert agree.
|
|
||||||
# ------------------------------------------------------------------
|
|
||||||
- alert: KmsKeyRotationOverdue
|
|
||||||
expr: |
|
|
||||||
rustfs_kms_oldest_key_rotation_age_seconds > (400 * 86400)
|
|
||||||
for: 1h
|
|
||||||
labels:
|
|
||||||
severity: warning
|
|
||||||
component: kms
|
|
||||||
annotations:
|
|
||||||
summary: "Oldest KMS key unrotated for more than 400 days"
|
|
||||||
description: >-
|
|
||||||
The least recently rotated usable KMS key was last rotated
|
|
||||||
{{ $value | humanizeDuration }} ago (measured from creation
|
|
||||||
for keys with no recorded rotation). List keys through the
|
|
||||||
admin API and read rotation_due / rotation_due_reason for
|
|
||||||
the per-key verdict; an "unsupported" reason means the
|
|
||||||
backend cannot rotate at all.
|
|
||||||
runbook_url: "https://github.com/rustfs/rustfs/blob/main/docs/operations/kms-observability-runbook.md#kmskeyrotationoverdue"
|
|
||||||
|
|||||||
@@ -85,7 +85,7 @@ runs:
|
|||||||
repo-token: ${{ github.token }}
|
repo-token: ${{ github.token }}
|
||||||
|
|
||||||
- name: Install flatc
|
- name: Install flatc
|
||||||
uses: Nugine/setup-flatc@698800de72a96bfb22cf60431dc21a2ff9a7e07b # v1
|
uses: Nugine/setup-flatc@e7855e994773ce90094a3f1626d4afc9080c23ae # v1
|
||||||
with:
|
with:
|
||||||
version: "25.12.19"
|
version: "25.12.19"
|
||||||
|
|
||||||
|
|||||||
@@ -46,9 +46,10 @@ lists when upstream changes.
|
|||||||
the PR.
|
the PR.
|
||||||
- **Weekly + manual**: `.github/workflows/e2e-s3tests.yml` runs the full
|
- **Weekly + manual**: `.github/workflows/e2e-s3tests.yml` runs the full
|
||||||
upstream suite (`TEST_SCOPE=all`) against a Docker deployment (single node
|
upstream suite (`TEST_SCOPE=all`) against a Docker deployment (single node
|
||||||
or a 4-node distributed cluster behind HAProxy). The canonical gate policy
|
or a 4-node distributed cluster behind HAProxy). It fails only on
|
||||||
and compatibility-report behavior are documented in
|
regressions in the implemented whitelist and publishes a classification
|
||||||
[`scripts/s3-tests/README.md`](../../scripts/s3-tests/README.md).
|
report (`compat-report.md`, also shown in the job summary) listing promotion
|
||||||
|
candidates and unclassified tests.
|
||||||
|
|
||||||
## Running Tests Locally
|
## Running Tests Locally
|
||||||
|
|
||||||
|
|||||||
@@ -53,9 +53,9 @@ jobs:
|
|||||||
persist-credentials: false
|
persist-credentials: false
|
||||||
|
|
||||||
- name: Install ripgrep
|
- name: Install ripgrep
|
||||||
uses: taiki-e/install-action@bffeee26d4db9be238a4ea78d8826604ebcb594d # v2
|
run: |
|
||||||
with:
|
sudo apt-get update
|
||||||
tool: ripgrep@15.2.0
|
sudo apt-get install -y ripgrep
|
||||||
|
|
||||||
- name: Check architecture migration rules
|
- name: Check architecture migration rules
|
||||||
run: ./scripts/check_architecture_migration_rules.sh
|
run: ./scripts/check_architecture_migration_rules.sh
|
||||||
|
|||||||
@@ -182,12 +182,7 @@ jobs:
|
|||||||
echo '```'
|
echo '```'
|
||||||
} >> "$GITHUB_STEP_SUMMARY"
|
} >> "$GITHUB_STEP_SUMMARY"
|
||||||
|
|
||||||
# Readers: test-and-lint-rio-v2 (per-PR), build-rustfs-debug-binary-rio-v2
|
# Readers: test-and-lint-rio-v2, build-rustfs-debug-binary-rio-v2.
|
||||||
# (weekly schedule / manual dispatch only — dormant rio-v2 variant, see
|
|
||||||
# rustfs/backlog#1835 and docs/architecture/minio-file-format-compat.md).
|
|
||||||
# The second build below stays despite the reduced cadence: it warms the
|
|
||||||
# rio-v2,e2e-test-hooks feature resolution the scheduled build restores,
|
|
||||||
# which keeps that lane inside its 30-minute timeout.
|
|
||||||
warm-ci-feat-rio:
|
warm-ci-feat-rio:
|
||||||
name: Warm ci-feat-rio
|
name: Warm ci-feat-rio
|
||||||
runs-on: sm-standard-4
|
runs-on: sm-standard-4
|
||||||
|
|||||||
@@ -83,9 +83,7 @@ jobs:
|
|||||||
persist-credentials: false
|
persist-credentials: false
|
||||||
|
|
||||||
- name: Install ripgrep
|
- name: Install ripgrep
|
||||||
uses: taiki-e/install-action@bffeee26d4db9be238a4ea78d8826604ebcb594d # v2
|
run: sudo apt-get update && sudo apt-get install -y ripgrep
|
||||||
with:
|
|
||||||
tool: ripgrep@15.2.0
|
|
||||||
|
|
||||||
- name: Install Rust toolchain
|
- name: Install Rust toolchain
|
||||||
uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
|
uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
|
||||||
@@ -119,15 +117,6 @@ jobs:
|
|||||||
- name: Check s3s footprint ratchet
|
- name: Check s3s footprint ratchet
|
||||||
run: ./scripts/check_s3s_footprint.sh
|
run: ./scripts/check_s3s_footprint.sh
|
||||||
|
|
||||||
- name: Check cryptographic capability wording
|
|
||||||
run: ./scripts/check_fips_wording.sh
|
|
||||||
|
|
||||||
- name: Check no embedded secret material
|
|
||||||
run: ./scripts/check_embedded_secrets.sh
|
|
||||||
|
|
||||||
- name: Check test wiring
|
|
||||||
run: python3 ./scripts/check_test_wiring.py
|
|
||||||
|
|
||||||
- name: Check no planning docs committed
|
- name: Check no planning docs committed
|
||||||
run: ./scripts/check_no_planning_docs.sh
|
run: ./scripts/check_no_planning_docs.sh
|
||||||
|
|
||||||
|
|||||||
@@ -118,9 +118,7 @@ jobs:
|
|||||||
persist-credentials: false
|
persist-credentials: false
|
||||||
|
|
||||||
- name: Install ripgrep
|
- name: Install ripgrep
|
||||||
uses: taiki-e/install-action@bffeee26d4db9be238a4ea78d8826604ebcb594d # v2
|
run: sudo apt-get update && sudo apt-get install -y ripgrep
|
||||||
with:
|
|
||||||
tool: ripgrep@15.2.0
|
|
||||||
|
|
||||||
- name: Install Rust toolchain
|
- name: Install Rust toolchain
|
||||||
uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
|
uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
|
||||||
@@ -154,15 +152,6 @@ jobs:
|
|||||||
- name: Check s3s footprint ratchet
|
- name: Check s3s footprint ratchet
|
||||||
run: ./scripts/check_s3s_footprint.sh
|
run: ./scripts/check_s3s_footprint.sh
|
||||||
|
|
||||||
- name: Check cryptographic capability wording
|
|
||||||
run: ./scripts/check_fips_wording.sh
|
|
||||||
|
|
||||||
- name: Check no embedded secret material
|
|
||||||
run: ./scripts/check_embedded_secrets.sh
|
|
||||||
|
|
||||||
- name: Check test wiring
|
|
||||||
run: python3 ./scripts/check_test_wiring.py
|
|
||||||
|
|
||||||
- name: Check no planning docs committed
|
- name: Check no planning docs committed
|
||||||
run: ./scripts/check_no_planning_docs.sh
|
run: ./scripts/check_no_planning_docs.sh
|
||||||
|
|
||||||
@@ -544,12 +533,7 @@ jobs:
|
|||||||
|
|
||||||
build-rustfs-debug-binary-rio-v2:
|
build-rustfs-debug-binary-rio-v2:
|
||||||
name: Build RustFS Debug Binary (rio-v2)
|
name: Build RustFS Debug Binary (rio-v2)
|
||||||
# Dormant rio-v2 variant (rustfs/backlog#1835): the feature ships in no
|
if: github.event_name != 'pull_request' || github.event.action != 'closed'
|
||||||
# default build, so this full-suite lane runs only on the weekly schedule
|
|
||||||
# and manual dispatch. Per-PR cfg-seam coverage stays with
|
|
||||||
# test-and-lint-rio-v2. Lifecycle and the promote-or-delete condition:
|
|
||||||
# docs/architecture/minio-file-format-compat.md ("rio-v2 variant lifecycle").
|
|
||||||
if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
|
|
||||||
needs: [ quick-checks ]
|
needs: [ quick-checks ]
|
||||||
runs-on: sm-standard-4
|
runs-on: sm-standard-4
|
||||||
timeout-minutes: 30
|
timeout-minutes: 30
|
||||||
@@ -689,9 +673,9 @@ jobs:
|
|||||||
- name: Make binary executable
|
- name: Make binary executable
|
||||||
run: chmod +x ./target/debug/rustfs
|
run: chmod +x ./target/debug/rustfs
|
||||||
|
|
||||||
# Build the e2e test graph once. The archive is reused by the smoke
|
# Build the e2e test graph once. The archive is reused by the security
|
||||||
# selection guard, security exact-count check, and run below, avoiding a
|
# count-floor check and the smoke run below, avoiding a second compile of
|
||||||
# second compile of the same e2e_test target on cold runners (backlog#1645).
|
# the same e2e_test target on cold runners (backlog#1645).
|
||||||
- name: Archive e2e smoke test binaries
|
- name: Archive e2e smoke test binaries
|
||||||
env:
|
env:
|
||||||
NEXTEST_ARCHIVE: ${{ runner.temp }}/rustfs-e2e-smoke.tar.zst
|
NEXTEST_ARCHIVE: ${{ runner.temp }}/rustfs-e2e-smoke.tar.zst
|
||||||
@@ -699,7 +683,6 @@ jobs:
|
|||||||
run: |
|
run: |
|
||||||
cargo nextest archive --profile e2e-smoke -p e2e_test --archive-file "${NEXTEST_ARCHIVE}"
|
cargo nextest archive --profile e2e-smoke -p e2e_test --archive-file "${NEXTEST_ARCHIVE}"
|
||||||
cargo nextest list --profile e2e-smoke --archive-file "${NEXTEST_ARCHIVE}" --message-format json > "${NEXTEST_LISTING}"
|
cargo nextest list --profile e2e-smoke --archive-file "${NEXTEST_ARCHIVE}" --message-format json > "${NEXTEST_LISTING}"
|
||||||
python3 ./scripts/check_test_wiring.py --check-profile e2e-smoke "${NEXTEST_LISTING}"
|
|
||||||
./scripts/check_security_smoke_count.sh check "${NEXTEST_LISTING}"
|
./scripts/check_security_smoke_count.sh check "${NEXTEST_LISTING}"
|
||||||
|
|
||||||
# PR smoke subset of the in-repo e2e suite (backlog#1149 ci-4). The
|
# PR smoke subset of the in-repo e2e suite (backlog#1149 ci-4). The
|
||||||
@@ -764,7 +747,7 @@ jobs:
|
|||||||
# suites — KMS, object_lock, multipart_auth, quota, checksum, encryption,
|
# suites — KMS, object_lock, multipart_auth, quota, checksum, encryption,
|
||||||
# security-boundary, ... — via the e2e-full nextest profile. Too heavy for
|
# security-boundary, ... — via the e2e-full nextest profile. Too heavy for
|
||||||
# every PR, so it is gated to main pushes, the merge queue, and manual
|
# every PR, so it is gated to main pushes, the merge queue, and manual
|
||||||
# dispatch. protocols / the 7 cluster suites / replication / #[ignore] are
|
# dispatch. protocols / the 6 cluster suites / replication / #[ignore] are
|
||||||
# owned by other lanes (see .config/nextest.toml profile.e2e-full).
|
# owned by other lanes (see .config/nextest.toml profile.e2e-full).
|
||||||
if: >-
|
if: >-
|
||||||
github.event_name == 'workflow_dispatch' ||
|
github.event_name == 'workflow_dispatch' ||
|
||||||
@@ -824,13 +807,6 @@ jobs:
|
|||||||
- name: Make binary executable
|
- name: Make binary executable
|
||||||
run: chmod +x ./target/debug/rustfs
|
run: chmod +x ./target/debug/rustfs
|
||||||
|
|
||||||
- name: Verify e2e full membership
|
|
||||||
env:
|
|
||||||
NEXTEST_LISTING: ${{ runner.temp }}/rustfs-e2e-full-list.json
|
|
||||||
run: |
|
|
||||||
cargo nextest list --profile e2e-full -p e2e_test --message-format json > "${NEXTEST_LISTING}"
|
|
||||||
python3 ./scripts/check_test_wiring.py --check-profile e2e-full "${NEXTEST_LISTING}"
|
|
||||||
|
|
||||||
# Full single-node e2e lane (backlog#1149 ci-5). The e2e-full
|
# Full single-node e2e lane (backlog#1149 ci-5). The e2e-full
|
||||||
# default-filter in .config/nextest.toml is the single wiring mechanism —
|
# default-filter in .config/nextest.toml is the single wiring mechanism —
|
||||||
# extend that filter, never add ad-hoc e2e jobs here. Reuses the downloaded
|
# extend that filter, never add ad-hoc e2e jobs here. Reuses the downloaded
|
||||||
@@ -843,16 +819,11 @@ jobs:
|
|||||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||||
with:
|
with:
|
||||||
name: e2e-full-junit-${{ github.run_number }}
|
name: e2e-full-junit-${{ github.run_number }}
|
||||||
path: |
|
path: target/nextest/e2e-full/junit.xml
|
||||||
target/nextest/e2e-full/junit.xml
|
|
||||||
${{ runner.temp }}/rustfs-e2e-full-list.json
|
|
||||||
retention-days: 7
|
retention-days: 7
|
||||||
|
|
||||||
e2e-tests-rio-v2:
|
e2e-tests-rio-v2:
|
||||||
name: End-to-End Tests (rio-v2)
|
name: End-to-End Tests (rio-v2)
|
||||||
# Inherits the schedule/dispatch-only gate through needs: on every other
|
|
||||||
# event build-rustfs-debug-binary-rio-v2 is skipped, so this job skips
|
|
||||||
# with it (see the dormant-variant comment on that job).
|
|
||||||
needs: [ build-rustfs-debug-binary-rio-v2 ]
|
needs: [ build-rustfs-debug-binary-rio-v2 ]
|
||||||
runs-on: sm-standard-2
|
runs-on: sm-standard-2
|
||||||
timeout-minutes: 30
|
timeout-minutes: 30
|
||||||
|
|||||||
@@ -94,7 +94,6 @@ jobs:
|
|||||||
short_sha: ${{ steps.check.outputs.short_sha }}
|
short_sha: ${{ steps.check.outputs.short_sha }}
|
||||||
is_prerelease: ${{ steps.check.outputs.is_prerelease }}
|
is_prerelease: ${{ steps.check.outputs.is_prerelease }}
|
||||||
create_latest: ${{ steps.check.outputs.create_latest }}
|
create_latest: ${{ steps.check.outputs.create_latest }}
|
||||||
source_ref: ${{ steps.check.outputs.source_ref }}
|
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout repository
|
- name: Checkout repository
|
||||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||||
@@ -119,7 +118,6 @@ jobs:
|
|||||||
short_sha=""
|
short_sha=""
|
||||||
is_prerelease=false
|
is_prerelease=false
|
||||||
create_latest=false
|
create_latest=false
|
||||||
source_ref="$GITHUB_SHA"
|
|
||||||
|
|
||||||
if [[ "${{ github.event_name }}" == "workflow_run" ]]; then
|
if [[ "${{ github.event_name }}" == "workflow_run" ]]; then
|
||||||
# Triggered by build workflow completion
|
# Triggered by build workflow completion
|
||||||
@@ -139,7 +137,6 @@ jobs:
|
|||||||
# Extract version info from commit message or use commit SHA
|
# Extract version info from commit message or use commit SHA
|
||||||
# Use Git to generate consistent short SHA (ensures uniqueness like build.yml)
|
# Use Git to generate consistent short SHA (ensures uniqueness like build.yml)
|
||||||
short_sha=$(git rev-parse --short "$HEAD_SHA")
|
short_sha=$(git rev-parse --short "$HEAD_SHA")
|
||||||
source_ref="$HEAD_SHA"
|
|
||||||
|
|
||||||
# Determine build type based on triggering workflow event and ref
|
# Determine build type based on triggering workflow event and ref
|
||||||
triggering_event="$TRIGGERING_EVENT"
|
triggering_event="$TRIGGERING_EVENT"
|
||||||
@@ -264,23 +261,6 @@ jobs:
|
|||||||
echo "⚠️ Only release versions (latest, v1.0.0, 1.0.0) and prereleases (v1.0.0-alpha1, 1.0.0-beta2) are supported"
|
echo "⚠️ Only release versions (latest, v1.0.0, 1.0.0) and prereleases (v1.0.0-alpha1, 1.0.0-beta2) are supported"
|
||||||
;;
|
;;
|
||||||
esac
|
esac
|
||||||
|
|
||||||
if [[ "$should_build" == true && "$input_version" != "latest" ]]; then
|
|
||||||
tag_ref="refs/tags/$input_version"
|
|
||||||
if ! git ls-remote --exit-code origin "$tag_ref" >/dev/null 2>&1; then
|
|
||||||
if [[ "$input_version" == v* ]]; then
|
|
||||||
tag_ref="refs/tags/${input_version#v}"
|
|
||||||
else
|
|
||||||
tag_ref="refs/tags/v$input_version"
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
if ! git ls-remote --exit-code origin "$tag_ref" >/dev/null 2>&1; then
|
|
||||||
echo "❌ Release tag not found for Docker build: $input_version"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
source_ref="$tag_ref"
|
|
||||||
fi
|
|
||||||
fi
|
fi
|
||||||
|
|
||||||
{
|
{
|
||||||
@@ -291,7 +271,6 @@ jobs:
|
|||||||
echo "short_sha=$short_sha"
|
echo "short_sha=$short_sha"
|
||||||
echo "is_prerelease=$is_prerelease"
|
echo "is_prerelease=$is_prerelease"
|
||||||
echo "create_latest=$create_latest"
|
echo "create_latest=$create_latest"
|
||||||
echo "source_ref=$source_ref"
|
|
||||||
} >> "$GITHUB_OUTPUT"
|
} >> "$GITHUB_OUTPUT"
|
||||||
|
|
||||||
echo "🐳 Docker Build Summary:"
|
echo "🐳 Docker Build Summary:"
|
||||||
@@ -302,7 +281,6 @@ jobs:
|
|||||||
echo " - Short SHA: $short_sha"
|
echo " - Short SHA: $short_sha"
|
||||||
echo " - Is prerelease: $is_prerelease"
|
echo " - Is prerelease: $is_prerelease"
|
||||||
echo " - Create latest: $create_latest"
|
echo " - Create latest: $create_latest"
|
||||||
echo " - Source ref: $source_ref"
|
|
||||||
|
|
||||||
# Build multi-arch Docker images
|
# Build multi-arch Docker images
|
||||||
# Strategy: Build images using pre-built binaries from dl.rustfs.com
|
# Strategy: Build images using pre-built binaries from dl.rustfs.com
|
||||||
@@ -330,7 +308,6 @@ jobs:
|
|||||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||||
with:
|
with:
|
||||||
persist-credentials: false
|
persist-credentials: false
|
||||||
ref: ${{ needs.build-check.outputs.source_ref }}
|
|
||||||
|
|
||||||
- name: Login to Docker Hub
|
- name: Login to Docker Hub
|
||||||
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
|
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
|
||||||
@@ -420,8 +397,7 @@ jobs:
|
|||||||
LABELS="org.opencontainers.image.title=RustFS"
|
LABELS="org.opencontainers.image.title=RustFS"
|
||||||
LABELS="$LABELS,org.opencontainers.image.description=RustFS distributed object storage system"
|
LABELS="$LABELS,org.opencontainers.image.description=RustFS distributed object storage system"
|
||||||
LABELS="$LABELS,org.opencontainers.image.version=$VERSION"
|
LABELS="$LABELS,org.opencontainers.image.version=$VERSION"
|
||||||
SOURCE_REVISION="$(git rev-parse HEAD)"
|
LABELS="$LABELS,org.opencontainers.image.revision=${{ github.sha }}"
|
||||||
LABELS="$LABELS,org.opencontainers.image.revision=$SOURCE_REVISION"
|
|
||||||
LABELS="$LABELS,org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }}"
|
LABELS="$LABELS,org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }}"
|
||||||
LABELS="$LABELS,org.opencontainers.image.created=$(date -u +'%Y-%m-%dT%H:%M:%SZ')"
|
LABELS="$LABELS,org.opencontainers.image.created=$(date -u +'%Y-%m-%dT%H:%M:%SZ')"
|
||||||
LABELS="$LABELS,org.opencontainers.image.build-type=$BUILD_TYPE"
|
LABELS="$LABELS,org.opencontainers.image.build-type=$BUILD_TYPE"
|
||||||
|
|||||||
@@ -12,7 +12,7 @@
|
|||||||
# See the License for the specific language governing permissions and
|
# See the License for the specific language governing permissions and
|
||||||
# limitations under the License.
|
# limitations under the License.
|
||||||
|
|
||||||
# Consolidated nightly e2e lane for replication, cluster faults, and protocols.
|
# Nightly full replication e2e lane (backlog#1147 repl-1, deps: ci-4).
|
||||||
#
|
#
|
||||||
# The per-PR gate (ci.yml `e2e-tests` job, `--profile e2e-smoke`) runs the
|
# The per-PR gate (ci.yml `e2e-tests` job, `--profile e2e-smoke`) runs the
|
||||||
# FAST replication tests. This scheduled lane runs the remaining heavier
|
# FAST replication tests. This scheduled lane runs the remaining heavier
|
||||||
@@ -28,12 +28,15 @@
|
|||||||
# add ad-hoc cargo-test steps here; change the filterset instead. The
|
# add ad-hoc cargo-test steps here; change the filterset instead. The
|
||||||
# authoritative membership and count come from
|
# authoritative membership and count come from
|
||||||
# `cargo nextest list -p e2e_test --profile e2e-repl-nightly`; the PR/nightly
|
# `cargo nextest list -p e2e_test --profile e2e-repl-nightly`; the PR/nightly
|
||||||
# selection digest is committed under .config/.
|
# count invariant is maintained next to the filtersets in .config/nextest.toml
|
||||||
|
# (deliberately not duplicated here).
|
||||||
#
|
#
|
||||||
# Explicit division of labor: these subsets run only here and never double-run
|
# Explicit division of labor: the nightly subset runs ONLY here, never double-run
|
||||||
# in the e2e-full merge gate.
|
# in ci-5's future e2e-full merge gate. TODO(ci-7): once the ci domain's
|
||||||
|
# consolidated scheduled e2e workflow exists, fold this interim repl-owned lane
|
||||||
|
# into it rather than growing a second scheduled entrypoint.
|
||||||
|
|
||||||
name: e2e-nightly
|
name: e2e-replication-nightly
|
||||||
|
|
||||||
on:
|
on:
|
||||||
workflow_dispatch:
|
workflow_dispatch:
|
||||||
@@ -47,10 +50,6 @@ on:
|
|||||||
permissions:
|
permissions:
|
||||||
contents: read
|
contents: read
|
||||||
|
|
||||||
concurrency:
|
|
||||||
group: ${{ github.workflow }}-${{ github.ref }}
|
|
||||||
cancel-in-progress: false
|
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
repl-nightly:
|
repl-nightly:
|
||||||
name: Replication e2e (nightly)
|
name: Replication e2e (nightly)
|
||||||
@@ -98,20 +97,9 @@ jobs:
|
|||||||
# demand otherwise, but a single explicit build avoids several parallel
|
# demand otherwise, but a single explicit build avoids several parallel
|
||||||
# nextest test processes racing to build it at once.
|
# nextest test processes racing to build it at once.
|
||||||
- name: Build rustfs binary
|
- name: Build rustfs binary
|
||||||
run: |
|
run: cargo build -p rustfs --bins
|
||||||
cargo build -p rustfs --bins
|
|
||||||
: > target/debug/rustfs.features
|
|
||||||
|
|
||||||
- name: Verify replication e2e membership
|
|
||||||
env:
|
|
||||||
NEXTEST_LISTING: ${{ runner.temp }}/rustfs-e2e-repl-nightly-list.json
|
|
||||||
run: |
|
|
||||||
cargo nextest list --profile e2e-repl-nightly -p e2e_test --message-format json > "${NEXTEST_LISTING}"
|
|
||||||
python3 ./scripts/check_test_wiring.py --check-profile e2e-repl-nightly "${NEXTEST_LISTING}"
|
|
||||||
|
|
||||||
- name: Run replication e2e nightly suite
|
- name: Run replication e2e nightly suite
|
||||||
env:
|
|
||||||
RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-e2e-repl-nightly-logs
|
|
||||||
run: cargo nextest run --profile e2e-repl-nightly -p e2e_test
|
run: cargo nextest run --profile e2e-repl-nightly -p e2e_test
|
||||||
|
|
||||||
- name: Upload nextest junit report
|
- name: Upload nextest junit report
|
||||||
@@ -119,112 +107,13 @@ jobs:
|
|||||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||||
with:
|
with:
|
||||||
name: e2e-replication-nightly-junit-${{ github.run_number }}
|
name: e2e-replication-nightly-junit-${{ github.run_number }}
|
||||||
path: |
|
path: target/nextest/e2e-repl-nightly/junit.xml
|
||||||
target/nextest/e2e-repl-nightly/junit.xml
|
|
||||||
${{ runner.temp }}/rustfs-e2e-repl-nightly-list.json
|
|
||||||
${{ runner.temp }}/rustfs-e2e-repl-nightly-logs/
|
|
||||||
retention-days: 7
|
retention-days: 7
|
||||||
if-no-files-found: ignore
|
if-no-files-found: ignore
|
||||||
|
|
||||||
cluster-nightly:
|
|
||||||
name: Cluster fault e2e (nightly)
|
|
||||||
runs-on: sm-standard-4
|
|
||||||
timeout-minutes: 90
|
|
||||||
env:
|
|
||||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
|
||||||
steps:
|
|
||||||
- name: Checkout repository
|
|
||||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
|
||||||
with:
|
|
||||||
persist-credentials: false
|
|
||||||
|
|
||||||
- name: Setup Rust environment
|
|
||||||
uses: ./.github/actions/setup
|
|
||||||
with:
|
|
||||||
rust-version: stable
|
|
||||||
cache-shared-key: ci-e2e-nightly
|
|
||||||
cache-save-if: 'false'
|
|
||||||
install-build-packaging-tools: 'false'
|
|
||||||
|
|
||||||
- name: Build rustfs binary
|
|
||||||
run: |
|
|
||||||
cargo build -p rustfs --bins --features e2e-test-hooks
|
|
||||||
: > target/debug/rustfs.features
|
|
||||||
|
|
||||||
- name: Verify cluster fault e2e membership
|
|
||||||
env:
|
|
||||||
NEXTEST_LISTING: ${{ runner.temp }}/rustfs-e2e-nightly-list.json
|
|
||||||
run: |
|
|
||||||
cargo nextest list --profile e2e-nightly -p e2e_test --message-format json > "${NEXTEST_LISTING}"
|
|
||||||
python3 ./scripts/check_test_wiring.py --check-profile e2e-nightly "${NEXTEST_LISTING}"
|
|
||||||
|
|
||||||
- name: Run cluster fault e2e nightly suite
|
|
||||||
env:
|
|
||||||
RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-e2e-nightly-logs
|
|
||||||
run: cargo nextest run --profile e2e-nightly -p e2e_test
|
|
||||||
|
|
||||||
- name: Upload cluster fault diagnostics
|
|
||||||
if: always()
|
|
||||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
|
||||||
with:
|
|
||||||
name: e2e-cluster-nightly-${{ github.run_number }}
|
|
||||||
path: |
|
|
||||||
target/nextest/e2e-nightly/junit.xml
|
|
||||||
${{ runner.temp }}/rustfs-e2e-nightly-list.json
|
|
||||||
${{ runner.temp }}/rustfs-e2e-nightly-logs/
|
|
||||||
retention-days: 7
|
|
||||||
if-no-files-found: warn
|
|
||||||
|
|
||||||
protocols-nightly:
|
|
||||||
name: Protocol e2e (nightly)
|
|
||||||
runs-on: sm-standard-4
|
|
||||||
timeout-minutes: 90
|
|
||||||
env:
|
|
||||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
|
||||||
RUSTFS_BUILD_FEATURES: ftps,webdav,sftp
|
|
||||||
steps:
|
|
||||||
- name: Checkout repository
|
|
||||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
|
||||||
with:
|
|
||||||
persist-credentials: false
|
|
||||||
|
|
||||||
- name: Setup Rust environment
|
|
||||||
uses: ./.github/actions/setup
|
|
||||||
with:
|
|
||||||
rust-version: stable
|
|
||||||
cache-shared-key: ci-e2e-protocols
|
|
||||||
cache-save-if: 'false'
|
|
||||||
install-build-packaging-tools: 'false'
|
|
||||||
|
|
||||||
# The suite owns fixed protocol ports and serializes its internal cases.
|
|
||||||
- name: Verify protocol e2e membership
|
|
||||||
env:
|
|
||||||
NEXTEST_LISTING: ${{ runner.temp }}/rustfs-e2e-protocols-list.json
|
|
||||||
run: |
|
|
||||||
cargo nextest list --profile e2e-protocols -p e2e_test --message-format json > "${NEXTEST_LISTING}"
|
|
||||||
python3 ./scripts/check_test_wiring.py --check-profile e2e-protocols "${NEXTEST_LISTING}"
|
|
||||||
|
|
||||||
- name: Run protocol e2e nightly suite
|
|
||||||
env:
|
|
||||||
RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-protocol-e2e-logs
|
|
||||||
run: >-
|
|
||||||
cargo nextest run -j 1 --profile e2e-protocols -p e2e_test --no-capture
|
|
||||||
|
|
||||||
- name: Upload protocol diagnostics
|
|
||||||
if: always()
|
|
||||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
|
||||||
with:
|
|
||||||
name: e2e-protocol-nightly-${{ github.run_number }}
|
|
||||||
path: |
|
|
||||||
target/nextest/e2e-protocols/junit.xml
|
|
||||||
${{ runner.temp }}/rustfs-e2e-protocols-list.json
|
|
||||||
${{ runner.temp }}/rustfs-protocol-e2e-logs/
|
|
||||||
retention-days: 7
|
|
||||||
if-no-files-found: warn
|
|
||||||
|
|
||||||
alert-on-failure:
|
alert-on-failure:
|
||||||
name: Alert on scheduled failure
|
name: Alert on scheduled failure
|
||||||
needs: [repl-nightly, cluster-nightly, protocols-nightly]
|
needs: [repl-nightly]
|
||||||
# Only scheduled runs open/append the tracking issue (backlog#1149 ci-8);
|
# Only scheduled runs open/append the tracking issue (backlog#1149 ci-8);
|
||||||
# manual workflow_dispatch runs stay quiet so a debugging run never files a
|
# manual workflow_dispatch runs stay quiet so a debugging run never files a
|
||||||
# spurious alert.
|
# spurious alert.
|
||||||
|
|||||||
@@ -18,9 +18,10 @@
|
|||||||
# runs only the implemented_tests.txt whitelist. This workflow complements it:
|
# runs only the implemented_tests.txt whitelist. This workflow complements it:
|
||||||
#
|
#
|
||||||
# - Scheduled weekly full sweep (TEST_SCOPE=all): runs the ENTIRE upstream
|
# - Scheduled weekly full sweep (TEST_SCOPE=all): runs the ENTIRE upstream
|
||||||
# suite and reports promotion candidates. Regressions, unclassified tests,
|
# suite and reports promotion candidates (tests that newly pass) and
|
||||||
# incomplete execution, and infrastructure errors fail the job; classified
|
# unclassified tests. The job fails only on regressions in the implemented
|
||||||
# failures for not-yet-implemented features remain informational.
|
# whitelist or on infrastructure errors — expected failures from
|
||||||
|
# not-yet-implemented features do not turn the run red.
|
||||||
# - Manual runs (workflow_dispatch): same, with configurable mode/scope.
|
# - Manual runs (workflow_dispatch): same, with configurable mode/scope.
|
||||||
#
|
#
|
||||||
# All test execution is delegated to scripts/s3-tests/run.sh (single source of
|
# All test execution is delegated to scripts/s3-tests/run.sh (single source of
|
||||||
@@ -44,6 +45,13 @@
|
|||||||
# The PR gate (ci.yml s3-implemented-tests) is unaffected: it avoids Docker
|
# The PR gate (ci.yml s3-implemented-tests) is unaffected: it avoids Docker
|
||||||
# via DEPLOY_MODE=binary and defers all pip setup to run.sh's self-bootstrap.
|
# via DEPLOY_MODE=binary and defers all pip setup to run.sh's self-bootstrap.
|
||||||
|
|
||||||
|
# DISABLED. This workflow is switched off in the repository's Actions settings
|
||||||
|
# (state: disabled_manually) and does not run on any trigger, including its cron
|
||||||
|
# and workflow_dispatch. That state lives in GitHub's UI and is invisible when
|
||||||
|
# reading this file, which has already misled at least one audit — hence this
|
||||||
|
# banner. Re-enabling is a UI action; anyone doing so should first check that the
|
||||||
|
# workflow still matches the current CI layout. See rustfs/backlog#1603.
|
||||||
|
#
|
||||||
name: e2e-s3tests
|
name: e2e-s3tests
|
||||||
|
|
||||||
on:
|
on:
|
||||||
@@ -73,19 +81,6 @@ on:
|
|||||||
description: "Stop after N failures. '0' to run everything."
|
description: "Stop after N failures. '0' to run everything."
|
||||||
required: false
|
required: false
|
||||||
default: "0"
|
default: "0"
|
||||||
shard-count:
|
|
||||||
description: "Exact-node-ID shard count for a targeted manual run"
|
|
||||||
required: false
|
|
||||||
default: "1"
|
|
||||||
type: choice
|
|
||||||
options:
|
|
||||||
- "1"
|
|
||||||
- "2"
|
|
||||||
- "4"
|
|
||||||
shard-index:
|
|
||||||
description: "Zero-based shard index for a targeted manual run"
|
|
||||||
required: false
|
|
||||||
default: "0"
|
|
||||||
markexpr:
|
markexpr:
|
||||||
description: "Optional pytest -m expression"
|
description: "Optional pytest -m expression"
|
||||||
required: false
|
required: false
|
||||||
@@ -116,8 +111,6 @@ env:
|
|||||||
XDIST: ${{ github.event.inputs.xdist || '4' }}
|
XDIST: ${{ github.event.inputs.xdist || '4' }}
|
||||||
MAXFAIL: ${{ github.event.inputs.maxfail || '0' }}
|
MAXFAIL: ${{ github.event.inputs.maxfail || '0' }}
|
||||||
MARKEXPR: ${{ github.event.inputs.markexpr || '' }}
|
MARKEXPR: ${{ github.event.inputs.markexpr || '' }}
|
||||||
S3_SHARD_COUNT: ${{ github.event_name == 'schedule' && '4' || github.event.inputs.shard-count || '1' }}
|
|
||||||
TEST_TIMEOUT: "300"
|
|
||||||
|
|
||||||
concurrency:
|
concurrency:
|
||||||
group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event.inputs['test-mode'] || 'single' }}
|
group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event.inputs['test-mode'] || 'single' }}
|
||||||
@@ -134,22 +127,19 @@ defaults:
|
|||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
s3tests:
|
s3tests:
|
||||||
name: s3tests (${{ matrix.test-mode }}, shard ${{ matrix.shard-index }})
|
|
||||||
# GitHub-hosted: reliably provides Docker + docker compose + python3/pip.
|
# GitHub-hosted: reliably provides Docker + docker compose + python3/pip.
|
||||||
# See the header note (ci-1) for why the self-hosted sm-standard-4 label
|
# See the header note (ci-1) for why the self-hosted sm-standard-4 label
|
||||||
# was abandoned. Scheduled failures are handled by alert-on-failure below.
|
# was abandoned. TODO(ci-8): scheduled-failure alerting (auto-open issue)
|
||||||
|
# is added by the ci-8 composite action; do not implement it here.
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
timeout-minutes: 180
|
timeout-minutes: 180
|
||||||
strategy:
|
strategy:
|
||||||
fail-fast: false
|
fail-fast: false
|
||||||
max-parallel: 2
|
|
||||||
matrix:
|
matrix:
|
||||||
# Scheduled sweeps cover both topologies; manual runs use the input.
|
# Scheduled sweeps cover both topologies; manual runs use the input.
|
||||||
test-mode: ${{ github.event_name == 'schedule' && fromJSON('["single", "multi"]') || fromJSON(format('["{0}"]', github.event.inputs.test-mode || 'single')) }}
|
test-mode: ${{ github.event_name == 'schedule' && fromJSON('["single", "multi"]') || fromJSON(format('["{0}"]', github.event.inputs.test-mode || 'single')) }}
|
||||||
shard-index: ${{ github.event_name == 'schedule' && fromJSON('[0, 1, 2, 3]') || fromJSON(format('[{0}]', github.event.inputs.shard-index || '0')) }}
|
|
||||||
env:
|
env:
|
||||||
TEST_MODE: ${{ matrix.test-mode }}
|
TEST_MODE: ${{ matrix.test-mode }}
|
||||||
S3_SHARD_INDEX: ${{ matrix.shard-index }}
|
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||||
with:
|
with:
|
||||||
@@ -191,7 +181,6 @@ jobs:
|
|||||||
- name: Start single RustFS
|
- name: Start single RustFS
|
||||||
if: env.TEST_MODE == 'single'
|
if: env.TEST_MODE == 'single'
|
||||||
run: |
|
run: |
|
||||||
SSE_KEY="$(head -c 32 /dev/zero | base64 -w0)"
|
|
||||||
docker network inspect rustfs-net >/dev/null 2>&1 || docker network create rustfs-net
|
docker network inspect rustfs-net >/dev/null 2>&1 || docker network create rustfs-net
|
||||||
docker rm -f rustfs-single >/dev/null 2>&1 || true
|
docker rm -f rustfs-single >/dev/null 2>&1 || true
|
||||||
# The four disks share one physical device on the runner (a single
|
# The four disks share one physical device on the runner (a single
|
||||||
@@ -204,7 +193,6 @@ jobs:
|
|||||||
-e RUSTFS_ADDRESS=0.0.0.0:9000 \
|
-e RUSTFS_ADDRESS=0.0.0.0:9000 \
|
||||||
-e RUSTFS_ACCESS_KEY="${S3_ACCESS_KEY}" \
|
-e RUSTFS_ACCESS_KEY="${S3_ACCESS_KEY}" \
|
||||||
-e RUSTFS_SECRET_KEY="${S3_SECRET_KEY}" \
|
-e RUSTFS_SECRET_KEY="${S3_SECRET_KEY}" \
|
||||||
-e RUSTFS_SSE_S3_MASTER_KEY="${SSE_KEY}" \
|
|
||||||
-e RUSTFS_VOLUMES="/data/rustfs{0...3}" \
|
-e RUSTFS_VOLUMES="/data/rustfs{0...3}" \
|
||||||
-e RUSTFS_UNSAFE_BYPASS_DISK_CHECK=true \
|
-e RUSTFS_UNSAFE_BYPASS_DISK_CHECK=true \
|
||||||
-v /tmp/rustfs-single:/data \
|
-v /tmp/rustfs-single:/data \
|
||||||
@@ -213,7 +201,6 @@ jobs:
|
|||||||
- name: Start 4-node distributed cluster
|
- name: Start 4-node distributed cluster
|
||||||
if: env.TEST_MODE == 'multi'
|
if: env.TEST_MODE == 'multi'
|
||||||
run: |
|
run: |
|
||||||
SSE_KEY="$(head -c 32 /dev/zero | base64 -w0)"
|
|
||||||
# A real distributed deployment: every node lists all endpoints in
|
# A real distributed deployment: every node lists all endpoints in
|
||||||
# RUSTFS_VOLUMES so data is erasure-coded ACROSS nodes. Do not use
|
# RUSTFS_VOLUMES so data is erasure-coded ACROSS nodes. Do not use
|
||||||
# node-local volume paths here — that would create four independent
|
# node-local volume paths here — that would create four independent
|
||||||
@@ -226,7 +213,6 @@ jobs:
|
|||||||
RUSTFS_ADDRESS: "0.0.0.0:9000"
|
RUSTFS_ADDRESS: "0.0.0.0:9000"
|
||||||
RUSTFS_ACCESS_KEY: ${S3_ACCESS_KEY}
|
RUSTFS_ACCESS_KEY: ${S3_ACCESS_KEY}
|
||||||
RUSTFS_SECRET_KEY: ${S3_SECRET_KEY}
|
RUSTFS_SECRET_KEY: ${S3_SECRET_KEY}
|
||||||
RUSTFS_SSE_S3_MASTER_KEY: "${SSE_KEY}"
|
|
||||||
RUSTFS_VOLUMES: "http://rustfs{1...4}:9000/data/rustfs{0...3}"
|
RUSTFS_VOLUMES: "http://rustfs{1...4}:9000/data/rustfs{0...3}"
|
||||||
# Each node's four disks share one physical device inside its
|
# Each node's four disks share one physical device inside its
|
||||||
# container, so bypass the local physical-disk-independence guard
|
# container, so bypass the local physical-disk-independence guard
|
||||||
@@ -308,6 +294,7 @@ jobs:
|
|||||||
|
|
||||||
- name: Run ceph s3-tests
|
- name: Run ceph s3-tests
|
||||||
run: |
|
run: |
|
||||||
|
set +e
|
||||||
DEPLOY_MODE=existing \
|
DEPLOY_MODE=existing \
|
||||||
TEST_MODE="${TEST_MODE}" \
|
TEST_MODE="${TEST_MODE}" \
|
||||||
TEST_SCOPE="${TEST_SCOPE}" \
|
TEST_SCOPE="${TEST_SCOPE}" \
|
||||||
@@ -315,6 +302,26 @@ jobs:
|
|||||||
MAXFAIL="${MAXFAIL}" \
|
MAXFAIL="${MAXFAIL}" \
|
||||||
MARKEXPR="${MARKEXPR}" \
|
MARKEXPR="${MARKEXPR}" \
|
||||||
./scripts/s3-tests/run.sh
|
./scripts/s3-tests/run.sh
|
||||||
|
RC=$?
|
||||||
|
set -e
|
||||||
|
|
||||||
|
if [ "${TEST_SCOPE}" = "implemented" ]; then
|
||||||
|
# Whitelist run: every failure is a regression.
|
||||||
|
exit "${RC}"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Full sweep: failures outside the implemented whitelist are
|
||||||
|
# inventory (promotion candidates / unimplemented features), not a
|
||||||
|
# gate. Fail only on whitelist regressions or infrastructure errors.
|
||||||
|
JUNIT="artifacts/s3tests-${TEST_MODE}/junit.xml"
|
||||||
|
if [ ! -f "${JUNIT}" ]; then
|
||||||
|
echo "No junit.xml produced — infrastructure failure (exit ${RC})" >&2
|
||||||
|
exit "${RC}"
|
||||||
|
fi
|
||||||
|
python3 scripts/s3-tests/report_compat.py \
|
||||||
|
--junit "${JUNIT}" \
|
||||||
|
--lists-dir scripts/s3-tests \
|
||||||
|
--fail-on-regression
|
||||||
|
|
||||||
- name: Publish compatibility report
|
- name: Publish compatibility report
|
||||||
if: always()
|
if: always()
|
||||||
@@ -339,7 +346,7 @@ jobs:
|
|||||||
if: always() && env.ACT != 'true'
|
if: always() && env.ACT != 'true'
|
||||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||||
with:
|
with:
|
||||||
name: s3tests-${{ env.TEST_MODE }}-shard-${{ matrix.shard-index }}
|
name: s3tests-${{ env.TEST_MODE }}
|
||||||
path: artifacts/**
|
path: artifacts/**
|
||||||
|
|
||||||
alert-on-failure:
|
alert-on-failure:
|
||||||
|
|||||||
+24
-11
@@ -12,22 +12,27 @@
|
|||||||
# See the License for the specific language governing permissions and
|
# See the License for the specific language governing permissions and
|
||||||
# limitations under the License.
|
# limitations under the License.
|
||||||
|
|
||||||
|
# DISABLED. This workflow is switched off in the repository's Actions settings
|
||||||
|
# (state: disabled_manually) and does not run on any trigger, including its cron
|
||||||
|
# and workflow_dispatch. That state lives in GitHub's UI and is invisible when
|
||||||
|
# reading this file, which has already misled at least one audit — hence this
|
||||||
|
# banner. Re-enabling is a UI action; anyone doing so should first check that the
|
||||||
|
# workflow still matches the current CI layout. See rustfs/backlog#1603.
|
||||||
|
#
|
||||||
name: Fuzz
|
name: Fuzz
|
||||||
|
|
||||||
on:
|
on:
|
||||||
pull_request:
|
pull_request:
|
||||||
types: [ opened, synchronize, reopened, closed ]
|
types: [ opened, synchronize, reopened, closed ]
|
||||||
# Run when the harness or any directly fuzzed production crate changes.
|
# PR trigger is intentionally narrow: only changes to the fuzz harness
|
||||||
|
# itself gate a PR. Broad crate paths (ecstore/filemeta/utils/policy/…)
|
||||||
|
# are covered by the nightly `schedule` run below, which fuzzes against
|
||||||
|
# whatever landed on main. Widening these paths previously queued a
|
||||||
|
# ~45min fuzz-build on nearly every PR and is why this workflow was
|
||||||
|
# disabled; do not re-add crate paths here.
|
||||||
paths:
|
paths:
|
||||||
- "fuzz/**"
|
- "fuzz/**"
|
||||||
- "scripts/fuzz/**"
|
- "scripts/fuzz/**"
|
||||||
- "crates/ecstore/**"
|
|
||||||
- "crates/filemeta/**"
|
|
||||||
- "crates/policy/**"
|
|
||||||
- "crates/security-governance/**"
|
|
||||||
- "crates/utils/**"
|
|
||||||
- "Cargo.toml"
|
|
||||||
- "Cargo.lock"
|
|
||||||
- ".github/workflows/fuzz.yml"
|
- ".github/workflows/fuzz.yml"
|
||||||
schedule:
|
schedule:
|
||||||
- cron: "0 2 * * *"
|
- cron: "0 2 * * *"
|
||||||
@@ -76,7 +81,7 @@ jobs:
|
|||||||
github.event_name == 'schedule' ||
|
github.event_name == 'schedule' ||
|
||||||
github.event_name == 'workflow_dispatch'
|
github.event_name == 'workflow_dispatch'
|
||||||
runs-on: sm-standard-4
|
runs-on: sm-standard-4
|
||||||
timeout-minutes: 60
|
timeout-minutes: 45
|
||||||
env:
|
env:
|
||||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||||
steps:
|
steps:
|
||||||
@@ -116,7 +121,12 @@ jobs:
|
|||||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||||
with:
|
with:
|
||||||
name: fuzz-prebuilt-binaries-${{ github.run_number }}
|
name: fuzz-prebuilt-binaries-${{ github.run_number }}
|
||||||
path: fuzz/prebuilt/${{ env.CARGO_BUILD_TARGET }}/release/
|
path: |
|
||||||
|
fuzz/prebuilt/${{ env.CARGO_BUILD_TARGET }}/release/archive_extract
|
||||||
|
fuzz/prebuilt/${{ env.CARGO_BUILD_TARGET }}/release/bucket_validation
|
||||||
|
fuzz/prebuilt/${{ env.CARGO_BUILD_TARGET }}/release/local_metadata
|
||||||
|
fuzz/prebuilt/${{ env.CARGO_BUILD_TARGET }}/release/path_containment
|
||||||
|
fuzz/prebuilt/${{ env.CARGO_BUILD_TARGET }}/release/policy_ingress
|
||||||
if-no-files-found: error
|
if-no-files-found: error
|
||||||
retention-days: 1
|
retention-days: 1
|
||||||
compression-level: 0
|
compression-level: 0
|
||||||
@@ -182,7 +192,10 @@ jobs:
|
|||||||
nightly-fuzz-corpus:
|
nightly-fuzz-corpus:
|
||||||
name: "Nightly / ${{ matrix.target }}"
|
name: "Nightly / ${{ matrix.target }}"
|
||||||
needs: fuzz-build
|
needs: fuzz-build
|
||||||
# Scheduled failures are handled by alert-on-failure below.
|
# TODO(ci-8): when the schedule-failure-issue composite action lands,
|
||||||
|
# add a step here (or a dependent job) that opens/updates a GitHub issue
|
||||||
|
# on nightly failure. ci-8 is the single alerting mechanism for all
|
||||||
|
# scheduled workflows; do not self-roll alerting in this workflow.
|
||||||
if: >
|
if: >
|
||||||
github.event_name == 'schedule' ||
|
github.event_name == 'schedule' ||
|
||||||
(github.event_name == 'workflow_dispatch' &&
|
(github.event_name == 'workflow_dispatch' &&
|
||||||
|
|||||||
@@ -55,142 +55,3 @@ jobs:
|
|||||||
|
|
||||||
- name: Build RustFS
|
- name: Build RustFS
|
||||||
run: cargo build --release --locked --target x86_64-unknown-linux-gnu -p rustfs --bins
|
run: cargo build --release --locked --target x86_64-unknown-linux-gnu -p rustfs --bins
|
||||||
|
|
||||||
# Live-Vault lane for the rustfs-kms suite (rustfs/backlog#1774).
|
|
||||||
#
|
|
||||||
# RUSTFS_KMS_VAULT_TOKEN is the single switch that adds the Vault KV2 and
|
|
||||||
# Vault Transit backends to every for_each_backend spec in
|
|
||||||
# crates/kms/tests/behavior_*.rs (see crates/kms/AGENTS.md). rotate and
|
|
||||||
# versioning are advertised only by the Vault backends, so without this lane
|
|
||||||
# no CI run ever asserts the working half of behavior_rotation.rs — a
|
|
||||||
# rotation that silently dropped historical key versions would stay green.
|
|
||||||
# The same lane runs the dev-Vault #[ignore] tests and the two self-hosting
|
|
||||||
# live scripts (AppRole login, three-node Raft leader failover).
|
|
||||||
#
|
|
||||||
# GitHub-hosted ubuntu-latest, deliberately not the self-hosted sm-standard
|
|
||||||
# fleet: the HA failover script needs a working Docker daemon, and the
|
|
||||||
# self-hosted fleet is heterogeneous — a docker-dependent workflow has been
|
|
||||||
# burned by it before (see the banner in e2e-s3tests.yml, rustfs/backlog#1149).
|
|
||||||
kms-vault-lane:
|
|
||||||
name: KMS live Vault lane
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
timeout-minutes: 90
|
|
||||||
env:
|
|
||||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
|
||||||
# Root token of the ephemeral loopback dev server. Not a secret: the
|
|
||||||
# server lives only for this job, listens on 127.0.0.1, and holds only
|
|
||||||
# keys the tests create. The literal value matters — the dev-Vault
|
|
||||||
# #[ignore] fixtures in crates/kms/src/backends/vault.rs hardcode it.
|
|
||||||
VAULT_LANE_TOKEN: dev-only-token
|
|
||||||
VAULT_LANE_ADDR: http://127.0.0.1:8200
|
|
||||||
# Keeps a runner-level proxy from swallowing the loopback dev-server
|
|
||||||
# traffic (see crates/kms/AGENTS.md). Actions env keys are
|
|
||||||
# case-insensitive, so only the uppercase form is set; reqwest reads
|
|
||||||
# either casing.
|
|
||||||
NO_PROXY: 127.0.0.1,localhost
|
|
||||||
steps:
|
|
||||||
- name: Checkout main branch
|
|
||||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
|
||||||
with:
|
|
||||||
persist-credentials: false
|
|
||||||
ref: main
|
|
||||||
|
|
||||||
- name: Setup Rust environment
|
|
||||||
uses: ./.github/actions/setup
|
|
||||||
with:
|
|
||||||
# Dedicated key: rust-cache cannot tell runner images apart, so
|
|
||||||
# sharing a key with an sm-standard lane would let two different
|
|
||||||
# system images overwrite each other's artifacts (same reasoning as
|
|
||||||
# ci.yml's ci-uring lane). Saved from this nightly job itself so the
|
|
||||||
# next night starts warm.
|
|
||||||
cache-shared-key: kms-vault-lane
|
|
||||||
cache-save-if: 'true'
|
|
||||||
install-build-packaging-tools: 'false'
|
|
||||||
install-test-tools: 'false'
|
|
||||||
|
|
||||||
- name: Install Vault CLI
|
|
||||||
run: |
|
|
||||||
set -euo pipefail
|
|
||||||
wget -qO- https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg
|
|
||||||
echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list >/dev/null
|
|
||||||
sudo apt-get update -qq
|
|
||||||
sudo apt-get install -y -qq vault
|
|
||||||
vault version
|
|
||||||
|
|
||||||
- name: Start Vault dev server with KV2 and Transit engines
|
|
||||||
run: |
|
|
||||||
set -euo pipefail
|
|
||||||
nohup vault server -dev \
|
|
||||||
-dev-root-token-id="${VAULT_LANE_TOKEN}" \
|
|
||||||
-dev-listen-address=127.0.0.1:8200 >/tmp/vault-dev.log 2>&1 &
|
|
||||||
for _ in $(seq 1 60); do
|
|
||||||
if curl -fsS "${VAULT_LANE_ADDR}/v1/sys/health" >/dev/null 2>&1; then
|
|
||||||
break
|
|
||||||
fi
|
|
||||||
sleep 1
|
|
||||||
done
|
|
||||||
curl -fsS "${VAULT_LANE_ADDR}/v1/sys/health"
|
|
||||||
export VAULT_ADDR="${VAULT_LANE_ADDR}" VAULT_TOKEN="${VAULT_LANE_TOKEN}"
|
|
||||||
# Dev mode mounts KV v2 at secret/ by default; Transit is explicit.
|
|
||||||
# Prove both engines actually work rather than assuming the defaults.
|
|
||||||
vault secrets enable transit
|
|
||||||
vault kv put secret/rustfs-ci-lane-probe value=ok >/dev/null
|
|
||||||
vault kv get secret/rustfs-ci-lane-probe >/dev/null
|
|
||||||
vault write -f transit/keys/rustfs-ci-lane-probe >/dev/null
|
|
||||||
|
|
||||||
- name: Run rustfs-kms suite with the Vault lane on
|
|
||||||
env:
|
|
||||||
RUSTFS_KMS_VAULT_TOKEN: ${{ env.VAULT_LANE_TOKEN }}
|
|
||||||
RUSTFS_KMS_VAULT_ADDR: ${{ env.VAULT_LANE_ADDR }}
|
|
||||||
run: cargo test -p rustfs-kms --locked
|
|
||||||
|
|
||||||
- name: Run dev-Vault ignored tests
|
|
||||||
env:
|
|
||||||
RUSTFS_KMS_VAULT_TOKEN: ${{ env.VAULT_LANE_TOKEN }}
|
|
||||||
RUSTFS_KMS_VAULT_ADDR: ${{ env.VAULT_LANE_ADDR }}
|
|
||||||
# Filters select the dev-Vault-only #[ignore] tests. The AWS #[ignore]
|
|
||||||
# tests (backends::aws, service_manager) stay excluded — they need real
|
|
||||||
# AWS credentials and create billable keys. The AppRole and HA #[ignore]
|
|
||||||
# tests are excluded here because their own scripts below provision the
|
|
||||||
# Vault topology they need.
|
|
||||||
run: |
|
|
||||||
set -euo pipefail
|
|
||||||
cargo test -p rustfs-kms --locked --lib backends::contract_tests -- --ignored
|
|
||||||
cargo test -p rustfs-kms --locked --lib backends::vault -- --ignored
|
|
||||||
cargo test -p rustfs-kms --locked --test vault_fault_injection -- --ignored
|
|
||||||
|
|
||||||
- name: Run AppRole live checks (self-hosting ephemeral Vault)
|
|
||||||
run: bash scripts/test/vault_approle_kms_live.sh
|
|
||||||
|
|
||||||
- name: Show Vault dev server log on failure
|
|
||||||
if: failure()
|
|
||||||
run: tail -n 200 /tmp/vault-dev.log || true
|
|
||||||
|
|
||||||
# Three-node Raft leader failover (crates/kms/tests/vault_ha_failover_live.rs,
|
|
||||||
# first validated by rustfs/rustfs#5653). Its own job so an election-timing
|
|
||||||
# flake cannot mask the main lane's verdict, and vice versa. The script
|
|
||||||
# provisions and tears down its own Docker cluster.
|
|
||||||
kms-vault-ha-failover:
|
|
||||||
name: KMS Vault HA failover lane
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
timeout-minutes: 60
|
|
||||||
env:
|
|
||||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
|
||||||
NO_PROXY: 127.0.0.1,localhost
|
|
||||||
steps:
|
|
||||||
- name: Checkout main branch
|
|
||||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
|
||||||
with:
|
|
||||||
persist-credentials: false
|
|
||||||
ref: main
|
|
||||||
|
|
||||||
- name: Setup Rust environment
|
|
||||||
uses: ./.github/actions/setup
|
|
||||||
with:
|
|
||||||
cache-shared-key: kms-vault-lane
|
|
||||||
cache-save-if: 'false'
|
|
||||||
install-build-packaging-tools: 'false'
|
|
||||||
install-test-tools: 'false'
|
|
||||||
|
|
||||||
- name: Run HA leader failover live checks (three-node Raft cluster in Docker)
|
|
||||||
run: bash scripts/test/vault_ha_kms_live.sh
|
|
||||||
|
|||||||
@@ -15,35 +15,28 @@
|
|||||||
# Package Workflow - Build DEB/RPM packages
|
# Package Workflow - Build DEB/RPM packages
|
||||||
#
|
#
|
||||||
# This workflow builds DEB and RPM packages from pre-built Linux binaries
|
# This workflow builds DEB and RPM packages from pre-built Linux binaries
|
||||||
# and uploads them to Cloudflare R2 and the GitHub release.
|
# and uploads them to Cloudflare R2.
|
||||||
#
|
#
|
||||||
# Trigger:
|
# Trigger:
|
||||||
# - workflow_run: automatically package after "Build and Release" completes
|
# - release published: automatically package when a GitHub release is published
|
||||||
# for a release tag (the mac/windows/linux binaries are already uploaded
|
# - workflow_dispatch: manual trigger with optional tag/run_id
|
||||||
# to the GitHub release before packaging starts)
|
|
||||||
# - workflow_dispatch: manual fallback (backfill / re-run) with optional tag/run_id
|
|
||||||
#
|
#
|
||||||
# Flow:
|
# Flow:
|
||||||
# 1. Resolve the triggering Build workflow run for the release tag
|
# 1. Find the Build workflow run for the release tag
|
||||||
# 2. Download Linux binaries (x86_64-gnu, aarch64-gnu) from build artifacts
|
# 2. Download Linux binaries (x86_64-gnu, aarch64-gnu) from build artifacts
|
||||||
# 3. Build DEB packages for amd64 and arm64
|
# 3. Build DEB packages for amd64 and arm64
|
||||||
# 4. Build RPM packages for x86_64 and aarch64
|
# 4. Build RPM packages for x86_64 and aarch64
|
||||||
# 5. Upload all packages to Cloudflare R2 and the GitHub release
|
# 5. Upload all packages to Cloudflare R2
|
||||||
|
|
||||||
name: Package DEB/RPM
|
name: Package DEB/RPM
|
||||||
|
|
||||||
permissions:
|
permissions:
|
||||||
# contents: write is required to upload packages to the GitHub release
|
contents: read
|
||||||
contents: write
|
|
||||||
actions: read
|
actions: read
|
||||||
|
|
||||||
on:
|
on:
|
||||||
# Follows the same pattern as docker.yml: run after the release build
|
release:
|
||||||
# workflow completes, so packaging is triggered only by release tags
|
types: [ published ]
|
||||||
# (e.g. 1.0.0-rc.2, 1.0.0-rc.3), never by development builds.
|
|
||||||
workflow_run:
|
|
||||||
workflows: [ "Build and Release" ]
|
|
||||||
types: [ completed ]
|
|
||||||
workflow_dispatch:
|
workflow_dispatch:
|
||||||
inputs:
|
inputs:
|
||||||
tag:
|
tag:
|
||||||
@@ -56,26 +49,13 @@ on:
|
|||||||
type: string
|
type: string
|
||||||
|
|
||||||
concurrency:
|
concurrency:
|
||||||
group: ${{ github.workflow }}-${{ github.event.workflow_run.head_branch || github.event.inputs.tag || github.run_id }}
|
group: ${{ github.workflow }}-${{ github.event.release.tag_name || github.event.inputs.tag || github.run_id }}
|
||||||
cancel-in-progress: true
|
cancel-in-progress: true
|
||||||
|
|
||||||
env:
|
|
||||||
HEAD_BRANCH: ${{ github.event.workflow_run.head_branch }}
|
|
||||||
WORKFLOW_RUN_ID: ${{ github.event.workflow_run.id }}
|
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
# Resolve which build run to use and extract version info
|
# Resolve which build run to use and extract version info
|
||||||
resolve:
|
resolve:
|
||||||
name: Resolve Build
|
name: Resolve Build
|
||||||
# Auto-trigger only from successful tag builds of "Build and Release".
|
|
||||||
# Tag pushes arrive as event == push with head_branch != main (a
|
|
||||||
# non-main push head_branch is the release tag name). Manual dispatch
|
|
||||||
# stays available as a fallback for backfills and re-runs.
|
|
||||||
if: >-
|
|
||||||
github.event_name == 'workflow_dispatch' ||
|
|
||||||
(github.event.workflow_run.conclusion == 'success' &&
|
|
||||||
github.event.workflow_run.event == 'push' &&
|
|
||||||
github.event.workflow_run.head_branch != 'main')
|
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
timeout-minutes: 10
|
timeout-minutes: 10
|
||||||
outputs:
|
outputs:
|
||||||
@@ -95,8 +75,8 @@ jobs:
|
|||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
|
|
||||||
# Determine tag
|
# Determine tag
|
||||||
if [[ "${{ github.event_name }}" == "workflow_run" ]]; then
|
if [[ "${{ github.event_name }}" == "release" ]]; then
|
||||||
TAG="${HEAD_BRANCH}"
|
TAG="${{ github.event.release.tag_name }}"
|
||||||
elif [[ -n "$INPUT_TAG" ]]; then
|
elif [[ -n "$INPUT_TAG" ]]; then
|
||||||
TAG="$INPUT_TAG"
|
TAG="$INPUT_TAG"
|
||||||
else
|
else
|
||||||
@@ -113,11 +93,6 @@ jobs:
|
|||||||
BUILD_RUN_ID="$INPUT_RUN_ID"
|
BUILD_RUN_ID="$INPUT_RUN_ID"
|
||||||
echo "Using explicit build run ID: $BUILD_RUN_ID"
|
echo "Using explicit build run ID: $BUILD_RUN_ID"
|
||||||
|
|
||||||
elif [[ "${{ github.event_name }}" == "workflow_run" ]]; then
|
|
||||||
# Use the Build and Release run that triggered this workflow
|
|
||||||
BUILD_RUN_ID="${WORKFLOW_RUN_ID}"
|
|
||||||
echo "Using triggering workflow run: $BUILD_RUN_ID"
|
|
||||||
|
|
||||||
elif [[ -n "$TAG" ]]; then
|
elif [[ -n "$TAG" ]]; then
|
||||||
# Find the build run that produced this tag
|
# Find the build run that produced this tag
|
||||||
echo "Looking for build run for tag: $TAG"
|
echo "Looking for build run for tag: $TAG"
|
||||||
@@ -189,7 +164,6 @@ jobs:
|
|||||||
timeout-minutes: 30
|
timeout-minutes: 30
|
||||||
strategy:
|
strategy:
|
||||||
fail-fast: false
|
fail-fast: false
|
||||||
max-parallel: 1
|
|
||||||
matrix:
|
matrix:
|
||||||
include:
|
include:
|
||||||
- arch: x86_64
|
- arch: x86_64
|
||||||
@@ -291,12 +265,6 @@ jobs:
|
|||||||
Homepage: https://rustfs.com
|
Homepage: https://rustfs.com
|
||||||
EOF
|
EOF
|
||||||
|
|
||||||
# Declare /etc/default/rustfs as a conffile so dpkg preserves user
|
|
||||||
# modifications on upgrade instead of silently overwriting them.
|
|
||||||
cat > "${PKG_DIR}/DEBIAN/conffiles" << 'CONFFILES'
|
|
||||||
/etc/default/rustfs
|
|
||||||
CONFFILES
|
|
||||||
|
|
||||||
cat > "${PKG_DIR}/DEBIAN/postinst" << 'POSTINST'
|
cat > "${PKG_DIR}/DEBIAN/postinst" << 'POSTINST'
|
||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
set -e
|
set -e
|
||||||
@@ -488,59 +456,6 @@ jobs:
|
|||||||
echo "✅ Latest packages updated"
|
echo "✅ Latest packages updated"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
- name: Upload packages to GitHub Release
|
|
||||||
if: needs.resolve.outputs.tag != ''
|
|
||||||
env:
|
|
||||||
GH_TOKEN: ${{ github.token }}
|
|
||||||
shell: bash
|
|
||||||
run: |
|
|
||||||
set -euo pipefail
|
|
||||||
|
|
||||||
TAG="${{ needs.resolve.outputs.tag }}"
|
|
||||||
DEB_FILE="${{ steps.deb.outputs.deb_file }}"
|
|
||||||
RPM_FILE="${{ steps.rpm.outputs.rpm_file }}"
|
|
||||||
|
|
||||||
# Upload the packages, then refresh the release checksums so the new
|
|
||||||
# assets are covered, matching the binary release flow.
|
|
||||||
for f in "$DEB_FILE" "$RPM_FILE"; do
|
|
||||||
if [[ -n "$f" && -f "$f" ]]; then
|
|
||||||
echo "📤 Uploading $(basename "$f") to GitHub release ${TAG}..."
|
|
||||||
gh release upload "$TAG" "$f" --clobber
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
|
|
||||||
CHECKSUM_DIR="$(mktemp -d)"
|
|
||||||
gh release download "$TAG" -p 'SHA256SUMS' -p 'SHA512SUMS' \
|
|
||||||
-D "$CHECKSUM_DIR" --clobber
|
|
||||||
|
|
||||||
for spec in "SHA256SUMS:sha256sum" "SHA512SUMS:sha512sum"; do
|
|
||||||
asset="${spec%%:*}"
|
|
||||||
checksum_cmd="${spec##*:}"
|
|
||||||
checksum_file="${CHECKSUM_DIR}/${asset}"
|
|
||||||
|
|
||||||
for f in "$DEB_FILE" "$RPM_FILE"; do
|
|
||||||
if [[ -n "$f" && -f "$f" ]]; then
|
|
||||||
base="$(basename "$f")"
|
|
||||||
# GitHub stores release asset names with '~' normalized to '.'
|
|
||||||
# (e.g. rustfs_1.0.0~rc.2_amd64.deb is stored as
|
|
||||||
# rustfs_1.0.0.rc.2_amd64.deb), so checksum entries must
|
|
||||||
# reference the name as stored on the release.
|
|
||||||
github_base="${base//\~/.}"
|
|
||||||
# Remove any stale entry (both naming variants), then append
|
|
||||||
grep -Fv -- "$base" "$checksum_file" > "${checksum_file}.tmp" || true
|
|
||||||
grep -Fv -- "$github_base" "${checksum_file}.tmp" > "${checksum_file}.tmp2" || true
|
|
||||||
mv "${checksum_file}.tmp2" "$checksum_file"
|
|
||||||
digest=$("$checksum_cmd" -- "$f" | awk '{print $1}')
|
|
||||||
printf '%s %s\n' "$digest" "$github_base" >> "$checksum_file"
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
|
|
||||||
echo "📤 Updating ${asset} for release ${TAG}..."
|
|
||||||
gh release upload "$TAG" "$checksum_file" --clobber
|
|
||||||
done
|
|
||||||
|
|
||||||
echo "✅ GitHub release assets updated"
|
|
||||||
|
|
||||||
# Summary
|
# Summary
|
||||||
summary:
|
summary:
|
||||||
name: Summary
|
name: Summary
|
||||||
|
|||||||
@@ -127,9 +127,8 @@ the broadest gate. Inspect only the final task-owned diff, classify it by
|
|||||||
behavioral impact rather than line count or path alone, and run the smallest
|
behavioral impact rather than line count or path alone, and run the smallest
|
||||||
set of checks that provides meaningful coverage. Do not let unrelated
|
set of checks that provides meaningful coverage. Do not let unrelated
|
||||||
worktree changes or a generic contributor checklist expand the scope.
|
worktree changes or a generic contributor checklist expand the scope.
|
||||||
For non-exempt changes, complete the applicable multi-role adversarial review
|
Non-exempt changes must also pass Adversarial Validation (next section) before
|
||||||
before running `make pre-pr` (or an equivalent full gate). Resolve or rebut
|
the checks below count as completion.
|
||||||
every finding first, then run the gate against the reviewed final diff.
|
|
||||||
|
|
||||||
### Validation floor
|
### Validation floor
|
||||||
|
|
||||||
@@ -167,9 +166,8 @@ every finding first, then run the gate against the reviewed final diff.
|
|||||||
dependency set is identifiable, validate those packages and known
|
dependency set is identifiable, validate those packages and known
|
||||||
dependents instead of the whole workspace. Use `make pre-commit` only when
|
dependents instead of the whole workspace. Use `make pre-commit` only when
|
||||||
a repository-wide fast gate adds useful confidence beyond those checks.
|
a repository-wide fast gate adds useful confidence beyond those checks.
|
||||||
4. **Broad or high-risk change:** After the applicable adversarial review has
|
4. **Broad or high-risk change:** Run `make pre-pr` only when targeted coverage
|
||||||
completed, run `make pre-pr` only when targeted coverage cannot bound the
|
cannot bound the impact, including:
|
||||||
impact, including:
|
|
||||||
- dependency, feature, build-script, procedural-macro, code-generation,
|
- dependency, feature, build-script, procedural-macro, code-generation,
|
||||||
toolchain, or CI changes that alter compilation or the test matrix;
|
toolchain, or CI changes that alter compilation or the test matrix;
|
||||||
- cross-crate public APIs, shared foundational code, or broad refactors with
|
- cross-crate public APIs, shared foundational code, or broad refactors with
|
||||||
@@ -289,9 +287,8 @@ High risk: all seven roles.
|
|||||||
- Every testable behavior change has a focused regression check. Exceptions
|
- Every testable behavior change has a focused regression check. Exceptions
|
||||||
follow the validation floor and state why a check is impractical and what
|
follow the validation floor and state why a check is impractical and what
|
||||||
risk remains.
|
risk remains.
|
||||||
- After the applicable adversarial review has completed, the Verification
|
- The Verification Before PR gates pass — adversarial review supplements
|
||||||
Before PR gates pass; adversarial review supplements those gates, never
|
those gates, never replaces them.
|
||||||
replaces them.
|
|
||||||
- High risk only: record a one-line verdict per role in the PR description.
|
- High risk only: record a one-line verdict per role in the PR description.
|
||||||
|
|
||||||
## Git and PR Baseline
|
## Git and PR Baseline
|
||||||
|
|||||||
+24
-71
@@ -1,6 +1,6 @@
|
|||||||
# ARCHITECTURE.md
|
# ARCHITECTURE.md
|
||||||
|
|
||||||
> Last updated: 2026-08-12 · Revision: 3
|
> Last updated: 2026-07-02 · Revision: 2
|
||||||
>
|
>
|
||||||
> This document describes the high-level architecture of RustFS.
|
> This document describes the high-level architecture of RustFS.
|
||||||
> If you want to familiarize yourself with the code base, you are in the right place!
|
> If you want to familiarize yourself with the code base, you are in the right place!
|
||||||
@@ -31,7 +31,7 @@ HTTP request
|
|||||||
→ storage/ecfs (erasure coding, encryption, checksums)
|
→ storage/ecfs (erasure coding, encryption, checksums)
|
||||||
→ ecstore (disk pool selection, data distribution)
|
→ ecstore (disk pool selection, data distribution)
|
||||||
→ rio (reader pipeline: encrypt → compress → hash → write)
|
→ rio (reader pipeline: encrypt → compress → hash → write)
|
||||||
→ io-core (buffer pool, storage profiling, admission control)
|
→ io-core (zero-copy I/O, buffer pool, direct I/O)
|
||||||
→ local disk / remote disk via RPC
|
→ local disk / remote disk via RPC
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -55,7 +55,7 @@ rustfs/ # Workspace root (virtual manifest)
|
|||||||
├── crates/ # library crates (authoritative list: Cargo.toml [workspace].members)
|
├── crates/ # library crates (authoritative list: Cargo.toml [workspace].members)
|
||||||
│ ├── ecstore/ # Erasure-coded storage engine
|
│ ├── ecstore/ # Erasure-coded storage engine
|
||||||
│ ├── rio/ # Reader I/O pipeline (encrypt, compress, hash)
|
│ ├── rio/ # Reader I/O pipeline (encrypt, compress, hash)
|
||||||
│ ├── io-core/ # Buffer pool, storage profiling, backpressure/deadlock policy, lock optimizer, operation progress
|
│ ├── io-core/ # Zero-copy I/O, scheduling, buffer pool
|
||||||
│ ├── io-metrics/ # I/O metrics collection
|
│ ├── io-metrics/ # I/O metrics collection
|
||||||
│ ├── common/ # Shared runtime state, globals, data usage types
|
│ ├── common/ # Shared runtime state, globals, data usage types
|
||||||
│ ├── config/ # Configuration types and parsing
|
│ ├── config/ # Configuration types and parsing
|
||||||
@@ -101,10 +101,7 @@ refactors.
|
|||||||
|
|
||||||
The `rustfs` binary crate composes these libraries into the running server.
|
The `rustfs` binary crate composes these libraries into the running server.
|
||||||
`ecstore` remains the storage engine at the architectural center; its internal
|
`ecstore` remains the storage engine at the architectural center; its internal
|
||||||
module split is tracked under `docs/architecture/`. `rio-v2` is the
|
module split is tracked under `docs/architecture/`.
|
||||||
feature-gated MinIO on-disk format compatibility I/O layer; it ships in no
|
|
||||||
default build (lifecycle:
|
|
||||||
[docs/architecture/minio-file-format-compat.md](docs/architecture/minio-file-format-compat.md)).
|
|
||||||
|
|
||||||
## Architecture Invariants
|
## Architecture Invariants
|
||||||
|
|
||||||
@@ -122,44 +119,19 @@ default build (lifecycle:
|
|||||||
|
|
||||||
3. **Each type has exactly one definition.** Types shared across crates must be defined
|
3. **Each type has exactly one definition.** Types shared across crates must be defined
|
||||||
in one crate and re-exported or imported by others.
|
in one crate and re-exported or imported by others.
|
||||||
- ⚠️ VIOLATED: `ReplicationStats` names three unrelated types
|
- ⚠️ VIOLATED: `ReplicationStats` (4 copies), `LastMinuteLatency` (3 copies),
|
||||||
(`crates/data-usage/src/data_usage.rs`,
|
`BackpressureConfig` (3 copies), `DataUsageInfo` (2 copies).
|
||||||
`crates/obs/src/metrics/collectors/replication.rs`,
|
|
||||||
`crates/ecstore/src/bucket/replication/replication_state.rs`) — a naming
|
|
||||||
collision, not copies; renaming is tracked in rustfs/backlog#1847.
|
|
||||||
- `LastMinuteLatency` has two deliberately different implementations: the
|
|
||||||
per-second bucketed accumulator in `crates/common/src/last_minute.rs` and
|
|
||||||
the in-memory endpoint-health sample tracker in
|
|
||||||
`crates/ecstore/src/bucket/bucket_target_sys.rs` (its doc comment explains
|
|
||||||
why it stays local).
|
|
||||||
- ✅ RESOLVED: `BackpressureConfig` and `DataUsageInfo` each have exactly one
|
|
||||||
definition (`crates/io-core/src/backpressure.rs`,
|
|
||||||
`crates/data-usage/src/data_usage.rs`). The zero-consumer
|
|
||||||
`BackpressureSettings` copy that lingered in io-metrics was removed
|
|
||||||
(rustfs/backlog#1833).
|
|
||||||
|
|
||||||
4. **ecstore does not know about HTTP or S3 protocol details.** It operates on
|
4. **ecstore does not know about HTTP or S3 protocol details.** It operates on
|
||||||
storage-level abstractions (objects, buckets, disks, pools).
|
storage-level abstractions (objects, buckets, disks, pools).
|
||||||
- ⚠️ VIOLATED: 58 files under `crates/ecstore/src` reference `s3s`
|
|
||||||
(`rg -l 's3s' crates/ecstore/src | wc -l`), `crates/ecstore/src/client/`
|
|
||||||
is a ~9.4K-line embedded S3 HTTP client, and `crates/ecstore/Cargo.toml`
|
|
||||||
depends on `s3s`, `http`, `hyper`/`hyper-util`/`hyper-rustls`, and
|
|
||||||
`reqwest`. Target state: the engine's need to act as an S3 client
|
|
||||||
(tiering, replication targets) is served by an extracted client crate,
|
|
||||||
and ecstore holds no wire or DTO types.
|
|
||||||
|
|
||||||
5. **The `rustfs` binary crate is the only place that wires everything together.**
|
5. **The `rustfs` binary crate is the only place that wires everything together.**
|
||||||
Individual crates should be testable in isolation.
|
Individual crates should be testable in isolation.
|
||||||
|
|
||||||
6. **Error types use `thiserror` with descriptive names** (e.g., `StorageError`,
|
6. **Error types use `thiserror` with descriptive names** (e.g., `StorageError`,
|
||||||
not bare `Error`).
|
not bare `Error`).
|
||||||
- ✅ RESOLVED (strategy): `snafu` is gone from source
|
- ⚠️ VIOLATED: 6 crates use `pub enum Error`; 2 crates use `snafu`;
|
||||||
(`rg -l snafu crates/ rustfs/` is empty) and library code no longer uses
|
`heal` use `anyhow` in library code.
|
||||||
`anyhow` (remaining hits are test code and the `e2e_test` crate; `heal`
|
|
||||||
uses `thiserror`).
|
|
||||||
- ⚠️ VIOLATED (naming): 6 crates still export a bare `pub enum Error`:
|
|
||||||
`crypto`, `filemeta`, `heal`, `iam`, `policy`, and `replication`
|
|
||||||
(`src/resync.rs`) — all `thiserror`-derived.
|
|
||||||
|
|
||||||
## Known Structural Issues
|
## Known Structural Issues
|
||||||
|
|
||||||
@@ -168,25 +140,13 @@ default build (lifecycle:
|
|||||||
|
|
||||||
### Critical
|
### Critical
|
||||||
|
|
||||||
- **scanner/data-usage duplicate `.usage-cache.bin` serialization types.** The
|
- **common/scanner code duplication (~3K lines).** `scanner` depends on `common`
|
||||||
original finding ("common/scanner code duplication, ~3K lines") is resolved:
|
but maintains its own copies of `DataUsageInfo`, `LastMinuteLatency`, and related
|
||||||
`scanner` imports the shared data-usage types from `rustfs-data-usage` (see
|
types instead of importing them.
|
||||||
the `pub use rustfs_data_usage::…` re-exports at the top of
|
|
||||||
`crates/scanner/src/data_usage_define.rs`). What remains: `scanner` and
|
|
||||||
`data-usage` each hold their own serialization types for the scanner cache
|
|
||||||
file (`DataUsageCacheInfo`/`DataUsageEntryInfo` in
|
|
||||||
`crates/scanner/src/data_usage_define.rs` vs
|
|
||||||
`DataUsageCacheInfo`/`DataUsageEntry` in
|
|
||||||
`crates/data-usage/src/data_usage.rs`); convergence is tracked in
|
|
||||||
rustfs/backlog#1828.
|
|
||||||
|
|
||||||
- **ecstore is a monolith (265 files, ~288K lines — roughly half is inline
|
- **ecstore is a monolith (87K lines, 163 files).** It contains disk management,
|
||||||
`#[cfg(test)]` code).** Measured with
|
bucket management, erasure coding, replication, lifecycle, RPC, and configuration
|
||||||
`find crates/ecstore/src -name '*.rs' | xargs wc -l`. It contains disk
|
— all in one crate. It should be decomposed along its existing subdirectories.
|
||||||
management, bucket management, erasure coding, replication, lifecycle, RPC,
|
|
||||||
and configuration — all in one crate. It should be decomposed along its
|
|
||||||
existing subdirectories; the split plan lives in
|
|
||||||
[docs/architecture/ecstore-module-split-plan.md](docs/architecture/ecstore-module-split-plan.md).
|
|
||||||
|
|
||||||
### High
|
### High
|
||||||
|
|
||||||
@@ -194,26 +154,19 @@ default build (lifecycle:
|
|||||||
`common → filemeta/madmin` edges must stay removed so leaf/helper crates do
|
`common → filemeta/madmin` edges must stay removed so leaf/helper crates do
|
||||||
not regain upward dependencies.
|
not regain upward dependencies.
|
||||||
|
|
||||||
- **Three-layer backpressure/deadlock policy bridging** across io-core,
|
- **Three-layer BackpressureConfig/DeadlockConfig duplication** across io-core,
|
||||||
concurrency, and `rustfs/src/storage`. The config types are no longer
|
concurrency, and `rustfs/src/storage`. Storage policies now expose and consume
|
||||||
duplicated (`BackpressureConfig` and `DeadlockDetectorConfig` are each
|
explicit projections into the concurrency/io-core policy shapes, and workload
|
||||||
defined once, in io-core). Storage policies expose and consume explicit
|
|
||||||
projections into the concurrency/io-core policy shapes, and workload
|
|
||||||
admission snapshots are composed through provider registries; later work
|
admission snapshots are composed through provider registries; later work
|
||||||
should use those bridges before deleting compatibility wrappers.
|
should use those bridges before deleting compatibility wrappers.
|
||||||
|
|
||||||
### Medium
|
### Medium
|
||||||
|
|
||||||
- **Bare `Error` naming.** Error-handling strategy has converged on `thiserror`
|
- **Inconsistent error handling.** Three strategies (thiserror/snafu/anyhow) and
|
||||||
(no `snafu`, no `anyhow` in library code); the remaining inconsistency is the
|
mixed naming (bare `Error` vs descriptive names).
|
||||||
bare `pub enum Error` naming in the 6 crates listed under Invariant 6.
|
|
||||||
|
|
||||||
- **`common` is mostly parked domain code, not shared utilities.** Of its
|
- **Ambiguous common vs utils boundary.** Both described as "utilities and data
|
||||||
6,724 lines, ~83% is scanner/heal domain code stranded there to break
|
structures." Need clear ownership rules.
|
||||||
dependency cycles (`metrics.rs`, ~4,810 lines of scanner-domain metrics;
|
|
||||||
`heal_channel.rs`, ~776 lines of heal-domain channel types). The
|
|
||||||
"common vs utils" naming ambiguity is secondary to moving that code to its
|
|
||||||
domain owners.
|
|
||||||
|
|
||||||
## Cross-Cutting Concerns
|
## Cross-Cutting Concerns
|
||||||
|
|
||||||
@@ -279,7 +232,7 @@ The binary (`main.rs`) boots in this order:
|
|||||||
|
|
||||||
```
|
```
|
||||||
┌─────────┐
|
┌─────────┐
|
||||||
│ rustfs │ (binary + lib)
|
│ rustfs │ (binary + lib, 75K lines)
|
||||||
│ main │
|
│ main │
|
||||||
└────┬────┘
|
└────┬────┘
|
||||||
│
|
│
|
||||||
@@ -302,7 +255,7 @@ The binary (`main.rs`) boots in this order:
|
|||||||
│ │ │
|
│ │ │
|
||||||
┌─────▼──────┐ ┌──────▼──────┐ ┌──────▼──────┐
|
┌─────▼──────┐ ┌──────▼──────┐ ┌──────▼──────┐
|
||||||
│ ecstore │ │ rio │ │ io-core │
|
│ ecstore │ │ rio │ │ io-core │
|
||||||
│ (core) │ │ (readers) │ │ (buffers) │
|
│ (87K,core) │ │ (readers) │ │ (zero-copy) │
|
||||||
└─────┬──────┘ └─────────────┘ └─────────────┘
|
└─────┬──────┘ └─────────────┘ └─────────────┘
|
||||||
│
|
│
|
||||||
┌─────┬──┼──┬─────┬──────┐
|
┌─────┬──┼──┬─────┬──────┐
|
||||||
@@ -314,7 +267,7 @@ The binary (`main.rs`) boots in this order:
|
|||||||
|
|
||||||
- **"Where does S3 PutObject go?"**
|
- **"Where does S3 PutObject go?"**
|
||||||
`server/` routes → `app/object_usecase` validates → `storage/ecfs` encodes →
|
`server/` routes → `app/object_usecase` validates → `storage/ecfs` encodes →
|
||||||
`ecstore` distributes → `rio` encrypts/compresses → `io-core` supplies buffers
|
`ecstore` distributes → `rio` encrypts/compresses → `io-core` writes
|
||||||
|
|
||||||
- **"Where are bucket policies enforced?"**
|
- **"Where are bucket policies enforced?"**
|
||||||
`app/bucket_usecase` calls into `crates/policy/`
|
`app/bucket_usecase` calls into `crates/policy/`
|
||||||
|
|||||||
+4
-6
@@ -91,9 +91,8 @@ A green `make pre-commit` is not enough to open a pull request.
|
|||||||
`make pre-pr` is the **full** gate: it runs all of the guard checks above,
|
`make pre-pr` is the **full** gate: it runs all of the guard checks above,
|
||||||
then `clippy-check` (`cargo clippy --all-targets --all-features -- -D warnings`)
|
then `clippy-check` (`cargo clippy --all-targets --all-features -- -D warnings`)
|
||||||
and `test` (shell script tests, workspace tests excluding `e2e_test`, and doc
|
and `test` (shell script tests, workspace tests excluding `e2e_test`, and doc
|
||||||
tests). Complete the applicable multi-role adversarial review described in
|
tests). Run `make pre-pr` before opening or updating a pull request — this is
|
||||||
`AGENTS.md` before running `make pre-pr`; then run the gate before opening or
|
what CI enforces.
|
||||||
updating a pull request. This is what CI enforces.
|
|
||||||
|
|
||||||
### 🔒 Git Pre-commit Hooks (optional)
|
### 🔒 Git Pre-commit Hooks (optional)
|
||||||
|
|
||||||
@@ -151,9 +150,8 @@ Example output when formatting fails:
|
|||||||
2. **Format your code**: `make fmt` or `cargo fmt --all`
|
2. **Format your code**: `make fmt` or `cargo fmt --all`
|
||||||
3. **Run the fast gate**: `make pre-commit` (no clippy, no tests)
|
3. **Run the fast gate**: `make pre-commit` (no clippy, no tests)
|
||||||
4. **Commit your changes**: `git commit -m "your message"`
|
4. **Commit your changes**: `git commit -m "your message"`
|
||||||
5. **Complete the applicable multi-role adversarial review** for non-exempt changes (see `AGENTS.md`)
|
5. **Run the full gate before opening/updating a PR**: `make pre-pr` (clippy + tests)
|
||||||
6. **Run the full gate before opening/updating a PR**: `make pre-pr` (clippy + tests)
|
6. **Push to your branch**: `git push`
|
||||||
7. **Push to your branch**: `git push`
|
|
||||||
|
|
||||||
### 🛠️ IDE Integration
|
### 🛠️ IDE Integration
|
||||||
|
|
||||||
|
|||||||
Generated
+325
-426
File diff suppressed because it is too large
Load Diff
+73
-75
@@ -41,7 +41,7 @@ members = [
|
|||||||
"crates/protocols", # Protocol implementations (FTPS, SFTP, etc.)
|
"crates/protocols", # Protocol implementations (FTPS, SFTP, etc.)
|
||||||
"crates/protos", # Protocol buffer definitions
|
"crates/protos", # Protocol buffer definitions
|
||||||
"crates/rio", # Rust I/O utilities and abstractions
|
"crates/rio", # Rust I/O utilities and abstractions
|
||||||
"crates/rio-v2", # MinIO on-disk format compatibility I/O layer (feature-gated, ships in no default build)
|
"crates/rio-v2", # Next-generation Rust I/O compatibility layer
|
||||||
"crates/replication", # Replication contracts and wire formats
|
"crates/replication", # Replication contracts and wire formats
|
||||||
"crates/concurrency", # Concurrency management for RustFS - timeout, locking, backpressure, and I/O scheduling
|
"crates/concurrency", # Concurrency management for RustFS - timeout, locking, backpressure, and I/O scheduling
|
||||||
"crates/s3-types", # S3 event type definitions
|
"crates/s3-types", # S3 event type definitions
|
||||||
@@ -69,7 +69,7 @@ edition = "2024"
|
|||||||
license = "Apache-2.0"
|
license = "Apache-2.0"
|
||||||
repository = "https://github.com/rustfs/rustfs"
|
repository = "https://github.com/rustfs/rustfs"
|
||||||
rust-version = "1.97.1"
|
rust-version = "1.97.1"
|
||||||
version = "1.0.0-rc.3"
|
version = "1.0.0-rc.1"
|
||||||
homepage = "https://rustfs.com"
|
homepage = "https://rustfs.com"
|
||||||
description = "RustFS is a high-performance distributed object storage software built using Rust, one of the most popular languages worldwide. "
|
description = "RustFS is a high-performance distributed object storage software built using Rust, one of the most popular languages worldwide. "
|
||||||
keywords = ["RustFS", "Minio", "object-storage", "filesystem", "s3"]
|
keywords = ["RustFS", "Minio", "object-storage", "filesystem", "s3"]
|
||||||
@@ -86,52 +86,52 @@ redundant_clone = "warn"
|
|||||||
|
|
||||||
[workspace.dependencies]
|
[workspace.dependencies]
|
||||||
# RustFS Internal Crates
|
# RustFS Internal Crates
|
||||||
rustfs = { path = "./rustfs", version = "1.0.0-rc.3" }
|
rustfs = { path = "./rustfs", version = "1.0.0-rc.1" }
|
||||||
rustfs-heal = { path = "crates/heal", version = "1.0.0-rc.3" }
|
rustfs-heal = { path = "crates/heal", version = "1.0.0-rc.1" }
|
||||||
rustfs-audit = { path = "crates/audit", version = "1.0.0-rc.3" }
|
rustfs-audit = { path = "crates/audit", version = "1.0.0-rc.1" }
|
||||||
rustfs-checksums = { path = "crates/checksums", version = "1.0.0-rc.3" }
|
rustfs-checksums = { path = "crates/checksums", version = "1.0.0-rc.1" }
|
||||||
rustfs-common = { path = "crates/common", version = "1.0.0-rc.3" }
|
rustfs-common = { path = "crates/common", version = "1.0.0-rc.1" }
|
||||||
rustfs-data-usage = { path = "crates/data-usage", version = "1.0.0-rc.3" }
|
rustfs-data-usage = { path = "crates/data-usage", version = "1.0.0-rc.1" }
|
||||||
rustfs-config = { path = "./crates/config", version = "1.0.0-rc.3" }
|
rustfs-config = { path = "./crates/config", version = "1.0.0-rc.1" }
|
||||||
rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.0-rc.3" }
|
rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.0-rc.1" }
|
||||||
rustfs-credentials = { path = "crates/credentials", version = "1.0.0-rc.3" }
|
rustfs-credentials = { path = "crates/credentials", version = "1.0.0-rc.1" }
|
||||||
rustfs-crypto = { path = "crates/crypto", version = "1.0.0-rc.3" }
|
rustfs-crypto = { path = "crates/crypto", version = "1.0.0-rc.1" }
|
||||||
rustfs-ecstore = { path = "crates/ecstore", version = "1.0.0-rc.3" }
|
rustfs-ecstore = { path = "crates/ecstore", version = "1.0.0-rc.1" }
|
||||||
rustfs-filemeta = { path = "crates/filemeta", version = "1.0.0-rc.3" }
|
rustfs-filemeta = { path = "crates/filemeta", version = "1.0.0-rc.1" }
|
||||||
rustfs-iam = { path = "crates/iam", version = "1.0.0-rc.3" }
|
rustfs-iam = { path = "crates/iam", version = "1.0.0-rc.1" }
|
||||||
rustfs-keystone = { path = "crates/keystone", version = "1.0.0-rc.3" }
|
rustfs-keystone = { path = "crates/keystone", version = "1.0.0-rc.1" }
|
||||||
rustfs-lifecycle = { path = "crates/lifecycle", version = "1.0.0-rc.3" }
|
rustfs-lifecycle = { path = "crates/lifecycle", version = "1.0.0-rc.1" }
|
||||||
rustfs-kms = { path = "crates/kms", version = "1.0.0-rc.3" }
|
rustfs-kms = { path = "crates/kms", version = "1.0.0-rc.1" }
|
||||||
rustfs-lock = { path = "crates/lock", version = "1.0.0-rc.3" }
|
rustfs-lock = { path = "crates/lock", version = "1.0.0-rc.1" }
|
||||||
rustfs-madmin = { path = "crates/madmin", version = "1.0.0-rc.3" }
|
rustfs-madmin = { path = "crates/madmin", version = "1.0.0-rc.1" }
|
||||||
rustfs-notify = { path = "crates/notify", version = "1.0.0-rc.3" }
|
rustfs-notify = { path = "crates/notify", version = "1.0.0-rc.1" }
|
||||||
rustfs-io-metrics = { path = "crates/io-metrics", version = "1.0.0-rc.3" }
|
rustfs-io-metrics = { path = "crates/io-metrics", version = "1.0.0-rc.1" }
|
||||||
rustfs-io-core = { path = "crates/io-core", version = "1.0.0-rc.3" }
|
rustfs-io-core = { path = "crates/io-core", version = "1.0.0-rc.1" }
|
||||||
rustfs-object-capacity = { path = "crates/object-capacity", version = "1.0.0-rc.3" }
|
rustfs-object-capacity = { path = "crates/object-capacity", version = "1.0.0-rc.1" }
|
||||||
rustfs-object-data-cache = { path = "crates/object-data-cache", version = "1.0.0-rc.3", default-features = false }
|
rustfs-object-data-cache = { path = "crates/object-data-cache", version = "1.0.0-rc.1", default-features = false }
|
||||||
rustfs-log-analyzer = { path = "crates/log-analyzer", version = "1.0.0-rc.3" }
|
rustfs-log-analyzer = { path = "crates/log-analyzer", version = "1.0.0-rc.1" }
|
||||||
rustfs-obs = { path = "crates/obs", version = "1.0.0-rc.3" }
|
rustfs-obs = { path = "crates/obs", version = "1.0.0-rc.1" }
|
||||||
rustfs-policy = { path = "crates/policy", version = "1.0.0-rc.3" }
|
rustfs-policy = { path = "crates/policy", version = "1.0.0-rc.1" }
|
||||||
rustfs-protos = { path = "crates/protos", version = "1.0.0-rc.3" }
|
rustfs-protos = { path = "crates/protos", version = "1.0.0-rc.1" }
|
||||||
rustfs-protocols = { path = "crates/protocols", version = "1.0.0-rc.3" }
|
rustfs-protocols = { path = "crates/protocols", version = "1.0.0-rc.1" }
|
||||||
rustfs-replication = { path = "crates/replication", version = "1.0.0-rc.3" }
|
rustfs-replication = { path = "crates/replication", version = "1.0.0-rc.1" }
|
||||||
rustfs-rio = { path = "crates/rio", version = "1.0.0-rc.3" }
|
rustfs-rio = { path = "crates/rio", version = "1.0.0-rc.1" }
|
||||||
rustfs-rio-v2 = { path = "crates/rio-v2", version = "1.0.0-rc.3" }
|
rustfs-rio-v2 = { path = "crates/rio-v2", version = "1.0.0-rc.1" }
|
||||||
rustfs-s3-types = { path = "crates/s3-types", version = "1.0.0-rc.3" }
|
rustfs-s3-types = { path = "crates/s3-types", version = "1.0.0-rc.1" }
|
||||||
rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.0-rc.3" }
|
rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.0-rc.1" }
|
||||||
rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.0-rc.3" }
|
rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.0-rc.1" }
|
||||||
rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.0-rc.3" }
|
rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.0-rc.1" }
|
||||||
rustfs-scanner = { path = "crates/scanner", version = "1.0.0-rc.3" }
|
rustfs-scanner = { path = "crates/scanner", version = "1.0.0-rc.1" }
|
||||||
rustfs-security-governance = { path = "crates/security-governance", version = "1.0.0-rc.3" }
|
rustfs-security-governance = { path = "crates/security-governance", version = "1.0.0-rc.1" }
|
||||||
rustfs-extension-schema = { path = "crates/extension-schema", version = "1.0.0-rc.3" }
|
rustfs-extension-schema = { path = "crates/extension-schema", version = "1.0.0-rc.1" }
|
||||||
rustfs-signer = { path = "crates/signer", version = "1.0.0-rc.3" }
|
rustfs-signer = { path = "crates/signer", version = "1.0.0-rc.1" }
|
||||||
rustfs-storage-api = { path = "crates/storage-api", version = "1.0.0-rc.3" }
|
rustfs-storage-api = { path = "crates/storage-api", version = "1.0.0-rc.1" }
|
||||||
rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.0-rc.3" }
|
rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.0-rc.1" }
|
||||||
rustfs-targets = { path = "crates/targets", version = "1.0.0-rc.3" }
|
rustfs-targets = { path = "crates/targets", version = "1.0.0-rc.1" }
|
||||||
rustfs-test-utils = { path = "crates/test-utils", version = "1.0.0-rc.3" }
|
rustfs-test-utils = { path = "crates/test-utils", version = "1.0.0-rc.1" }
|
||||||
rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.0-rc.3" }
|
rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.0-rc.1" }
|
||||||
rustfs-utils = { path = "crates/utils", version = "1.0.0-rc.3" }
|
rustfs-utils = { path = "crates/utils", version = "1.0.0-rc.1" }
|
||||||
rustfs-zip = { path = "./crates/zip", version = "1.0.0-rc.3" }
|
rustfs-zip = { path = "./crates/zip", version = "1.0.0-rc.1" }
|
||||||
|
|
||||||
# Async Runtime and Networking
|
# Async Runtime and Networking
|
||||||
async-channel = "2.5.0"
|
async-channel = "2.5.0"
|
||||||
@@ -142,10 +142,10 @@ async-recursion = "1.1.1"
|
|||||||
async-trait = "0.1.92"
|
async-trait = "0.1.92"
|
||||||
async-nats = { version = "0.50.0", default-features = false }
|
async-nats = { version = "0.50.0", default-features = false }
|
||||||
axum = "0.8.9"
|
axum = "0.8.9"
|
||||||
futures = "0.3.34"
|
futures = "0.3.33"
|
||||||
futures-core = "0.3.34"
|
futures-core = "0.3.33"
|
||||||
futures-lite = "2.6.1"
|
futures-lite = "2.6.1"
|
||||||
futures-util = "0.3.34"
|
futures-util = "0.3.33"
|
||||||
pollster = "1.0.1"
|
pollster = "1.0.1"
|
||||||
pulsar = { default-features = false, version = "6.8.0" }
|
pulsar = { default-features = false, version = "6.8.0" }
|
||||||
lapin = { default-features = false, version = "4.10.0" }
|
lapin = { default-features = false, version = "4.10.0" }
|
||||||
@@ -154,7 +154,7 @@ hyper-rustls = { default-features = false, version = "0.27.9" }
|
|||||||
hyper-util = { version = "0.1.20" }
|
hyper-util = { version = "0.1.20" }
|
||||||
http = "1.5.0"
|
http = "1.5.0"
|
||||||
http-body = "1.1.0"
|
http-body = "1.1.0"
|
||||||
http-body-util = "0.1.5"
|
http-body-util = "0.1.4"
|
||||||
minlz = "1.2.3"
|
minlz = "1.2.3"
|
||||||
reqwest = "0.13.4"
|
reqwest = "0.13.4"
|
||||||
rustfs-kafka-async = { version = "1.2.0" }
|
rustfs-kafka-async = { version = "1.2.0" }
|
||||||
@@ -171,7 +171,7 @@ tower = { version = "0.5.3" }
|
|||||||
tower-http = { version = "0.7.0" }
|
tower-http = { version = "0.7.0" }
|
||||||
|
|
||||||
# Serialization and Data Formats
|
# Serialization and Data Formats
|
||||||
apache-avro = { version = "0.22.0", features = ["snappy", "zstandard"] }
|
apache-avro = "0.21.0"
|
||||||
bytes = { version = "1.12.1" }
|
bytes = { version = "1.12.1" }
|
||||||
bytesize = "2.7.0"
|
bytesize = "2.7.0"
|
||||||
byteorder = "1.5.0"
|
byteorder = "1.5.0"
|
||||||
@@ -182,7 +182,6 @@ quick-xml = "0.41.0"
|
|||||||
rmp = { version = "0.8.15" }
|
rmp = { version = "0.8.15" }
|
||||||
rmp-serde = { version = "1.3.1" }
|
rmp-serde = { version = "1.3.1" }
|
||||||
serde = { version = "1.0.229" }
|
serde = { version = "1.0.229" }
|
||||||
serde_ignored = { version = "0.1" }
|
|
||||||
serde_json = { version = "1.0.151" }
|
serde_json = { version = "1.0.151" }
|
||||||
serde_urlencoded = "0.7.1"
|
serde_urlencoded = "0.7.1"
|
||||||
|
|
||||||
@@ -228,12 +227,12 @@ atoi = "3.1.0"
|
|||||||
atomic_enum = "0.3.0"
|
atomic_enum = "0.3.0"
|
||||||
aws-config = { version = "1.10.1" }
|
aws-config = { version = "1.10.1" }
|
||||||
aws-credential-types = { version = "1.3.0" }
|
aws-credential-types = { version = "1.3.0" }
|
||||||
aws-sdk-kms = { default-features = false, version = "1.115.0" }
|
aws-sdk-kms = { default-features = false, version = "1.114.0" }
|
||||||
aws-sdk-s3 = { default-features = false, version = "1.142.0" }
|
aws-sdk-s3 = { default-features = false, version = "1.141.0" }
|
||||||
aws-sdk-sts = { default-features = false, version = "1.111.0" }
|
aws-sdk-sts = { default-features = false, version = "1.110.0" }
|
||||||
aws-smithy-http-client = { default-features = false, version = "1.4.0" }
|
aws-smithy-http-client = { default-features = false, version = "1.2.0" }
|
||||||
aws-smithy-runtime-api = { version = "1.15.0" }
|
aws-smithy-runtime-api = { version = "1.14.0" }
|
||||||
aws-smithy-types = { version = "1.6.2" }
|
aws-smithy-types = { version = "1.6.1" }
|
||||||
base64 = "0.23.1"
|
base64 = "0.23.1"
|
||||||
base64-simd = "0.8.0"
|
base64-simd = "0.8.0"
|
||||||
brotli = "8.0.4"
|
brotli = "8.0.4"
|
||||||
@@ -245,7 +244,8 @@ crossbeam-queue = "0.3.13"
|
|||||||
crossbeam-channel = "0.5.16"
|
crossbeam-channel = "0.5.16"
|
||||||
crossbeam-deque = "0.8.7"
|
crossbeam-deque = "0.8.7"
|
||||||
crossbeam-utils = "0.8.22"
|
crossbeam-utils = "0.8.22"
|
||||||
datafusion = { default-features = false, version = "55.0.0" }
|
datafusion = { default-features = false, git = "https://github.com/apache/datafusion.git", rev = "e08aed1e5de41dcf81d529140dae07723b942a5e" }
|
||||||
|
#datafusion = { default-features = false, version = "54.1.0" }
|
||||||
derive_builder = "0.20.2"
|
derive_builder = "0.20.2"
|
||||||
enumset = "1.1.14"
|
enumset = "1.1.14"
|
||||||
faster-hex = "0.10.0"
|
faster-hex = "0.10.0"
|
||||||
@@ -263,13 +263,12 @@ lazy_static = "1.5.0"
|
|||||||
libc = "0.2.189"
|
libc = "0.2.189"
|
||||||
libsystemd = "0.7.2"
|
libsystemd = "0.7.2"
|
||||||
local-ip-address = "0.6.13"
|
local-ip-address = "0.6.13"
|
||||||
log = "0.4"
|
|
||||||
memmap2 = "0.9.11"
|
memmap2 = "0.9.11"
|
||||||
lz4 = "1.28.1"
|
lz4 = "1.28.1"
|
||||||
matchit = "0.9.2"
|
matchit = "0.9.2"
|
||||||
md-5 = "0.11.0"
|
md-5 = "0.11.0"
|
||||||
mime_guess = "2.0.5"
|
mime_guess = "2.0.5"
|
||||||
moka = { version = "0.12.16" }
|
moka = { version = "0.12.15" }
|
||||||
netif = "0.1.6"
|
netif = "0.1.6"
|
||||||
num_cpus = { version = "1.17.0" }
|
num_cpus = { version = "1.17.0" }
|
||||||
nvml-wrapper = "0.12.1"
|
nvml-wrapper = "0.12.1"
|
||||||
@@ -284,18 +283,18 @@ rayon = "1.12.0"
|
|||||||
reed-solomon-erasure = { package = "rustfs-erasure-codec", version = "8.0.2" }
|
reed-solomon-erasure = { package = "rustfs-erasure-codec", version = "8.0.2" }
|
||||||
reed-solomon-simd = "3.1.0"
|
reed-solomon-simd = "3.1.0"
|
||||||
regex = { version = "1.13.1" }
|
regex = { version = "1.13.1" }
|
||||||
rumqttc = { package = "rumqttc-next", version = "0.34.0" }
|
rumqttc = { package = "rumqttc-next", version = "0.33.3" }
|
||||||
redis = { version = "1.6.0" }
|
redis = { version = "1.5.0" }
|
||||||
rustify = { version = "0.7", default-features = false }
|
rustify = { version = "0.7", default-features = false }
|
||||||
rustix = { version = "1.1.4" }
|
rustix = { version = "1.1.4" }
|
||||||
rust-embed = { version = "8.12.0" }
|
rust-embed = { version = "8.12.0" }
|
||||||
rustc-hash = { version = "2.1.3" }
|
rustc-hash = { version = "2.1.3" }
|
||||||
s3s = { git = "https://github.com/rustfs/s3s.git", rev = "ed70cb048cc4be168419d461cb9ac3c2c7fa6d5a" }
|
s3s = { git = "https://github.com/rustfs/s3s.git", rev = "d7028511a53f69d41ed3c69f36899f9b1aede647" }
|
||||||
serial_test = "4.0.1"
|
serial_test = "4.0.1"
|
||||||
shadow-rs = { default-features = false, version = "2.0.0" }
|
shadow-rs = { default-features = false, version = "2.0.0" }
|
||||||
siphasher = "1.0.3"
|
siphasher = "1.0.3"
|
||||||
smallvec = { version = "1.15.2" }
|
smallvec = { version = "1.15.2" }
|
||||||
compact_str = "0.10.0"
|
smartstring = "1.0.1"
|
||||||
snap = "1.1.2"
|
snap = "1.1.2"
|
||||||
starshard = { version = "2.2.2" }
|
starshard = { version = "2.2.2" }
|
||||||
strum = { version = "0.28.0" }
|
strum = { version = "0.28.0" }
|
||||||
@@ -313,7 +312,7 @@ tracing-subscriber = { version = "0.3.23" }
|
|||||||
transform-stream = "0.3.1"
|
transform-stream = "0.3.1"
|
||||||
url = "2.5.8"
|
url = "2.5.8"
|
||||||
urlencoding = "2.1.3"
|
urlencoding = "2.1.3"
|
||||||
uuid = { version = "1.24.1" }
|
uuid = { version = "1.24.0" }
|
||||||
vaultrs = { version = "0.8.0" }
|
vaultrs = { version = "0.8.0" }
|
||||||
tar = "0.4.46"
|
tar = "0.4.46"
|
||||||
walkdir = "2.5.0"
|
walkdir = "2.5.0"
|
||||||
@@ -326,7 +325,6 @@ zstd = "0.13.3"
|
|||||||
|
|
||||||
# Observability and Metrics
|
# Observability and Metrics
|
||||||
metrics = "0.24.6"
|
metrics = "0.24.6"
|
||||||
metrics-util = "0.20"
|
|
||||||
dial9-tokio-telemetry = "0.3"
|
dial9-tokio-telemetry = "0.3"
|
||||||
opentelemetry = { version = "0.32.0" }
|
opentelemetry = { version = "0.32.0" }
|
||||||
opentelemetry-appender-tracing = { version = "0.32.0" }
|
opentelemetry-appender-tracing = { version = "0.32.0" }
|
||||||
@@ -340,18 +338,18 @@ pyroscope = { version = "2.1.1" }
|
|||||||
# FTP and SFTP
|
# FTP and SFTP
|
||||||
libunftp = { version = "0.23.0" }
|
libunftp = { version = "0.23.0" }
|
||||||
unftp-core = "0.1.0"
|
unftp-core = "0.1.0"
|
||||||
suppaftp = { version = "10.0.2" }
|
suppaftp = { version = "10.0.1" }
|
||||||
rcgen = { version = "0.14.9", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
|
rcgen = { version = "0.14.8", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
|
||||||
russh = { version = "0.62.7" }
|
russh = { version = "0.62.5" }
|
||||||
russh-sftp = "2.4.0"
|
russh-sftp = "2.4.0"
|
||||||
|
|
||||||
# WebDAV
|
# WebDAV
|
||||||
dav-server = "0.11.0"
|
dav-server = "0.11.0"
|
||||||
|
|
||||||
# Performance Analysis and Memory Profiling
|
# Performance Analysis and Memory Profiling
|
||||||
mimalloc = { version = "0.1.52", git = "https://github.com/xonatius/mimalloc_rust.git", rev = "6d4c41bb10c6d9da1d1b6f07b38c4cc051667f11" }
|
mimalloc = { version = "0.1.52", git = "https://github.com/xonatius/mimalloc_rust.git", rev = "ce6338661179c8be22e516b00af7483f151485a7" }
|
||||||
libmimalloc-sys = { version = "0.1.49", git = "https://github.com/xonatius/mimalloc_rust.git", rev = "6d4c41bb10c6d9da1d1b6f07b38c4cc051667f11", features = ["extended"] }
|
libmimalloc-sys = { version = "0.1.49", git = "https://github.com/xonatius/mimalloc_rust.git", rev = "ce6338661179c8be22e516b00af7483f151485a7", features = ["extended"] }
|
||||||
hotpath = { version = "0.23.3", default-features = false }
|
hotpath = { version = "0.23.1", default-features = false }
|
||||||
# Snapshot testing for output format regression detection
|
# Snapshot testing for output format regression detection
|
||||||
insta = { version = "1.48" }
|
insta = { version = "1.48" }
|
||||||
|
|
||||||
|
|||||||
@@ -116,7 +116,7 @@ chown -R 10001:10001 data logs
|
|||||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
|
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
|
||||||
|
|
||||||
# Using specific version
|
# Using specific version
|
||||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-rc.3
|
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-rc.1
|
||||||
```
|
```
|
||||||
|
|
||||||
If you use [podman](https://github.com/containers/podman) instead of docker, you can install the RustFS with the below command
|
If you use [podman](https://github.com/containers/podman) instead of docker, you can install the RustFS with the below command
|
||||||
|
|||||||
+1
-1
@@ -113,7 +113,7 @@ chown -R 10001:10001 data logs
|
|||||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
|
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
|
||||||
|
|
||||||
# 使用指定版本运行
|
# 使用指定版本运行
|
||||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-rc.3
|
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-rc.1
|
||||||
```
|
```
|
||||||
|
|
||||||
如果您通过绑定挂载启用 TLS 证书目录,也请用同样方式准备该目录:
|
如果您通过绑定挂载启用 TLS 证书目录,也请用同样方式准备该目录:
|
||||||
|
|||||||
@@ -40,7 +40,6 @@ mak = "mak"
|
|||||||
gae = "gae"
|
gae = "gae"
|
||||||
GAE = "GAE"
|
GAE = "GAE"
|
||||||
thr = "thr"
|
thr = "thr"
|
||||||
mis = "mis"
|
|
||||||
# s3-tests original test names (cannot be changed)
|
# s3-tests original test names (cannot be changed)
|
||||||
nonexisted = "nonexisted"
|
nonexisted = "nonexisted"
|
||||||
consts = "consts"
|
consts = "consts"
|
||||||
|
|||||||
@@ -236,19 +236,12 @@ async fn audit_pipeline_reports_empty_runtime_snapshots() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn stopping_audit_replay_workers_is_a_no_op_when_there_are_none() {
|
async fn audit_runtime_facade_stops_empty_replay_workers() {
|
||||||
let registry = Arc::new(Mutex::new(AuditRegistry::new()));
|
let registry = Arc::new(Mutex::new(AuditRegistry::new()));
|
||||||
let replay_workers = Arc::new(RwLock::new(rustfs_targets::ReplayWorkerManager::new()));
|
let replay_workers = Arc::new(RwLock::new(rustfs_targets::ReplayWorkerManager::new()));
|
||||||
let facade = AuditRuntimeFacade::new(registry, Arc::clone(&replay_workers));
|
let facade = AuditRuntimeFacade::new(registry, replay_workers);
|
||||||
|
|
||||||
facade.stop_replay_workers().await;
|
facade.stop_replay_workers().await;
|
||||||
|
|
||||||
// The stop path takes the manager's workers and hands them to the adapter,
|
|
||||||
// so an empty facade must leave it empty rather than wedge it, and a second
|
|
||||||
// call — which shutdown paths make — must stay harmless (rustfs/backlog#1836).
|
|
||||||
assert!(replay_workers.read().await.is_empty());
|
|
||||||
facade.stop_replay_workers().await;
|
|
||||||
assert!(replay_workers.read().await.is_empty());
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
|||||||
@@ -11,6 +11,7 @@
|
|||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
// See the License for the specific language governing permissions and
|
// See the License for the specific language governing permissions and
|
||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
#![allow(dead_code)]
|
||||||
|
|
||||||
use base64_simd::STANDARD;
|
use base64_simd::STANDARD;
|
||||||
|
|
||||||
|
|||||||
@@ -21,13 +21,6 @@ use crate::{
|
|||||||
Xxhash3, Xxhash64, Xxhash128,
|
Xxhash3, Xxhash64, Xxhash128,
|
||||||
};
|
};
|
||||||
|
|
||||||
// DELIBERATE DUPLICATION of the x-amz-checksum-* names that also exist as
|
|
||||||
// AMZ_CHECKSUM_* in rustfs-utils' headers module (crates/utils/src/http/
|
|
||||||
// headers.rs): this crate is a zero-internal-dependency leaf, so it cannot
|
|
||||||
// import them, and it additionally owns the RustFS extension names
|
|
||||||
// (sha512/xxhash*) that utils does not carry. Values are pinned by the S3
|
|
||||||
// wire protocol; do not merge without a maintainer decision on the leaf
|
|
||||||
// boundary (backlog#1833).
|
|
||||||
pub const CRC_32_HEADER_NAME: &str = "x-amz-checksum-crc32";
|
pub const CRC_32_HEADER_NAME: &str = "x-amz-checksum-crc32";
|
||||||
pub const CRC_32_C_HEADER_NAME: &str = "x-amz-checksum-crc32c";
|
pub const CRC_32_C_HEADER_NAME: &str = "x-amz-checksum-crc32c";
|
||||||
pub const SHA_1_HEADER_NAME: &str = "x-amz-checksum-sha1";
|
pub const SHA_1_HEADER_NAME: &str = "x-amz-checksum-sha1";
|
||||||
@@ -38,10 +31,7 @@ pub const XXHASH_3_HEADER_NAME: &str = "x-amz-checksum-xxhash3";
|
|||||||
pub const XXHASH_64_HEADER_NAME: &str = "x-amz-checksum-xxhash64";
|
pub const XXHASH_64_HEADER_NAME: &str = "x-amz-checksum-xxhash64";
|
||||||
pub const XXHASH_128_HEADER_NAME: &str = "x-amz-checksum-xxhash128";
|
pub const XXHASH_128_HEADER_NAME: &str = "x-amz-checksum-xxhash128";
|
||||||
|
|
||||||
#[allow(
|
#[allow(dead_code)]
|
||||||
dead_code,
|
|
||||||
reason = "Content-MD5 wire name, resolved by header_name() below and asserted by this crate's tests (backlog#1823)"
|
|
||||||
)]
|
|
||||||
pub(crate) static MD5_HEADER_NAME: &str = "content-md5";
|
pub(crate) static MD5_HEADER_NAME: &str = "content-md5";
|
||||||
|
|
||||||
pub const CHECKSUM_ALGORITHMS_IN_PRIORITY_ORDER: [&str; 5] =
|
pub const CHECKSUM_ALGORITHMS_IN_PRIORITY_ORDER: [&str; 5] =
|
||||||
|
|||||||
@@ -41,14 +41,6 @@ pub const XXHASH_64_NAME: &str = "xxhash64";
|
|||||||
pub const XXHASH_128_NAME: &str = "xxhash128";
|
pub const XXHASH_128_NAME: &str = "xxhash128";
|
||||||
pub const MD5_NAME: &str = "md5";
|
pub const MD5_NAME: &str = "md5";
|
||||||
|
|
||||||
/// One of three deliberately separate checksum registries (backlog#1833):
|
|
||||||
/// this enum owns the **streaming-hash algorithm registry**, including the
|
|
||||||
/// RustFS extensions (sha512, xxhash3/64/128). The on-disk xl.meta bitset
|
|
||||||
/// lives in `rustfs_rio::ChecksumType` (crates/rio/src/checksum.rs, varint
|
|
||||||
/// bits are append-only), and the MinIO-port client keeps its own
|
|
||||||
/// `ChecksumMode` (crates/ecstore/src/client/checksum.rs). When adding an
|
|
||||||
/// algorithm, extend all three (or record why not) — they do not derive from
|
|
||||||
/// each other.
|
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
|
||||||
#[non_exhaustive]
|
#[non_exhaustive]
|
||||||
pub enum ChecksumAlgorithm {
|
pub enum ChecksumAlgorithm {
|
||||||
@@ -476,19 +468,13 @@ impl Checksum for Xxhash64 {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(dead_code)]
|
||||||
#[derive(Debug, Default)]
|
#[derive(Debug, Default)]
|
||||||
#[allow(
|
|
||||||
dead_code,
|
|
||||||
reason = "Content-MD5 is not a ChecksumAlgorithm variant and has no arm in into_impl: S3 carries it as its own header, separate from the x-amz-checksum-* family. This impl exists so the two paths share the Checksum trait, and is asserted by this crate's tests (backlog#1823)"
|
|
||||||
)]
|
|
||||||
struct Md5 {
|
struct Md5 {
|
||||||
hasher: md5::Md5,
|
hasher: md5::Md5,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(
|
#[allow(dead_code)]
|
||||||
dead_code,
|
|
||||||
reason = "Content-MD5 is not a ChecksumAlgorithm variant and has no arm in into_impl: S3 carries it as its own header, separate from the x-amz-checksum-* family. This impl exists so the two paths share the Checksum trait, and is asserted by this crate's tests (backlog#1823)"
|
|
||||||
)]
|
|
||||||
impl Md5 {
|
impl Md5 {
|
||||||
fn update(&mut self, bytes: &[u8]) {
|
fn update(&mut self, bytes: &[u8]) {
|
||||||
use md5::Digest;
|
use md5::Digest;
|
||||||
|
|||||||
@@ -42,8 +42,8 @@ chrono = { workspace = true, features = ["serde"] }
|
|||||||
jiff = { workspace = true, features = ["serde"] }
|
jiff = { workspace = true, features = ["serde"] }
|
||||||
metrics = { workspace = true }
|
metrics = { workspace = true }
|
||||||
serde = { workspace = true, features = ["derive"] }
|
serde = { workspace = true, features = ["derive"] }
|
||||||
smallvec = { workspace = true }
|
|
||||||
rmp-serde = { workspace = true }
|
rmp-serde = { workspace = true }
|
||||||
|
s3s = { workspace = true, features = ["minio"] }
|
||||||
tracing = { workspace = true }
|
tracing = { workspace = true }
|
||||||
|
|
||||||
[dev-dependencies]
|
[dev-dependencies]
|
||||||
|
|||||||
@@ -0,0 +1,87 @@
|
|||||||
|
// Copyright 2024 RustFS Team
|
||||||
|
//
|
||||||
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
// you may not use this file except in compliance with the License.
|
||||||
|
// You may obtain a copy of the License at
|
||||||
|
//
|
||||||
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
//
|
||||||
|
// Unless required by applicable law or agreed to in writing, software
|
||||||
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
// See the License for the specific language governing permissions and
|
||||||
|
// limitations under the License.
|
||||||
|
|
||||||
|
use crate::last_minute::{self};
|
||||||
|
use std::collections::HashMap;
|
||||||
|
|
||||||
|
pub struct ReplicationLatency {
|
||||||
|
// Delays for single and multipart PUT requests
|
||||||
|
upload_histogram: last_minute::LastMinuteHistogram,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ReplicationLatency {
|
||||||
|
// Merge two ReplicationLatency
|
||||||
|
pub fn merge(&mut self, other: &mut ReplicationLatency) -> &ReplicationLatency {
|
||||||
|
self.upload_histogram.merge(&other.upload_histogram);
|
||||||
|
self
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get upload delay (categorized by object size interval)
|
||||||
|
pub fn get_upload_latency(&mut self) -> HashMap<String, u64> {
|
||||||
|
let mut ret = HashMap::new();
|
||||||
|
let avg = self.upload_histogram.get_avg_data();
|
||||||
|
for (i, v) in avg.iter().enumerate() {
|
||||||
|
let avg_duration = v.avg();
|
||||||
|
ret.insert(self.size_tag_to_string(i), avg_duration.as_millis() as u64);
|
||||||
|
}
|
||||||
|
ret
|
||||||
|
}
|
||||||
|
pub fn update(&mut self, size: i64, during: std::time::Duration) {
|
||||||
|
self.upload_histogram.add(size, during);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Simulate the conversion from size tag to string
|
||||||
|
fn size_tag_to_string(&self, tag: usize) -> String {
|
||||||
|
match tag {
|
||||||
|
0 => String::from("Size < 1 KiB"),
|
||||||
|
1 => String::from("Size < 1 MiB"),
|
||||||
|
2 => String::from("Size < 10 MiB"),
|
||||||
|
3 => String::from("Size < 100 MiB"),
|
||||||
|
4 => String::from("Size < 1 GiB"),
|
||||||
|
_ => String::from("Size > 1 GiB"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// #[derive(Debug, Clone, Default)]
|
||||||
|
// pub struct ReplicationLastMinute {
|
||||||
|
// pub last_minute: LastMinuteLatency,
|
||||||
|
// }
|
||||||
|
|
||||||
|
// impl ReplicationLastMinute {
|
||||||
|
// pub fn merge(&mut self, other: ReplicationLastMinute) -> ReplicationLastMinute {
|
||||||
|
// let mut nl = ReplicationLastMinute::default();
|
||||||
|
// nl.last_minute = self.last_minute.merge(&mut other.last_minute);
|
||||||
|
// nl
|
||||||
|
// }
|
||||||
|
|
||||||
|
// pub fn add_size(&mut self, n: i64) {
|
||||||
|
// let t = SystemTime::now()
|
||||||
|
// .duration_since(UNIX_EPOCH)
|
||||||
|
// .expect("Time went backwards")
|
||||||
|
// .as_secs();
|
||||||
|
// self.last_minute.add_all(t - 1, &AccElem { total: t - 1, size: n as u64, n: 1 });
|
||||||
|
// }
|
||||||
|
|
||||||
|
// pub fn get_total(&self) -> AccElem {
|
||||||
|
// self.last_minute.get_total()
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
|
||||||
|
// impl fmt::Display for ReplicationLastMinute {
|
||||||
|
// fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
|
||||||
|
// let t = self.last_minute.get_total();
|
||||||
|
// write!(f, "ReplicationLastMinute sz= {}, n= {}, dur= {}", t.size, t.n, t.total)
|
||||||
|
// }
|
||||||
|
// }
|
||||||
@@ -12,6 +12,7 @@
|
|||||||
// See the License for the specific language governing permissions and
|
// See the License for the specific language governing permissions and
|
||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
|
use s3s::dto::{BucketLifecycleConfiguration, ExpirationStatus, LifecycleRule, ReplicationConfiguration, ReplicationRuleStatus};
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use std::{
|
use std::{
|
||||||
fmt::{self, Display},
|
fmt::{self, Display},
|
||||||
@@ -223,13 +224,6 @@ pub struct HealOpts {
|
|||||||
pub enum HealAdmissionDropReason {
|
pub enum HealAdmissionDropReason {
|
||||||
QueueFull,
|
QueueFull,
|
||||||
PolicyDropped,
|
PolicyDropped,
|
||||||
/// HS-06: an admin heal start overlaps (same bucket with mutually
|
|
||||||
/// containing prefixes, or the same erasure set) an already running or
|
|
||||||
/// queued task. Only produced when RUSTFS_HEAL_OVERLAP_POLICY=minio_error.
|
|
||||||
AlreadyRunning,
|
|
||||||
/// HS-06: same as [`Self::AlreadyRunning`] but for paths that merely
|
|
||||||
/// contain (or are contained by) the active task's path.
|
|
||||||
OverlappingPaths,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
impl HealAdmissionDropReason {
|
impl HealAdmissionDropReason {
|
||||||
@@ -237,8 +231,6 @@ impl HealAdmissionDropReason {
|
|||||||
match self {
|
match self {
|
||||||
Self::QueueFull => "queue_full",
|
Self::QueueFull => "queue_full",
|
||||||
Self::PolicyDropped => "policy_dropped",
|
Self::PolicyDropped => "policy_dropped",
|
||||||
Self::AlreadyRunning => "already_running",
|
|
||||||
Self::OverlappingPaths => "overlapping_paths",
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -295,9 +287,6 @@ pub enum HealRequestSource {
|
|||||||
Scanner,
|
Scanner,
|
||||||
AutoHeal,
|
AutoHeal,
|
||||||
ReadRepair,
|
ReadRepair,
|
||||||
/// Mission Repair Feed: intents delivered by error paths and replayed
|
|
||||||
/// from the durable MRF journal.
|
|
||||||
Mrf,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
impl HealRequestSource {
|
impl HealRequestSource {
|
||||||
@@ -308,7 +297,6 @@ impl HealRequestSource {
|
|||||||
Self::Scanner => "scanner",
|
Self::Scanner => "scanner",
|
||||||
Self::AutoHeal => "auto_heal",
|
Self::AutoHeal => "auto_heal",
|
||||||
Self::ReadRepair => "read_repair",
|
Self::ReadRepair => "read_repair",
|
||||||
Self::Mrf => "mrf",
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -325,9 +313,6 @@ pub enum HealChannelCommand {
|
|||||||
Query {
|
Query {
|
||||||
heal_path: String,
|
heal_path: String,
|
||||||
client_token: String,
|
client_token: String,
|
||||||
/// Incremental result cursor (HS-06): only items with a sequence
|
|
||||||
/// greater than this are returned; `None` keeps the full snapshot.
|
|
||||||
since_seq: Option<u64>,
|
|
||||||
response_tx: oneshot::Sender<Result<HealChannelResponse, String>>,
|
response_tx: oneshot::Sender<Result<HealChannelResponse, String>>,
|
||||||
},
|
},
|
||||||
/// Cancel heal task
|
/// Cancel heal task
|
||||||
@@ -533,21 +518,10 @@ async fn receive_heal_channel_response(
|
|||||||
|
|
||||||
/// Send heal query request
|
/// Send heal query request
|
||||||
pub async fn query_heal_status(heal_path: String, client_token: String) -> Result<HealChannelResponse, String> {
|
pub async fn query_heal_status(heal_path: String, client_token: String) -> Result<HealChannelResponse, String> {
|
||||||
query_heal_status_since(heal_path, client_token, None).await
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Incremental heal query (HS-06): pass the client's last seen sequence
|
|
||||||
/// number to receive only newer result items.
|
|
||||||
pub async fn query_heal_status_since(
|
|
||||||
heal_path: String,
|
|
||||||
client_token: String,
|
|
||||||
since_seq: Option<u64>,
|
|
||||||
) -> Result<HealChannelResponse, String> {
|
|
||||||
let (response_tx, response_rx) = oneshot::channel();
|
let (response_tx, response_rx) = oneshot::channel();
|
||||||
send_heal_command(HealChannelCommand::Query {
|
send_heal_command(HealChannelCommand::Query {
|
||||||
heal_path,
|
heal_path,
|
||||||
client_token,
|
client_token,
|
||||||
since_seq,
|
|
||||||
response_tx,
|
response_tx,
|
||||||
})
|
})
|
||||||
.await?;
|
.await?;
|
||||||
@@ -632,6 +606,104 @@ pub fn create_heal_response(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn lc_get_prefix(rule: &LifecycleRule) -> String {
|
||||||
|
if let Some(p) = &rule.prefix {
|
||||||
|
return p.to_string();
|
||||||
|
} else if let Some(filter) = &rule.filter {
|
||||||
|
if let Some(p) = &filter.prefix {
|
||||||
|
return p.to_string();
|
||||||
|
} else if let Some(and) = &filter.and
|
||||||
|
&& let Some(p) = &and.prefix
|
||||||
|
{
|
||||||
|
return p.to_string();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
"".into()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn lc_has_active_rules(config: &BucketLifecycleConfiguration, prefix: &str) -> bool {
|
||||||
|
if config.rules.is_empty() {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
for rule in config.rules.iter() {
|
||||||
|
if rule.status == ExpirationStatus::from_static(ExpirationStatus::DISABLED) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let rule_prefix = lc_get_prefix(rule);
|
||||||
|
if !prefix.is_empty() && !rule_prefix.is_empty() && !prefix.starts_with(&rule_prefix) && !rule_prefix.starts_with(prefix)
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if let Some(e) = &rule.noncurrent_version_expiration {
|
||||||
|
if e.noncurrent_days.is_some() {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
if let Some(true) = e.newer_noncurrent_versions.map(|d| d > 0) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if rule.noncurrent_version_transitions.is_some() {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
if let Some(true) = rule.expiration.as_ref().map(|e| e.date.is_some()) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
if let Some(true) = rule.expiration.as_ref().map(|e| e.days.is_some()) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
if let Some(Some(true)) = rule.expiration.as_ref().map(|e| e.expired_object_delete_marker) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
if let Some(true) = rule.transitions.as_ref().map(|t| !t.is_empty()) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
if rule.transitions.is_some() {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
false
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn rep_has_active_rules(config: &ReplicationConfiguration, prefix: &str, recursive: bool) -> bool {
|
||||||
|
if config.rules.is_empty() {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
for rule in config.rules.iter() {
|
||||||
|
if rule
|
||||||
|
.status
|
||||||
|
.eq(&ReplicationRuleStatus::from_static(ReplicationRuleStatus::DISABLED))
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if !prefix.is_empty()
|
||||||
|
&& let Some(filter) = &rule.filter
|
||||||
|
&& let Some(r_prefix) = &filter.prefix
|
||||||
|
&& !r_prefix.is_empty()
|
||||||
|
{
|
||||||
|
// incoming prefix must be in rule prefix
|
||||||
|
if !recursive && !prefix.starts_with(r_prefix) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
// If recursive, we can skip this rule if it doesn't match the tested prefix or level below prefix
|
||||||
|
// does not match
|
||||||
|
if recursive && !r_prefix.starts_with(prefix) && !prefix.starts_with(r_prefix) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
false
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn send_heal_disk(set_disk_id: String, priority: Option<HealChannelPriority>) -> Result<(), String> {
|
pub async fn send_heal_disk(set_disk_id: String, priority: Option<HealChannelPriority>) -> Result<(), String> {
|
||||||
let req = HealChannelRequest {
|
let req = HealChannelRequest {
|
||||||
id: Uuid::new_v4().to_string(),
|
id: Uuid::new_v4().to_string(),
|
||||||
|
|||||||
@@ -13,6 +13,82 @@
|
|||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
||||||
|
|
||||||
|
#[allow(dead_code)]
|
||||||
|
#[derive(Debug, Default)]
|
||||||
|
struct TimedAction {
|
||||||
|
count: u64,
|
||||||
|
acc_time: u64,
|
||||||
|
min_time: Option<u64>,
|
||||||
|
max_time: Option<u64>,
|
||||||
|
bytes: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[allow(dead_code)]
|
||||||
|
impl TimedAction {
|
||||||
|
// Avg returns the average time spent on the action.
|
||||||
|
pub fn avg(&self) -> Option<Duration> {
|
||||||
|
if self.count == 0 {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
Some(Duration::from_nanos(self.acc_time / self.count))
|
||||||
|
}
|
||||||
|
|
||||||
|
// AvgBytes returns the average bytes processed.
|
||||||
|
pub fn avg_bytes(&self) -> u64 {
|
||||||
|
if self.count == 0 {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
self.bytes / self.count
|
||||||
|
}
|
||||||
|
|
||||||
|
// Merge other into t.
|
||||||
|
pub fn merge(&mut self, other: TimedAction) {
|
||||||
|
self.count += other.count;
|
||||||
|
self.acc_time += other.acc_time;
|
||||||
|
self.bytes += other.bytes;
|
||||||
|
|
||||||
|
if self.count == 0 {
|
||||||
|
self.min_time = other.min_time;
|
||||||
|
}
|
||||||
|
if let Some(other_min) = other.min_time {
|
||||||
|
self.min_time = self.min_time.map_or(Some(other_min), |min| Some(min.min(other_min)));
|
||||||
|
}
|
||||||
|
|
||||||
|
self.max_time = self
|
||||||
|
.max_time
|
||||||
|
.map_or(other.max_time, |max| Some(max.max(other.max_time.unwrap_or(0))));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[allow(dead_code)]
|
||||||
|
#[derive(Debug)]
|
||||||
|
enum SizeCategory {
|
||||||
|
SizeLessThan1KiB = 0,
|
||||||
|
SizeLessThan1MiB,
|
||||||
|
SizeLessThan10MiB,
|
||||||
|
SizeLessThan100MiB,
|
||||||
|
SizeLessThan1GiB,
|
||||||
|
SizeGreaterThan1GiB,
|
||||||
|
// Add new entries here
|
||||||
|
SizeLastElemMarker,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl std::fmt::Display for SizeCategory {
|
||||||
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
|
let s = match *self {
|
||||||
|
SizeCategory::SizeLessThan1KiB => "SizeLessThan1KiB",
|
||||||
|
SizeCategory::SizeLessThan1MiB => "SizeLessThan1MiB",
|
||||||
|
SizeCategory::SizeLessThan10MiB => "SizeLessThan10MiB",
|
||||||
|
SizeCategory::SizeLessThan100MiB => "SizeLessThan100MiB",
|
||||||
|
SizeCategory::SizeLessThan1GiB => "SizeLessThan1GiB",
|
||||||
|
SizeCategory::SizeGreaterThan1GiB => "SizeGreaterThan1GiB",
|
||||||
|
SizeCategory::SizeLastElemMarker => "SizeLastElemMarker",
|
||||||
|
};
|
||||||
|
write!(f, "{s}")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Clone, Debug, Default, Copy)]
|
#[derive(Clone, Debug, Default, Copy)]
|
||||||
pub struct AccElem {
|
pub struct AccElem {
|
||||||
pub total: u64,
|
pub total: u64,
|
||||||
@@ -496,3 +572,44 @@ mod tests {
|
|||||||
assert_eq!(total.n, 6);
|
assert_eq!(total.n, 6);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const SIZE_LAST_ELEM_MARKER: usize = 10; // Assumed marker size is 10, modify according to actual situation
|
||||||
|
|
||||||
|
#[allow(dead_code)]
|
||||||
|
#[derive(Debug, Default)]
|
||||||
|
pub struct LastMinuteHistogram {
|
||||||
|
histogram: Vec<LastMinuteLatency>,
|
||||||
|
size: u32,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl LastMinuteHistogram {
|
||||||
|
pub fn merge(&mut self, other: &LastMinuteHistogram) {
|
||||||
|
for i in 0..self.histogram.len() {
|
||||||
|
self.histogram[i].merge(&other.histogram[i]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn add(&mut self, size: i64, t: Duration) {
|
||||||
|
let index = size_to_tag(size);
|
||||||
|
self.histogram[index].add(&t);
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn get_avg_data(&mut self) -> [AccElem; SIZE_LAST_ELEM_MARKER] {
|
||||||
|
let mut res = [AccElem::default(); SIZE_LAST_ELEM_MARKER];
|
||||||
|
for (i, elem) in self.histogram.iter_mut().enumerate() {
|
||||||
|
res[i] = elem.get_total();
|
||||||
|
}
|
||||||
|
res
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn size_to_tag(size: i64) -> usize {
|
||||||
|
match size {
|
||||||
|
_ if size < 1024 => 0, // sizeLessThan1KiB
|
||||||
|
_ if size < 1024 * 1024 => 1, // sizeLessThan1MiB
|
||||||
|
_ if size < 10 * 1024 * 1024 => 2, // sizeLessThan10MiB
|
||||||
|
_ if size < 100 * 1024 * 1024 => 3, // sizeLessThan100MiB
|
||||||
|
_ if size < 1024 * 1024 * 1024 => 4, // sizeLessThan1GiB
|
||||||
|
_ => 5, // sizeGreaterThan1GiB
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -12,15 +12,13 @@
|
|||||||
// See the License for the specific language governing permissions and
|
// See the License for the specific language governing permissions and
|
||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
|
pub mod bucket_stats;
|
||||||
// pub mod error;
|
// pub mod error;
|
||||||
pub mod globals;
|
pub mod globals;
|
||||||
pub mod heal_channel;
|
pub mod heal_channel;
|
||||||
pub mod last_minute;
|
pub mod last_minute;
|
||||||
pub mod metrics;
|
pub mod metrics;
|
||||||
pub mod mrf_channel;
|
|
||||||
mod readiness;
|
mod readiness;
|
||||||
pub mod table_catalog;
|
|
||||||
pub mod trace_bus;
|
|
||||||
|
|
||||||
pub use globals::*;
|
pub use globals::*;
|
||||||
pub use readiness::{GlobalReadiness, SystemStage};
|
pub use readiness::{GlobalReadiness, SystemStage};
|
||||||
|
|||||||
+25
-144
@@ -729,7 +729,7 @@ fn timestamp_elapsed_seconds_since(now: Timestamp, earlier: Timestamp) -> u64 {
|
|||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
u64::try_from(duration.as_secs()).unwrap_or(u64::MAX)
|
u64::try_from(duration.as_secs()).map_or(u64::MAX, |seconds| seconds)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone, Copy, Debug, Default)]
|
#[derive(Clone, Copy, Debug, Default)]
|
||||||
@@ -781,19 +781,6 @@ struct ScannerBucketDriveResultValue {
|
|||||||
last_seen: u64,
|
last_seen: u64,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone, Debug, Eq, Hash, PartialEq)]
|
|
||||||
struct ScannerActiveBucketDriveKey {
|
|
||||||
source: String,
|
|
||||||
bucket: String,
|
|
||||||
drive: String,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone, Copy, Debug)]
|
|
||||||
struct ScannerActiveBucketDriveValue {
|
|
||||||
count: u64,
|
|
||||||
started_at: Timestamp,
|
|
||||||
}
|
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Metrics
|
// Metrics
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
@@ -826,7 +813,6 @@ pub struct Metrics {
|
|||||||
scanner_set_scans_active: AtomicU64,
|
scanner_set_scans_active: AtomicU64,
|
||||||
scanner_disk_bucket_scan_states: Mutex<HashMap<ScannerDiskBucketScanKey, ScannerDiskBucketScanState>>,
|
scanner_disk_bucket_scan_states: Mutex<HashMap<ScannerDiskBucketScanKey, ScannerDiskBucketScanState>>,
|
||||||
scanner_bucket_drive_results: Mutex<ScannerBucketDriveResults>,
|
scanner_bucket_drive_results: Mutex<ScannerBucketDriveResults>,
|
||||||
scanner_active_bucket_drive_scans: Mutex<HashMap<ScannerActiveBucketDriveKey, ScannerActiveBucketDriveValue>>,
|
|
||||||
scanner_bucket_drive_result_clock: AtomicU64,
|
scanner_bucket_drive_result_clock: AtomicU64,
|
||||||
current_scan_cycle_bucket_drive_results_start: Mutex<HashMap<ScannerBucketDriveResultKey, u64>>,
|
current_scan_cycle_bucket_drive_results_start: Mutex<HashMap<ScannerBucketDriveResultKey, u64>>,
|
||||||
last_scan_cycle_bucket_drive_results: Mutex<Vec<ScannerBucketDriveResultSnapshot>>,
|
last_scan_cycle_bucket_drive_results: Mutex<Vec<ScannerBucketDriveResultSnapshot>>,
|
||||||
@@ -929,13 +915,11 @@ const SCAN_CYCLE_RESULT_SUCCESS: u8 = 1;
|
|||||||
const SCAN_CYCLE_RESULT_ERROR: u8 = 2;
|
const SCAN_CYCLE_RESULT_ERROR: u8 = 2;
|
||||||
const SCAN_CYCLE_RESULT_PARTIAL: u8 = 3;
|
const SCAN_CYCLE_RESULT_PARTIAL: u8 = 3;
|
||||||
const SCAN_CYCLE_RESULT_SUPERSEDED: u8 = 4;
|
const SCAN_CYCLE_RESULT_SUPERSEDED: u8 = 4;
|
||||||
const SCAN_CYCLE_RESULT_DEFERRED: u8 = 5;
|
|
||||||
const SCAN_CYCLE_RESULT_UNKNOWN_LABEL: &str = "unknown";
|
const SCAN_CYCLE_RESULT_UNKNOWN_LABEL: &str = "unknown";
|
||||||
const SCAN_CYCLE_RESULT_SUCCESS_LABEL: &str = "success";
|
const SCAN_CYCLE_RESULT_SUCCESS_LABEL: &str = "success";
|
||||||
const SCAN_CYCLE_RESULT_ERROR_LABEL: &str = "error";
|
const SCAN_CYCLE_RESULT_ERROR_LABEL: &str = "error";
|
||||||
const SCAN_CYCLE_RESULT_PARTIAL_LABEL: &str = "partial";
|
const SCAN_CYCLE_RESULT_PARTIAL_LABEL: &str = "partial";
|
||||||
const SCAN_CYCLE_RESULT_SUPERSEDED_LABEL: &str = "superseded";
|
const SCAN_CYCLE_RESULT_SUPERSEDED_LABEL: &str = "superseded";
|
||||||
const SCAN_CYCLE_RESULT_DEFERRED_LABEL: &str = "deferred";
|
|
||||||
|
|
||||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
|
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
|
||||||
pub enum ScanCyclePartialReason {
|
pub enum ScanCyclePartialReason {
|
||||||
@@ -1059,15 +1043,6 @@ pub struct ScannerBucketDriveResultSnapshot {
|
|||||||
pub count: u64,
|
pub count: u64,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
|
|
||||||
pub struct ScannerActiveBucketDriveSnapshot {
|
|
||||||
pub source: String,
|
|
||||||
pub bucket: String,
|
|
||||||
pub drive: String,
|
|
||||||
pub count: u64,
|
|
||||||
pub age_seconds: u64,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
|
#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
|
||||||
pub struct ScannerReplicationRepairSnapshot {
|
pub struct ScannerReplicationRepairSnapshot {
|
||||||
pub source: String,
|
pub source: String,
|
||||||
@@ -1410,8 +1385,6 @@ pub struct ScannerRuntimeDetailsReport {
|
|||||||
pub current_cycle_bucket_drive_results: Vec<ScannerBucketDriveResultSnapshot>,
|
pub current_cycle_bucket_drive_results: Vec<ScannerBucketDriveResultSnapshot>,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub last_cycle_bucket_drive_results: Vec<ScannerBucketDriveResultSnapshot>,
|
pub last_cycle_bucket_drive_results: Vec<ScannerBucketDriveResultSnapshot>,
|
||||||
#[serde(default)]
|
|
||||||
pub active_bucket_drive_scans: Vec<ScannerActiveBucketDriveSnapshot>,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
impl CurrentCycle {
|
impl CurrentCycle {
|
||||||
@@ -1426,18 +1399,31 @@ impl CurrentCycle {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// OTEL metric name constants for scanner metrics
|
/// OTEL metric name constants for scanner metrics
|
||||||
|
const OTEL_SCANNER_OBJECTS_SCANNED: &str = "rustfs_scanner_objects_scanned_total";
|
||||||
|
const OTEL_SCANNER_DIRECTORIES_SCANNED: &str = "rustfs_scanner_directories_scanned_total";
|
||||||
const OTEL_SCANNER_BUCKETS_SCANNED: &str = "rustfs_scanner_buckets_scanned_total";
|
const OTEL_SCANNER_BUCKETS_SCANNED: &str = "rustfs_scanner_buckets_scanned_total";
|
||||||
const OTEL_SCANNER_CYCLES: &str = "rustfs_scanner_cycles_total";
|
const OTEL_SCANNER_CYCLES: &str = "rustfs_scanner_cycles_total";
|
||||||
const OTEL_SCANNER_CYCLE_DURATION_SECONDS: &str = "rustfs_scanner_cycle_duration_seconds";
|
const OTEL_SCANNER_CYCLE_DURATION_SECONDS: &str = "rustfs_scanner_cycle_duration_seconds";
|
||||||
const OTEL_SCANNER_BUCKET_DRIVE_DURATION_SECONDS: &str = "rustfs_scanner_bucket_drive_duration_seconds";
|
const OTEL_SCANNER_BUCKET_DRIVE_DURATION_SECONDS: &str = "rustfs_scanner_bucket_drive_duration_seconds";
|
||||||
|
|
||||||
|
fn emit_otel_counter(metric: usize, count: u64) {
|
||||||
|
match Metric::from_index(metric) {
|
||||||
|
Some(Metric::ScanObject) => {
|
||||||
|
metrics::counter!(OTEL_SCANNER_OBJECTS_SCANNED).increment(count);
|
||||||
|
}
|
||||||
|
Some(Metric::ScanFolder) => {
|
||||||
|
metrics::counter!(OTEL_SCANNER_DIRECTORIES_SCANNED).increment(count);
|
||||||
|
}
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn scan_cycle_result_label(result: u8) -> &'static str {
|
fn scan_cycle_result_label(result: u8) -> &'static str {
|
||||||
match result {
|
match result {
|
||||||
SCAN_CYCLE_RESULT_SUCCESS => SCAN_CYCLE_RESULT_SUCCESS_LABEL,
|
SCAN_CYCLE_RESULT_SUCCESS => SCAN_CYCLE_RESULT_SUCCESS_LABEL,
|
||||||
SCAN_CYCLE_RESULT_ERROR => SCAN_CYCLE_RESULT_ERROR_LABEL,
|
SCAN_CYCLE_RESULT_ERROR => SCAN_CYCLE_RESULT_ERROR_LABEL,
|
||||||
SCAN_CYCLE_RESULT_PARTIAL => SCAN_CYCLE_RESULT_PARTIAL_LABEL,
|
SCAN_CYCLE_RESULT_PARTIAL => SCAN_CYCLE_RESULT_PARTIAL_LABEL,
|
||||||
SCAN_CYCLE_RESULT_SUPERSEDED => SCAN_CYCLE_RESULT_SUPERSEDED_LABEL,
|
SCAN_CYCLE_RESULT_SUPERSEDED => SCAN_CYCLE_RESULT_SUPERSEDED_LABEL,
|
||||||
SCAN_CYCLE_RESULT_DEFERRED => SCAN_CYCLE_RESULT_DEFERRED_LABEL,
|
|
||||||
_ => SCAN_CYCLE_RESULT_UNKNOWN_LABEL,
|
_ => SCAN_CYCLE_RESULT_UNKNOWN_LABEL,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1766,12 +1752,7 @@ pub fn emit_scan_cycle_superseded(duration: Duration) {
|
|||||||
metrics::counter!(OTEL_SCANNER_CYCLES, "result" => SCAN_CYCLE_RESULT_SUPERSEDED_LABEL).increment(1);
|
metrics::counter!(OTEL_SCANNER_CYCLES, "result" => SCAN_CYCLE_RESULT_SUPERSEDED_LABEL).increment(1);
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn emit_scan_cycle_deferred(duration: Duration) {
|
pub fn emit_scan_bucket_drive_complete(success: bool, bucket: &str, disk: &str, duration: Duration) {
|
||||||
global_metrics().record_scan_cycle_deferred(duration);
|
|
||||||
metrics::counter!(OTEL_SCANNER_CYCLES, "result" => SCAN_CYCLE_RESULT_DEFERRED_LABEL).increment(1);
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn emit_scan_bucket_drive_complete(_source: ScannerWorkSource, success: bool, bucket: &str, disk: &str, duration: Duration) {
|
|
||||||
let result = if success { "success" } else { "error" };
|
let result = if success { "success" } else { "error" };
|
||||||
global_metrics().record_scanner_bucket_drive_result(bucket, disk, result);
|
global_metrics().record_scanner_bucket_drive_result(bucket, disk, result);
|
||||||
metrics::counter!(
|
metrics::counter!(
|
||||||
@@ -1789,7 +1770,7 @@ pub fn emit_scan_bucket_drive_complete(_source: ScannerWorkSource, success: bool
|
|||||||
.record(duration.as_secs_f64());
|
.record(duration.as_secs_f64());
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn emit_scan_bucket_drive_partial(_source: ScannerWorkSource, bucket: &str, disk: &str, duration: Duration) {
|
pub fn emit_scan_bucket_drive_partial(bucket: &str, disk: &str, duration: Duration) {
|
||||||
global_metrics().record_scanner_bucket_drive_result(bucket, disk, SCAN_CYCLE_RESULT_PARTIAL_LABEL);
|
global_metrics().record_scanner_bucket_drive_result(bucket, disk, SCAN_CYCLE_RESULT_PARTIAL_LABEL);
|
||||||
metrics::counter!(
|
metrics::counter!(
|
||||||
OTEL_SCANNER_BUCKETS_SCANNED,
|
OTEL_SCANNER_BUCKETS_SCANNED,
|
||||||
@@ -1842,7 +1823,6 @@ impl Metrics {
|
|||||||
scanner_set_scans_active: AtomicU64::new(0),
|
scanner_set_scans_active: AtomicU64::new(0),
|
||||||
scanner_disk_bucket_scan_states: Mutex::new(HashMap::new()),
|
scanner_disk_bucket_scan_states: Mutex::new(HashMap::new()),
|
||||||
scanner_bucket_drive_results: Mutex::new(ScannerBucketDriveResults::default()),
|
scanner_bucket_drive_results: Mutex::new(ScannerBucketDriveResults::default()),
|
||||||
scanner_active_bucket_drive_scans: Mutex::new(HashMap::new()),
|
|
||||||
scanner_bucket_drive_result_clock: AtomicU64::new(0),
|
scanner_bucket_drive_result_clock: AtomicU64::new(0),
|
||||||
current_scan_cycle_bucket_drive_results_start: Mutex::new(HashMap::new()),
|
current_scan_cycle_bucket_drive_results_start: Mutex::new(HashMap::new()),
|
||||||
last_scan_cycle_bucket_drive_results: Mutex::new(Vec::new()),
|
last_scan_cycle_bucket_drive_results: Mutex::new(Vec::new()),
|
||||||
@@ -1972,6 +1952,7 @@ impl Metrics {
|
|||||||
let duration = SystemTime::now().duration_since(start).unwrap_or_default();
|
let duration = SystemTime::now().duration_since(start).unwrap_or_default();
|
||||||
global_metrics().operations[metric_idx].fetch_add(1, Ordering::Relaxed);
|
global_metrics().operations[metric_idx].fetch_add(1, Ordering::Relaxed);
|
||||||
global_metrics().record_source_work_for_metric(metric, 1);
|
global_metrics().record_source_work_for_metric(metric, 1);
|
||||||
|
emit_otel_counter(metric_idx, 1);
|
||||||
if metric_idx < Metric::LastRealtime as usize {
|
if metric_idx < Metric::LastRealtime as usize {
|
||||||
global_metrics().latency[metric_idx].add(duration);
|
global_metrics().latency[metric_idx].add(duration);
|
||||||
}
|
}
|
||||||
@@ -1987,6 +1968,7 @@ impl Metrics {
|
|||||||
let duration = SystemTime::now().duration_since(start).unwrap_or_default();
|
let duration = SystemTime::now().duration_since(start).unwrap_or_default();
|
||||||
global_metrics().operations[metric_idx].fetch_add(1, Ordering::Relaxed);
|
global_metrics().operations[metric_idx].fetch_add(1, Ordering::Relaxed);
|
||||||
global_metrics().record_source_work_for_metric(metric, 1);
|
global_metrics().record_source_work_for_metric(metric, 1);
|
||||||
|
emit_otel_counter(metric_idx, 1);
|
||||||
if metric_idx < Metric::LastRealtime as usize {
|
if metric_idx < Metric::LastRealtime as usize {
|
||||||
global_metrics().latency[metric_idx].add_size(duration, size);
|
global_metrics().latency[metric_idx].add_size(duration, size);
|
||||||
}
|
}
|
||||||
@@ -2002,6 +1984,7 @@ impl Metrics {
|
|||||||
let duration = SystemTime::now().duration_since(start).unwrap_or_default();
|
let duration = SystemTime::now().duration_since(start).unwrap_or_default();
|
||||||
global_metrics().operations[metric_idx].fetch_add(1, Ordering::Relaxed);
|
global_metrics().operations[metric_idx].fetch_add(1, Ordering::Relaxed);
|
||||||
global_metrics().record_source_work_for_metric(metric, 1);
|
global_metrics().record_source_work_for_metric(metric, 1);
|
||||||
|
emit_otel_counter(metric_idx, 1);
|
||||||
if metric_idx < Metric::LastRealtime as usize {
|
if metric_idx < Metric::LastRealtime as usize {
|
||||||
global_metrics().latency[metric_idx].add(duration);
|
global_metrics().latency[metric_idx].add(duration);
|
||||||
}
|
}
|
||||||
@@ -2019,6 +2002,7 @@ impl Metrics {
|
|||||||
let count = usize_to_u64_saturated(count);
|
let count = usize_to_u64_saturated(count);
|
||||||
global_metrics().operations[metric_idx].fetch_add(count, Ordering::Relaxed);
|
global_metrics().operations[metric_idx].fetch_add(count, Ordering::Relaxed);
|
||||||
global_metrics().record_source_work_for_metric(metric, count);
|
global_metrics().record_source_work_for_metric(metric, count);
|
||||||
|
emit_otel_counter(metric_idx, count);
|
||||||
if metric_idx < Metric::LastRealtime as usize {
|
if metric_idx < Metric::LastRealtime as usize {
|
||||||
global_metrics().latency[metric_idx].add(duration);
|
global_metrics().latency[metric_idx].add(duration);
|
||||||
}
|
}
|
||||||
@@ -2039,6 +2023,7 @@ impl Metrics {
|
|||||||
let duration = SystemTime::now().duration_since(start).unwrap_or_default();
|
let duration = SystemTime::now().duration_since(start).unwrap_or_default();
|
||||||
let metric_idx = Metric::Ilm as usize;
|
let metric_idx = Metric::Ilm as usize;
|
||||||
global_metrics().operations[metric_idx].fetch_add(versions, Ordering::Relaxed);
|
global_metrics().operations[metric_idx].fetch_add(versions, Ordering::Relaxed);
|
||||||
|
emit_otel_counter(metric_idx, versions);
|
||||||
global_metrics().actions[a_idx].fetch_add(versions, Ordering::Relaxed);
|
global_metrics().actions[a_idx].fetch_add(versions, Ordering::Relaxed);
|
||||||
global_metrics().actions_latency[a_idx].add(duration);
|
global_metrics().actions_latency[a_idx].add(duration);
|
||||||
})
|
})
|
||||||
@@ -2051,6 +2036,7 @@ impl Metrics {
|
|||||||
let metric_idx = metric as usize;
|
let metric_idx = metric as usize;
|
||||||
global_metrics().operations[metric_idx].fetch_add(1, Ordering::Relaxed);
|
global_metrics().operations[metric_idx].fetch_add(1, Ordering::Relaxed);
|
||||||
global_metrics().record_source_work_for_metric(metric, 1);
|
global_metrics().record_source_work_for_metric(metric, 1);
|
||||||
|
emit_otel_counter(metric_idx, 1);
|
||||||
if metric_idx < Metric::LastRealtime as usize {
|
if metric_idx < Metric::LastRealtime as usize {
|
||||||
global_metrics().latency[metric_idx].add(duration);
|
global_metrics().latency[metric_idx].add(duration);
|
||||||
}
|
}
|
||||||
@@ -2334,45 +2320,8 @@ impl Metrics {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn record_scan_bucket_drive_start(&self, source: ScannerWorkSource, bucket: &str, drive: &str) {
|
pub fn record_scan_bucket_drive_start(&self) {
|
||||||
self.operations[Metric::ScanBucketDriveStart as usize].fetch_add(1, Ordering::Relaxed);
|
self.operations[Metric::ScanBucketDriveStart as usize].fetch_add(1, Ordering::Relaxed);
|
||||||
if bucket.is_empty() || drive.is_empty() {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
let key = ScannerActiveBucketDriveKey {
|
|
||||||
source: source.as_str().to_string(),
|
|
||||||
bucket: bucket.to_string(),
|
|
||||||
drive: drive.to_string(),
|
|
||||||
};
|
|
||||||
let mut active = self
|
|
||||||
.scanner_active_bucket_drive_scans
|
|
||||||
.lock()
|
|
||||||
.unwrap_or_else(|poisoned| poisoned.into_inner());
|
|
||||||
active
|
|
||||||
.entry(key)
|
|
||||||
.and_modify(|value| value.count = value.count.saturating_add(1))
|
|
||||||
.or_insert(ScannerActiveBucketDriveValue {
|
|
||||||
count: 1,
|
|
||||||
started_at: Timestamp::now(),
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn record_scan_bucket_drive_end(&self, source: ScannerWorkSource, bucket: &str, drive: &str) {
|
|
||||||
let key = ScannerActiveBucketDriveKey {
|
|
||||||
source: source.as_str().to_string(),
|
|
||||||
bucket: bucket.to_string(),
|
|
||||||
drive: drive.to_string(),
|
|
||||||
};
|
|
||||||
let mut active = self
|
|
||||||
.scanner_active_bucket_drive_scans
|
|
||||||
.lock()
|
|
||||||
.unwrap_or_else(|poisoned| poisoned.into_inner());
|
|
||||||
if let Some(value) = active.get_mut(&key) {
|
|
||||||
value.count = value.count.saturating_sub(1);
|
|
||||||
if value.count == 0 {
|
|
||||||
active.remove(&key);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn record_scan_bucket_drive_failure(&self) {
|
pub fn record_scan_bucket_drive_failure(&self) {
|
||||||
@@ -2600,17 +2549,6 @@ impl Metrics {
|
|||||||
.store(duration_millis_saturated(duration), Ordering::Relaxed);
|
.store(duration_millis_saturated(duration), Ordering::Relaxed);
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn record_scan_cycle_deferred(&self, duration: Duration) {
|
|
||||||
self.record_scanner_cycle_end_time();
|
|
||||||
self.last_scan_cycle_result
|
|
||||||
.store(SCAN_CYCLE_RESULT_DEFERRED, Ordering::Relaxed);
|
|
||||||
self.last_scan_cycle_partial_reason
|
|
||||||
.store(ScanCyclePartialReason::Unknown as u8, Ordering::Relaxed);
|
|
||||||
self.last_scan_cycle_partial_source.store(0, Ordering::Relaxed);
|
|
||||||
self.last_scan_cycle_duration_millis
|
|
||||||
.store(duration_millis_saturated(duration), Ordering::Relaxed);
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn record_scan_cycle_partial(&self, duration: Duration, reason: ScanCyclePartialReason) {
|
pub fn record_scan_cycle_partial(&self, duration: Duration, reason: ScanCyclePartialReason) {
|
||||||
self.record_scan_cycle_partial_with_source(duration, reason, None);
|
self.record_scan_cycle_partial_with_source(duration, reason, None);
|
||||||
}
|
}
|
||||||
@@ -2845,26 +2783,6 @@ impl Metrics {
|
|||||||
} else {
|
} else {
|
||||||
Vec::new()
|
Vec::new()
|
||||||
};
|
};
|
||||||
let now = Timestamp::now();
|
|
||||||
let mut active_bucket_drive_scans = self
|
|
||||||
.scanner_active_bucket_drive_scans
|
|
||||||
.lock()
|
|
||||||
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
|
||||||
.iter()
|
|
||||||
.map(|(key, value)| ScannerActiveBucketDriveSnapshot {
|
|
||||||
source: key.source.clone(),
|
|
||||||
bucket: key.bucket.clone(),
|
|
||||||
drive: key.drive.clone(),
|
|
||||||
count: value.count,
|
|
||||||
age_seconds: timestamp_elapsed_seconds_since(now, value.started_at),
|
|
||||||
})
|
|
||||||
.collect::<Vec<_>>();
|
|
||||||
active_bucket_drive_scans.sort_by(|left, right| {
|
|
||||||
left.source
|
|
||||||
.cmp(&right.source)
|
|
||||||
.then_with(|| left.bucket.cmp(&right.bucket))
|
|
||||||
.then_with(|| left.drive.cmp(&right.drive))
|
|
||||||
});
|
|
||||||
ScannerRuntimeDetailsReport {
|
ScannerRuntimeDetailsReport {
|
||||||
disk_bucket_scan_states: self.scanner_disk_bucket_scan_state_snapshots(),
|
disk_bucket_scan_states: self.scanner_disk_bucket_scan_state_snapshots(),
|
||||||
bucket_drive_results: self.scanner_bucket_drive_result_counter_snapshots(),
|
bucket_drive_results: self.scanner_bucket_drive_result_counter_snapshots(),
|
||||||
@@ -2874,7 +2792,6 @@ impl Metrics {
|
|||||||
.lock()
|
.lock()
|
||||||
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||||
.clone(),
|
.clone(),
|
||||||
active_bucket_drive_scans,
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -4347,21 +4264,6 @@ mod tests {
|
|||||||
assert_eq!(report.partial_cycles, 0);
|
assert_eq!(report.partial_cycles, 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn report_tracks_deferred_cycle_without_failed_increment() {
|
|
||||||
let metrics = Metrics::new();
|
|
||||||
metrics.record_scan_cycle_deferred(Duration::from_millis(250));
|
|
||||||
|
|
||||||
let report = metrics.report().await;
|
|
||||||
|
|
||||||
assert_eq!(report.last_cycle_result, SCAN_CYCLE_RESULT_DEFERRED_LABEL);
|
|
||||||
assert_eq!(report.last_cycle_result_code, u64::from(SCAN_CYCLE_RESULT_DEFERRED));
|
|
||||||
assert_eq!(report.last_cycle_duration_seconds, 0.25);
|
|
||||||
assert_eq!(report.failed_cycles, 0);
|
|
||||||
assert_eq!(report.superseded_cycles, 0);
|
|
||||||
assert_eq!(report.partial_cycles, 0);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn report_tracks_successful_scan_cycle_without_failed_increment() {
|
async fn report_tracks_successful_scan_cycle_without_failed_increment() {
|
||||||
let metrics = Metrics::new();
|
let metrics = Metrics::new();
|
||||||
@@ -4455,7 +4357,7 @@ mod tests {
|
|||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn report_includes_bucket_drive_scan_starts() {
|
async fn report_includes_bucket_drive_scan_starts() {
|
||||||
let metrics = Metrics::new();
|
let metrics = Metrics::new();
|
||||||
metrics.record_scan_bucket_drive_start(ScannerWorkSource::Usage, "bucket-a", "/mnt/data/1");
|
metrics.record_scan_bucket_drive_start();
|
||||||
metrics.record_scan_bucket_drive_failure();
|
metrics.record_scan_bucket_drive_failure();
|
||||||
|
|
||||||
let report = metrics.report().await;
|
let report = metrics.report().await;
|
||||||
@@ -4464,27 +4366,6 @@ mod tests {
|
|||||||
assert_eq!(report.life_time_ops.get("scan_bucket_drive_failure"), Some(&1));
|
assert_eq!(report.life_time_ops.get("scan_bucket_drive_failure"), Some(&1));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn active_bucket_drive_snapshot_is_structured_and_retired_on_end() {
|
|
||||||
let metrics = Metrics::new();
|
|
||||||
metrics.record_scan_bucket_drive_start(ScannerWorkSource::Usage, "bucket-a", "/mnt/data/1");
|
|
||||||
metrics.record_scan_bucket_drive_start(ScannerWorkSource::Usage, "bucket-a", "/mnt/data/1");
|
|
||||||
let active = metrics.scanner_runtime_details_report().active_bucket_drive_scans;
|
|
||||||
assert_eq!(active.len(), 1);
|
|
||||||
assert_eq!(active[0].source, ScannerWorkSource::Usage.as_str());
|
|
||||||
assert_eq!(active[0].bucket, "bucket-a");
|
|
||||||
assert_eq!(active[0].drive, "/mnt/data/1");
|
|
||||||
assert_eq!(active[0].count, 2);
|
|
||||||
|
|
||||||
metrics.record_scan_bucket_drive_end(ScannerWorkSource::Usage, "bucket-a", "/mnt/data/1");
|
|
||||||
assert_eq!(metrics.scanner_runtime_details_report().active_bucket_drive_scans[0].count, 1);
|
|
||||||
metrics.record_scan_bucket_drive_end(ScannerWorkSource::Usage, "bucket-a", "/mnt/data/1");
|
|
||||||
assert!(metrics.scanner_runtime_details_report().active_bucket_drive_scans.is_empty());
|
|
||||||
|
|
||||||
metrics.record_scan_bucket_drive_start(ScannerWorkSource::Usage, "", "/mnt/data/1");
|
|
||||||
assert!(metrics.scanner_runtime_details_report().active_bucket_drive_scans.is_empty());
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn report_includes_structured_bucket_drive_results() {
|
async fn report_includes_structured_bucket_drive_results() {
|
||||||
let metrics = Metrics::new();
|
let metrics = Metrics::new();
|
||||||
|
|||||||
@@ -1,287 +0,0 @@
|
|||||||
// Copyright 2024 RustFS Team
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
//! Mission Repair Feed (MRF) intent channel.
|
|
||||||
//!
|
|
||||||
//! Producers on error paths (read decode failure, scanner metadata
|
|
||||||
//! corruption, partial-write recovery) hand a lightweight [`MrfIntent`] to the
|
|
||||||
//! heal crate through a global bounded channel. Delivery is strictly
|
|
||||||
//! non-blocking: `try_send_mrf_intent` never awaits and drops the intent
|
|
||||||
//! (counting it) when the channel is full or uninitialized — losing one heal
|
|
||||||
//! hint is always preferred over stalling an IO path. Durable replay of
|
|
||||||
//! unconsumed intents is the consumer's job (see `rustfs-heal`
|
|
||||||
//! `heal::mrf_queue`), mirroring MinIO's `.heal/mrf/list.bin`.
|
|
||||||
|
|
||||||
use std::sync::{
|
|
||||||
Arc, OnceLock,
|
|
||||||
atomic::{AtomicBool, Ordering},
|
|
||||||
};
|
|
||||||
use tokio::sync::mpsc;
|
|
||||||
use uuid::Uuid;
|
|
||||||
|
|
||||||
/// Bounded capacity of the global MRF channel. Backpressure is resolved by
|
|
||||||
/// dropping (and counting) intents, never by blocking the producer.
|
|
||||||
const MRF_CHANNEL_CAPACITY: usize = 8192;
|
|
||||||
|
|
||||||
/// Why an intent was produced. Drives the heal priority mapping on the
|
|
||||||
/// consumer side (DecodeFailure -> Urgent, MetadataCorruption -> High,
|
|
||||||
/// PartialWrite -> Normal).
|
|
||||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
|
||||||
pub enum MrfKind {
|
|
||||||
/// Erasure decode failed while serving a read (read path).
|
|
||||||
DecodeFailure,
|
|
||||||
/// Scanner classified object metadata as corrupt.
|
|
||||||
MetadataCorruption,
|
|
||||||
/// A write left the object with fewer committed shards than the set size.
|
|
||||||
PartialWrite,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl MrfKind {
|
|
||||||
pub const fn as_str(self) -> &'static str {
|
|
||||||
match self {
|
|
||||||
MrfKind::DecodeFailure => "decode-failure",
|
|
||||||
MrfKind::MetadataCorruption => "metadata-corruption",
|
|
||||||
MrfKind::PartialWrite => "partial-write",
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// One repair intent. Kept deliberately small so the in-memory queue and the
|
|
||||||
/// journal stay bounded; `bucket`/`object` are `Arc<str>` so re-arming an
|
|
||||||
/// intent never re-allocates the strings.
|
|
||||||
#[derive(Clone, Debug)]
|
|
||||||
pub struct MrfIntent {
|
|
||||||
pub bucket: Arc<str>,
|
|
||||||
pub object: Arc<str>,
|
|
||||||
/// Version the intent targets, as raw UUID bytes.
|
|
||||||
pub version_id: Option<[u8; 16]>,
|
|
||||||
pub kind: MrfKind,
|
|
||||||
pub enqueued_at_ms: u64,
|
|
||||||
/// Times this intent has already been offered to the heal manager.
|
|
||||||
/// Dropped by the consumer once it reaches `MRF_MAX_ATTEMPTS`.
|
|
||||||
pub attempts: u8,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Consumer-side retry ceiling before an intent is given up on.
|
|
||||||
pub const MRF_MAX_ATTEMPTS: u8 = 3;
|
|
||||||
|
|
||||||
impl MrfIntent {
|
|
||||||
/// Rough in-memory footprint used by the queue's byte budget.
|
|
||||||
pub fn estimated_bytes(&self) -> usize {
|
|
||||||
// Struct + strings + version bytes; buckets and objects are usually
|
|
||||||
// far below this bound, so rounding up keeps the budget conservative.
|
|
||||||
64 + self.bucket.len() + self.object.len()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
static GLOBAL_MRF_SENDER: OnceLock<mpsc::Sender<MrfIntent>> = OnceLock::new();
|
|
||||||
|
|
||||||
/// Delivery kill-switch, set from `RUSTFS_HEAL_MRF_ENABLE`. Producers check
|
|
||||||
/// this before touching the channel so the disabled path stays allocation- and
|
|
||||||
/// sync-free.
|
|
||||||
static MRF_DELIVERY_ENABLED: AtomicBool = AtomicBool::new(true);
|
|
||||||
|
|
||||||
/// Override delivery (used at heal-runtime startup from configuration).
|
|
||||||
pub fn set_mrf_delivery_enabled(enabled: bool) {
|
|
||||||
MRF_DELIVERY_ENABLED.store(enabled, Ordering::Relaxed);
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Whether producers currently deliver intents.
|
|
||||||
pub fn mrf_delivery_enabled() -> bool {
|
|
||||||
MRF_DELIVERY_ENABLED.load(Ordering::Relaxed)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Create the global MRF channel and return the consumer half. Fails if the
|
|
||||||
/// channel is already initialized (the heal runtime is a singleton).
|
|
||||||
pub fn init_mrf_channel() -> Result<mpsc::Receiver<MrfIntent>, &'static str> {
|
|
||||||
let (sender, receiver) = mpsc::channel(MRF_CHANNEL_CAPACITY);
|
|
||||||
GLOBAL_MRF_SENDER
|
|
||||||
.set(sender)
|
|
||||||
.map_err(|_| "MRF channel sender already initialized")?;
|
|
||||||
Ok(receiver)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Best-effort, non-blocking intent delivery from an error path.
|
|
||||||
///
|
|
||||||
/// Returns `true` when the intent was accepted into the channel. `false`
|
|
||||||
/// means the intent was dropped (feature disabled, channel not yet
|
|
||||||
/// initialized, or channel full) — callers must not retry or await; the
|
|
||||||
/// existing read-repair / scanner heal paths remain the safety net.
|
|
||||||
///
|
|
||||||
/// This runs on IO error paths, so it stays synchronous and cheap: one
|
|
||||||
/// bounded allocation for the two `Arc<str>` handles plus the channel slot.
|
|
||||||
pub fn try_send_mrf_intent(kind: MrfKind, bucket: &str, object: &str, version_id: Option<Uuid>) -> bool {
|
|
||||||
if !mrf_delivery_enabled() {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
let Some(sender) = GLOBAL_MRF_SENDER.get() else {
|
|
||||||
return false;
|
|
||||||
};
|
|
||||||
let intent = MrfIntent {
|
|
||||||
bucket: Arc::from(bucket),
|
|
||||||
object: Arc::from(object),
|
|
||||||
version_id: version_id.map(|vid| *vid.as_bytes()),
|
|
||||||
kind,
|
|
||||||
enqueued_at_ms: unix_now_ms(),
|
|
||||||
attempts: 0,
|
|
||||||
};
|
|
||||||
sender.try_send(intent).is_ok()
|
|
||||||
}
|
|
||||||
|
|
||||||
fn unix_now_ms() -> u64 {
|
|
||||||
// Kept trivial: the timestamp is diagnostic metadata only; wall-clock
|
|
||||||
// failure would be a bug rather than something to handle here.
|
|
||||||
std::time::SystemTime::now()
|
|
||||||
.duration_since(std::time::UNIX_EPOCH)
|
|
||||||
.map(|d| d.as_millis() as u64)
|
|
||||||
.unwrap_or(0)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// A repair the MRF consumer landed, fanned out so retry ledgers can drop
|
|
||||||
/// entries the journal no longer tracks (backlog#1894 axis B). The payload
|
|
||||||
/// mirrors the intent identity so consumers match without re-parsing.
|
|
||||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
||||||
pub struct MrfRepairedEvent {
|
|
||||||
pub bucket: Arc<str>,
|
|
||||||
pub object: Arc<str>,
|
|
||||||
pub version_id: Option<[u8; 16]>,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Bound on the repaired-event backlog. Notices are best-effort hints; when
|
|
||||||
/// the ring is full the oldest are dropped and the affected ledger entries
|
|
||||||
/// simply expire through their own attempts/age limits.
|
|
||||||
const MRF_REPAIRED_EVENT_CAP: usize = 4096;
|
|
||||||
|
|
||||||
static MRF_REPAIRED_EVENTS: OnceLock<std::sync::Mutex<std::collections::VecDeque<MrfRepairedEvent>>> = OnceLock::new();
|
|
||||||
|
|
||||||
/// Record that the MRF consumer landed a repair. Never blocks: the critical
|
|
||||||
/// section is a deque push under a std mutex.
|
|
||||||
pub fn note_mrf_repaired(bucket: &str, object: &str, version_id: Option<[u8; 16]>) {
|
|
||||||
let registry = MRF_REPAIRED_EVENTS.get_or_init(|| std::sync::Mutex::new(std::collections::VecDeque::new()));
|
|
||||||
let Ok(mut events) = registry.lock() else {
|
|
||||||
return;
|
|
||||||
};
|
|
||||||
if events.len() >= MRF_REPAIRED_EVENT_CAP {
|
|
||||||
events.pop_front();
|
|
||||||
}
|
|
||||||
events.push_back(MrfRepairedEvent {
|
|
||||||
bucket: Arc::from(bucket),
|
|
||||||
object: Arc::from(object),
|
|
||||||
version_id,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Take the repair notices recorded for `bucket`, leaving other buckets'
|
|
||||||
/// notices in place for their own scanners.
|
|
||||||
pub fn take_mrf_repaired_events_for(bucket: &str) -> Vec<MrfRepairedEvent> {
|
|
||||||
let Some(registry) = MRF_REPAIRED_EVENTS.get() else {
|
|
||||||
return Vec::new();
|
|
||||||
};
|
|
||||||
let Ok(mut events) = registry.lock() else {
|
|
||||||
return Vec::new();
|
|
||||||
};
|
|
||||||
let mut taken = Vec::new();
|
|
||||||
let mut retained = std::collections::VecDeque::with_capacity(events.len());
|
|
||||||
while let Some(event) = events.pop_front() {
|
|
||||||
if event.bucket.as_ref() == bucket {
|
|
||||||
taken.push(event);
|
|
||||||
} else {
|
|
||||||
retained.push_back(event);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
*events = retained;
|
|
||||||
taken
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(test)]
|
|
||||||
mod tests {
|
|
||||||
use super::*;
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn intents_estimate_is_conservative() {
|
|
||||||
let intent = MrfIntent {
|
|
||||||
bucket: Arc::from("bucket"),
|
|
||||||
object: Arc::from("object"),
|
|
||||||
version_id: Some([0u8; 16]),
|
|
||||||
kind: MrfKind::DecodeFailure,
|
|
||||||
enqueued_at_ms: 0,
|
|
||||||
attempts: 0,
|
|
||||||
};
|
|
||||||
assert!(intent.estimated_bytes() >= intent.bucket.len() + intent.object.len());
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn try_send_delivers_and_respects_capacity() {
|
|
||||||
let mut receiver = init_mrf_channel().expect("first initialization should succeed");
|
|
||||||
assert!(init_mrf_channel().is_err(), "double initialization must fail");
|
|
||||||
|
|
||||||
assert!(try_send_mrf_intent(MrfKind::DecodeFailure, "b", "o", Some(Uuid::nil())));
|
|
||||||
let intent = receiver.recv().await.expect("intent should arrive");
|
|
||||||
assert_eq!(intent.kind, MrfKind::DecodeFailure);
|
|
||||||
assert_eq!(intent.bucket.as_ref(), "b");
|
|
||||||
|
|
||||||
// Disable delivery: producers become no-ops.
|
|
||||||
set_mrf_delivery_enabled(false);
|
|
||||||
assert!(!try_send_mrf_intent(MrfKind::PartialWrite, "b", "o", None));
|
|
||||||
set_mrf_delivery_enabled(true);
|
|
||||||
|
|
||||||
// Fill the bounded channel past capacity: excess intents are dropped,
|
|
||||||
// never blocking.
|
|
||||||
let mut accepted = 0;
|
|
||||||
for _ in 0..(MRF_CHANNEL_CAPACITY + 64) {
|
|
||||||
if try_send_mrf_intent(MrfKind::PartialWrite, "b", "o", None) {
|
|
||||||
accepted += 1;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
assert_eq!(accepted, MRF_CHANNEL_CAPACITY);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn try_send_without_channel_is_false() {
|
|
||||||
// This test may run after the tokio test above in the same process;
|
|
||||||
// the singleton semantics make a clean "uninitialized" case hard, so
|
|
||||||
// assert the flag-off behavior only.
|
|
||||||
set_mrf_delivery_enabled(false);
|
|
||||||
assert!(!try_send_mrf_intent(MrfKind::MetadataCorruption, "b", "o", None));
|
|
||||||
set_mrf_delivery_enabled(true);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn repaired_events_take_is_bucket_scoped_and_cap_bounded() {
|
|
||||||
// Distinct buckets keep their notices until their own scanner takes
|
|
||||||
// them; a take for one bucket leaves the others' notices in place.
|
|
||||||
note_mrf_repaired("bucket-a", "object-1", None);
|
|
||||||
note_mrf_repaired("bucket-b", "object-2", None);
|
|
||||||
note_mrf_repaired("bucket-a", "object-3", None);
|
|
||||||
|
|
||||||
let taken_a = take_mrf_repaired_events_for("bucket-a");
|
|
||||||
assert_eq!(taken_a.len(), 2);
|
|
||||||
assert_eq!(taken_a[0].object.as_ref(), "object-1");
|
|
||||||
assert_eq!(taken_a[1].object.as_ref(), "object-3");
|
|
||||||
assert!(take_mrf_repaired_events_for("bucket-a").is_empty(), "take is destructive per bucket");
|
|
||||||
|
|
||||||
let taken_b = take_mrf_repaired_events_for("bucket-b");
|
|
||||||
assert_eq!(taken_b.len(), 1);
|
|
||||||
assert_eq!(taken_b[0].object.as_ref(), "object-2");
|
|
||||||
|
|
||||||
// Cap bound: flooding the ring drops the oldest notices rather than
|
|
||||||
// growing unbounded.
|
|
||||||
for i in 0..=(MRF_REPAIRED_EVENT_CAP + 8) {
|
|
||||||
note_mrf_repaired("flood-bucket", &format!("object-{i}"), None);
|
|
||||||
}
|
|
||||||
let flooded = take_mrf_repaired_events_for("flood-bucket");
|
|
||||||
assert_eq!(flooded.len(), MRF_REPAIRED_EVENT_CAP);
|
|
||||||
assert_eq!(flooded[0].object.as_ref(), "object-9", "the oldest notices past the cap are dropped");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,333 +0,0 @@
|
|||||||
// Copyright 2024 RustFS Team
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
use smallvec::SmallVec;
|
|
||||||
use std::{
|
|
||||||
sync::{
|
|
||||||
Arc, OnceLock,
|
|
||||||
atomic::{AtomicUsize, Ordering},
|
|
||||||
},
|
|
||||||
time::{Duration, SystemTime},
|
|
||||||
};
|
|
||||||
use tokio::sync::broadcast;
|
|
||||||
|
|
||||||
const DEFAULT_TRACE_BUS_CAPACITY: usize = 1024;
|
|
||||||
const TRACE_ATTR_INLINE_CAPACITY: usize = 8;
|
|
||||||
|
|
||||||
static GLOBAL_TRACE_BUS: OnceLock<TraceBus> = OnceLock::new();
|
|
||||||
|
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
||||||
pub enum TraceKind {
|
|
||||||
Heal,
|
|
||||||
Scanner,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl TraceKind {
|
|
||||||
pub const fn as_str(self) -> &'static str {
|
|
||||||
match self {
|
|
||||||
Self::Heal => "heal",
|
|
||||||
Self::Scanner => "scanner",
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
||||||
pub enum TraceFunc {
|
|
||||||
HealTask,
|
|
||||||
HealBucket,
|
|
||||||
HealObject,
|
|
||||||
HealCheckAbandonedParts,
|
|
||||||
HealErasureSetPage,
|
|
||||||
ScannerFolder,
|
|
||||||
ScannerIlmAction,
|
|
||||||
ScannerHealCandidate,
|
|
||||||
Dropped,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl TraceFunc {
|
|
||||||
pub const fn as_str(self) -> &'static str {
|
|
||||||
match self {
|
|
||||||
Self::HealTask => "heal.Task",
|
|
||||||
Self::HealBucket => "heal.Bucket",
|
|
||||||
Self::HealObject => "heal.Object",
|
|
||||||
Self::HealCheckAbandonedParts => "heal.CheckAbandonedParts",
|
|
||||||
Self::HealErasureSetPage => "heal.ErasureSetPage",
|
|
||||||
Self::ScannerFolder => "scanner.Folder",
|
|
||||||
Self::ScannerIlmAction => "scanner.IlmAction",
|
|
||||||
Self::ScannerHealCandidate => "scanner.HealCandidate",
|
|
||||||
Self::Dropped => "trace.Dropped",
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
||||||
pub enum TraceVal {
|
|
||||||
Bool(bool),
|
|
||||||
U64(u64),
|
|
||||||
I64(i64),
|
|
||||||
Str(Arc<str>),
|
|
||||||
}
|
|
||||||
|
|
||||||
impl From<bool> for TraceVal {
|
|
||||||
fn from(value: bool) -> Self {
|
|
||||||
Self::Bool(value)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl From<u64> for TraceVal {
|
|
||||||
fn from(value: u64) -> Self {
|
|
||||||
Self::U64(value)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl From<i64> for TraceVal {
|
|
||||||
fn from(value: i64) -> Self {
|
|
||||||
Self::I64(value)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl From<&str> for TraceVal {
|
|
||||||
fn from(value: &str) -> Self {
|
|
||||||
Self::Str(Arc::from(value))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl From<String> for TraceVal {
|
|
||||||
fn from(value: String) -> Self {
|
|
||||||
Self::Str(Arc::from(value))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
||||||
pub struct TraceAttr {
|
|
||||||
pub key: &'static str,
|
|
||||||
pub value: TraceVal,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
||||||
pub struct TraceEvent {
|
|
||||||
pub kind: TraceKind,
|
|
||||||
pub func: TraceFunc,
|
|
||||||
pub time: SystemTime,
|
|
||||||
pub bucket: Option<Arc<str>>,
|
|
||||||
pub object: Option<Arc<str>>,
|
|
||||||
pub duration: Duration,
|
|
||||||
pub bytes: u64,
|
|
||||||
pub attrs: SmallVec<[TraceAttr; TRACE_ATTR_INLINE_CAPACITY]>,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl TraceEvent {
|
|
||||||
pub fn new(kind: TraceKind, func: TraceFunc) -> Self {
|
|
||||||
Self {
|
|
||||||
kind,
|
|
||||||
func,
|
|
||||||
time: SystemTime::now(),
|
|
||||||
bucket: None,
|
|
||||||
object: None,
|
|
||||||
duration: Duration::ZERO,
|
|
||||||
bytes: 0,
|
|
||||||
attrs: SmallVec::new(),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn with_bucket(mut self, bucket: impl Into<Arc<str>>) -> Self {
|
|
||||||
self.bucket = Some(bucket.into());
|
|
||||||
self
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn with_object(mut self, object: impl Into<Arc<str>>) -> Self {
|
|
||||||
self.object = Some(object.into());
|
|
||||||
self
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn with_duration(mut self, duration: Duration) -> Self {
|
|
||||||
self.duration = duration;
|
|
||||||
self
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn with_bytes(mut self, bytes: u64) -> Self {
|
|
||||||
self.bytes = bytes;
|
|
||||||
self
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn with_attr(mut self, key: &'static str, value: impl Into<TraceVal>) -> Self {
|
|
||||||
self.attrs.push(TraceAttr {
|
|
||||||
key,
|
|
||||||
value: value.into(),
|
|
||||||
});
|
|
||||||
self
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug)]
|
|
||||||
pub struct TraceBus {
|
|
||||||
sender: broadcast::Sender<Arc<TraceEvent>>,
|
|
||||||
subscriber_count: Arc<AtomicUsize>,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl TraceBus {
|
|
||||||
pub fn new(capacity: usize) -> Self {
|
|
||||||
let capacity = capacity.max(1);
|
|
||||||
let (sender, _receiver) = broadcast::channel(capacity);
|
|
||||||
Self {
|
|
||||||
sender,
|
|
||||||
subscriber_count: Arc::new(AtomicUsize::new(0)),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn subscriber_count(&self) -> usize {
|
|
||||||
self.subscriber_count.load(Ordering::Acquire)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn subscribe(&self) -> TraceSubscription {
|
|
||||||
let receiver = self.sender.subscribe();
|
|
||||||
self.subscriber_count.fetch_add(1, Ordering::AcqRel);
|
|
||||||
TraceSubscription {
|
|
||||||
receiver,
|
|
||||||
subscriber_count: Arc::clone(&self.subscriber_count),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn emit(&self, build: impl FnOnce() -> TraceEvent) -> bool {
|
|
||||||
if self.subscriber_count() == 0 {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
self.sender.send(Arc::new(build())).is_ok()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Default for TraceBus {
|
|
||||||
fn default() -> Self {
|
|
||||||
Self::new(DEFAULT_TRACE_BUS_CAPACITY)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug)]
|
|
||||||
pub struct TraceSubscription {
|
|
||||||
receiver: broadcast::Receiver<Arc<TraceEvent>>,
|
|
||||||
subscriber_count: Arc<AtomicUsize>,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl TraceSubscription {
|
|
||||||
pub async fn recv(&mut self) -> Result<Arc<TraceEvent>, broadcast::error::RecvError> {
|
|
||||||
self.receiver.recv().await
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn try_recv(&mut self) -> Result<Arc<TraceEvent>, broadcast::error::TryRecvError> {
|
|
||||||
self.receiver.try_recv()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Drop for TraceSubscription {
|
|
||||||
fn drop(&mut self) {
|
|
||||||
self.subscriber_count.fetch_sub(1, Ordering::AcqRel);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn global_trace_bus() -> &'static TraceBus {
|
|
||||||
GLOBAL_TRACE_BUS.get_or_init(TraceBus::default)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn subscribe_trace_events() -> TraceSubscription {
|
|
||||||
global_trace_bus().subscribe()
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn trace_emit(build: impl FnOnce() -> TraceEvent) -> bool {
|
|
||||||
global_trace_bus().emit(build)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn trace_subscriber_count() -> usize {
|
|
||||||
global_trace_bus().subscriber_count()
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(test)]
|
|
||||||
mod tests {
|
|
||||||
use super::*;
|
|
||||||
use std::sync::atomic::AtomicUsize;
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn trace_emit_skips_builder_without_subscribers() {
|
|
||||||
let bus = TraceBus::new(4);
|
|
||||||
let built = AtomicUsize::new(0);
|
|
||||||
|
|
||||||
let sent = bus.emit(|| {
|
|
||||||
built.fetch_add(1, Ordering::Relaxed);
|
|
||||||
TraceEvent::new(TraceKind::Heal, TraceFunc::HealTask)
|
|
||||||
});
|
|
||||||
|
|
||||||
assert!(!sent);
|
|
||||||
assert_eq!(built.load(Ordering::Relaxed), 0);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn trace_subscriber_receives_event() {
|
|
||||||
let bus = TraceBus::new(4);
|
|
||||||
let mut subscription = bus.subscribe();
|
|
||||||
|
|
||||||
assert!(bus.emit(|| {
|
|
||||||
TraceEvent::new(TraceKind::Heal, TraceFunc::HealObject)
|
|
||||||
.with_bucket("bucket")
|
|
||||||
.with_object("object")
|
|
||||||
.with_duration(Duration::from_millis(7))
|
|
||||||
.with_bytes(11)
|
|
||||||
.with_attr("dry", true)
|
|
||||||
}));
|
|
||||||
|
|
||||||
let event = subscription
|
|
||||||
.recv()
|
|
||||||
.await
|
|
||||||
.expect("subscriber should receive emitted trace event");
|
|
||||||
|
|
||||||
assert_eq!(event.kind, TraceKind::Heal);
|
|
||||||
assert_eq!(event.func, TraceFunc::HealObject);
|
|
||||||
assert_eq!(event.bucket.as_deref(), Some("bucket"));
|
|
||||||
assert_eq!(event.object.as_deref(), Some("object"));
|
|
||||||
assert_eq!(event.duration, Duration::from_millis(7));
|
|
||||||
assert_eq!(event.bytes, 11);
|
|
||||||
assert_eq!(
|
|
||||||
event.attrs.as_slice(),
|
|
||||||
&[TraceAttr {
|
|
||||||
key: "dry",
|
|
||||||
value: TraceVal::Bool(true)
|
|
||||||
}]
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn trace_subscription_drop_decrements_count() {
|
|
||||||
let bus = TraceBus::new(4);
|
|
||||||
let subscription = bus.subscribe();
|
|
||||||
|
|
||||||
assert_eq!(bus.subscriber_count(), 1);
|
|
||||||
drop(subscription);
|
|
||||||
assert_eq!(bus.subscriber_count(), 0);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn lagged_subscriber_drops_events_without_blocking_publishers() {
|
|
||||||
let bus = TraceBus::new(2);
|
|
||||||
let mut subscription = bus.subscribe();
|
|
||||||
|
|
||||||
for index in 0_u64..4 {
|
|
||||||
assert!(bus.emit(|| { TraceEvent::new(TraceKind::Scanner, TraceFunc::ScannerFolder).with_attr("index", index) }));
|
|
||||||
}
|
|
||||||
|
|
||||||
let err = subscription
|
|
||||||
.recv()
|
|
||||||
.await
|
|
||||||
.expect_err("receiver should observe lag instead of blocking publishers");
|
|
||||||
assert!(matches!(err, broadcast::error::RecvError::Lagged(_)));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -14,8 +14,9 @@
|
|||||||
|
|
||||||
//! Shared backpressure policy type.
|
//! Shared backpressure policy type.
|
||||||
//!
|
//!
|
||||||
//! This module only carries the watermark policy; the admission primitive it
|
//! The runtime backpressure implementation (byte-watermark pipes and
|
||||||
//! projects into lives in `rustfs-io-core`.
|
//! monitors) lives in `rustfs/src/storage/backpressure.rs`; this module only
|
||||||
|
//! carries the watermark policy type that implementation shares.
|
||||||
|
|
||||||
use rustfs_io_core::BackpressureConfig as CoreBackpressureConfig;
|
use rustfs_io_core::BackpressureConfig as CoreBackpressureConfig;
|
||||||
|
|
||||||
|
|||||||
@@ -115,15 +115,6 @@ Current guidance:
|
|||||||
- enables KMS readiness enforcement for `/health/ready`.
|
- enables KMS readiness enforcement for `/health/ready`.
|
||||||
- default is `false`.
|
- default is `false`.
|
||||||
|
|
||||||
## Object lock admission environment variables
|
|
||||||
|
|
||||||
- `RUSTFS_PUT_COMMIT_NAMESPACE_LOCK_ACQUIRE_TIMEOUT_MS`
|
|
||||||
- experimental same-object PUT commit namespace-lock admission budget.
|
|
||||||
- default is `0`, which disables this override and keeps `RUSTFS_OBJECT_LOCK_ACQUIRE_TIMEOUT` behavior.
|
|
||||||
- when set, only `put_object_commit` write-lock acquisition is bounded by this millisecond budget; other namespace lock users keep the global object-lock timeout.
|
|
||||||
- timeout returns S3 `SlowDown`, so clients should use normal SDK retry handling.
|
|
||||||
- this is not a fdatasync or group-commit switch. Track fdatasync batching separately with `rustfs_s3_put_object_rename_fdatasync_batch_files`.
|
|
||||||
|
|
||||||
## Drive timeout environment variables
|
## Drive timeout environment variables
|
||||||
|
|
||||||
- `RUSTFS_DRIVE_METADATA_TIMEOUT_SECS`
|
- `RUSTFS_DRIVE_METADATA_TIMEOUT_SECS`
|
||||||
|
|||||||
@@ -353,11 +353,6 @@ pub const DEFAULT_OBS_TRACES_EXPORT_ENABLED: bool = true;
|
|||||||
/// Environment variable: RUSTFS_OBS_METRICS_EXPORT_ENABLED
|
/// Environment variable: RUSTFS_OBS_METRICS_EXPORT_ENABLED
|
||||||
pub const DEFAULT_OBS_METRICS_EXPORT_ENABLED: bool = true;
|
pub const DEFAULT_OBS_METRICS_EXPORT_ENABLED: bool = true;
|
||||||
|
|
||||||
/// Default detailed PUT stage metrics enabled
|
|
||||||
/// Default value: false
|
|
||||||
/// Environment variable: RUSTFS_OBS_PUT_STAGE_METRICS_ENABLED
|
|
||||||
pub const DEFAULT_OBS_PUT_STAGE_METRICS_ENABLED: bool = false;
|
|
||||||
|
|
||||||
/// Default logs export enabled
|
/// Default logs export enabled
|
||||||
/// It is used to enable or disable exporting logs
|
/// It is used to enable or disable exporting logs
|
||||||
/// Default value: true
|
/// Default value: true
|
||||||
|
|||||||
@@ -177,40 +177,3 @@ pub const DEFAULT_HEAL_MAINLINE_WRITE_UTILIZATION_HIGH_PERCENT: usize = 80;
|
|||||||
|
|
||||||
/// Default foreground pressure recheck delay for heal scheduler, in milliseconds.
|
/// Default foreground pressure recheck delay for heal scheduler, in milliseconds.
|
||||||
pub const DEFAULT_HEAL_MAINLINE_MAX_SLEEP_MS: u64 = 250;
|
pub const DEFAULT_HEAL_MAINLINE_MAX_SLEEP_MS: u64 = 250;
|
||||||
|
|
||||||
/// Environment variable that toggles the MRF (mission repair feed) intent
|
|
||||||
/// pipeline: error paths deliver repair intents to the heal runtime, and
|
|
||||||
/// unconsumed intents are replayed from the durable journal after a restart.
|
|
||||||
pub const ENV_HEAL_MRF_ENABLE: &str = "RUSTFS_HEAL_MRF_ENABLE";
|
|
||||||
|
|
||||||
/// Environment variable for the MRF in-memory queue capacity (intent count).
|
|
||||||
pub const ENV_HEAL_MRF_QUEUE_SIZE: &str = "RUSTFS_HEAL_MRF_QUEUE_SIZE";
|
|
||||||
|
|
||||||
/// Environment variable for the MRF journal byte budget. The journal is
|
|
||||||
/// compacted once its on-disk size crosses this bound.
|
|
||||||
pub const ENV_HEAL_MRF_JOURNAL_MAX_BYTES: &str = "RUSTFS_HEAL_MRF_JOURNAL_MAX_BYTES";
|
|
||||||
|
|
||||||
/// Environment variable for the MRF journal replay batch size (intents per
|
|
||||||
/// replay push round).
|
|
||||||
pub const ENV_HEAL_MRF_REPLAY_BATCH: &str = "RUSTFS_HEAL_MRF_REPLAY_BATCH";
|
|
||||||
|
|
||||||
/// Default behavior keeps the MRF intent pipeline enabled.
|
|
||||||
pub const DEFAULT_HEAL_MRF_ENABLE: bool = true;
|
|
||||||
|
|
||||||
/// Default MRF queue capacity (matches MinIO's 100k MRF list ceiling).
|
|
||||||
pub const DEFAULT_HEAL_MRF_QUEUE_SIZE: usize = 100_000;
|
|
||||||
|
|
||||||
/// Default MRF journal byte budget (8 MiB), mirroring the channel payload cap.
|
|
||||||
pub const DEFAULT_HEAL_MRF_JOURNAL_MAX_BYTES: usize = 8 * 1024 * 1024;
|
|
||||||
|
|
||||||
/// Default MRF replay batch size.
|
|
||||||
pub const DEFAULT_HEAL_MRF_REPLAY_BATCH: usize = 256;
|
|
||||||
|
|
||||||
/// Environment variable selecting how admin heal starts behave when the
|
|
||||||
/// requested path overlaps an already running or queued heal: `merge`
|
|
||||||
/// (default, keep today's dedup/merge semantics) or `minio_error` (return a
|
|
||||||
/// typed already-running / overlapping-paths rejection like madmin).
|
|
||||||
pub const ENV_HEAL_OVERLAP_POLICY: &str = "RUSTFS_HEAL_OVERLAP_POLICY";
|
|
||||||
|
|
||||||
/// Default overlap policy: merge duplicate/overlapping requests.
|
|
||||||
pub const DEFAULT_HEAL_OVERLAP_POLICY: &str = "merge";
|
|
||||||
|
|||||||
@@ -137,37 +137,6 @@ pub const DEFAULT_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED: bool = false;
|
|||||||
const _: () = assert!(!DEFAULT_TIER_REMOTE_VERSION_STATE_WRITE);
|
const _: () = assert!(!DEFAULT_TIER_REMOTE_VERSION_STATE_WRITE);
|
||||||
const _: () = assert!(!DEFAULT_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED);
|
const _: () = assert!(!DEFAULT_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED);
|
||||||
|
|
||||||
/// Request the object-transaction fencing contract used by storage-owned
|
|
||||||
/// cleanup receipts and lock-window optimizations.
|
|
||||||
///
|
|
||||||
/// This is fail-closed: enabling the writer without a live fleet proof rejects
|
|
||||||
/// the commit rather than silently using a legacy-safe path.
|
|
||||||
pub const ENV_OBJECT_TRANSACTION_FENCING_WRITE: &str = "RUSTFS_OBJECT_TRANSACTION_FENCING_WRITE";
|
|
||||||
pub const DEFAULT_OBJECT_TRANSACTION_FENCING_WRITE: bool = false;
|
|
||||||
|
|
||||||
/// Operator-attested confirmation that every serving node understands the
|
|
||||||
/// object transaction fencing contract.
|
|
||||||
pub const ENV_OBJECT_TRANSACTION_FENCING_FLEET_CONFIRMED: &str = "RUSTFS_OBJECT_TRANSACTION_FENCING_FLEET_CONFIRMED";
|
|
||||||
pub const DEFAULT_OBJECT_TRANSACTION_FENCING_FLEET_CONFIRMED: bool = false;
|
|
||||||
|
|
||||||
const _: () = assert!(!DEFAULT_OBJECT_TRANSACTION_FENCING_WRITE);
|
|
||||||
const _: () = assert!(!DEFAULT_OBJECT_TRANSACTION_FENCING_FLEET_CONFIRMED);
|
|
||||||
|
|
||||||
/// Request preserving legacy per-part checksum metadata during data movement.
|
|
||||||
///
|
|
||||||
/// This remains ineffective until
|
|
||||||
/// [`ENV_DATA_MOVEMENT_PART_CHECKSUMS_FLEET_CONFIRMED`] is also enabled.
|
|
||||||
pub const ENV_DATA_MOVEMENT_PART_CHECKSUMS_WRITE: &str = "RUSTFS_DATA_MOVEMENT_PART_CHECKSUMS_WRITE";
|
|
||||||
pub const DEFAULT_DATA_MOVEMENT_PART_CHECKSUMS_WRITE: bool = false;
|
|
||||||
|
|
||||||
/// Operator-attested confirmation that every serving node understands the
|
|
||||||
/// data-movement per-part checksum sidecar.
|
|
||||||
pub const ENV_DATA_MOVEMENT_PART_CHECKSUMS_FLEET_CONFIRMED: &str = "RUSTFS_DATA_MOVEMENT_PART_CHECKSUMS_FLEET_CONFIRMED";
|
|
||||||
pub const DEFAULT_DATA_MOVEMENT_PART_CHECKSUMS_FLEET_CONFIRMED: bool = false;
|
|
||||||
|
|
||||||
const _: () = assert!(!DEFAULT_DATA_MOVEMENT_PART_CHECKSUMS_WRITE);
|
|
||||||
const _: () = assert!(!DEFAULT_DATA_MOVEMENT_PART_CHECKSUMS_FLEET_CONFIRMED);
|
|
||||||
|
|
||||||
// =============================================================================
|
// =============================================================================
|
||||||
// Concurrent Request Fix - Timeout and Backpressure Configuration
|
// Concurrent Request Fix - Timeout and Backpressure Configuration
|
||||||
// =============================================================================
|
// =============================================================================
|
||||||
@@ -234,31 +203,6 @@ pub const ENV_OBJECT_DISK_WRITE_ABSOLUTE_CAP: &str = "RUSTFS_OBJECT_DISK_WRITE_A
|
|||||||
/// Default absolute per-object erasure write cap in seconds (`0` = disabled).
|
/// Default absolute per-object erasure write cap in seconds (`0` = disabled).
|
||||||
pub const DEFAULT_OBJECT_DISK_WRITE_ABSOLUTE_CAP: u64 = 0;
|
pub const DEFAULT_OBJECT_DISK_WRITE_ABSOLUTE_CAP: u64 = 0;
|
||||||
|
|
||||||
/// Enable foreground PutObject request admission.
|
|
||||||
///
|
|
||||||
/// This is an experimental, default-off foreground write backpressure gate for
|
|
||||||
/// strict commit tail investigations. When disabled, PUTs follow the legacy
|
|
||||||
/// path and only the existing request counters are updated.
|
|
||||||
pub const ENV_PUT_FOREGROUND_ADMISSION_ENABLE: &str = "RUSTFS_PUT_FOREGROUND_ADMISSION_ENABLE";
|
|
||||||
pub const DEFAULT_PUT_FOREGROUND_ADMISSION_ENABLE: bool = false;
|
|
||||||
|
|
||||||
/// Maximum foreground PutObject requests admitted concurrently per process.
|
|
||||||
///
|
|
||||||
/// The limit is used only when [`ENV_PUT_FOREGROUND_ADMISSION_ENABLE`] is true.
|
|
||||||
/// A value of `0` disables the gate even when the enable flag is present, so a
|
|
||||||
/// partially configured rollout cannot reject every PUT.
|
|
||||||
pub const ENV_PUT_FOREGROUND_ADMISSION_LIMIT: &str = "RUSTFS_PUT_FOREGROUND_ADMISSION_LIMIT";
|
|
||||||
pub const DEFAULT_PUT_FOREGROUND_ADMISSION_LIMIT: usize = 0;
|
|
||||||
|
|
||||||
/// Time in milliseconds a foreground PutObject waits for an admission permit.
|
|
||||||
///
|
|
||||||
/// Once this timeout expires the request fails before body ingest/storage
|
|
||||||
/// mutation with S3 `SlowDown`/503. `0` means fail fast when the limit is full.
|
|
||||||
pub const ENV_PUT_FOREGROUND_ADMISSION_WAIT_TIMEOUT_MS: &str = "RUSTFS_PUT_FOREGROUND_ADMISSION_WAIT_TIMEOUT_MS";
|
|
||||||
pub const DEFAULT_PUT_FOREGROUND_ADMISSION_WAIT_TIMEOUT_MS: u64 = 0;
|
|
||||||
|
|
||||||
const _: () = assert!(!DEFAULT_PUT_FOREGROUND_ADMISSION_ENABLE);
|
|
||||||
|
|
||||||
/// Environment variable for minimum GetObject timeout in seconds.
|
/// Environment variable for minimum GetObject timeout in seconds.
|
||||||
///
|
///
|
||||||
/// When dynamic timeout calculation is enabled, this is the minimum timeout
|
/// When dynamic timeout calculation is enabled, this is the minimum timeout
|
||||||
@@ -427,19 +371,6 @@ pub const ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT: &str = "RUSTFS_OBJECT_LOCK_ACQUIRE_TI
|
|||||||
/// Default lock acquisition timeout: 5 seconds.
|
/// Default lock acquisition timeout: 5 seconds.
|
||||||
pub const DEFAULT_OBJECT_LOCK_ACQUIRE_TIMEOUT: u64 = 5;
|
pub const DEFAULT_OBJECT_LOCK_ACQUIRE_TIMEOUT: u64 = 5;
|
||||||
|
|
||||||
/// Environment variable for the experimental PUT commit namespace lock acquire timeout in milliseconds.
|
|
||||||
///
|
|
||||||
/// A value of `0` disables the experiment and keeps
|
|
||||||
/// `RUSTFS_OBJECT_LOCK_ACQUIRE_TIMEOUT` as the timeout. This only bounds the
|
|
||||||
/// `put_object_commit` namespace write-lock wait and is intended for #925
|
|
||||||
/// tail-drain admission experiments.
|
|
||||||
///
|
|
||||||
/// Default: 0 milliseconds (disabled).
|
|
||||||
pub const ENV_PUT_COMMIT_NAMESPACE_LOCK_ACQUIRE_TIMEOUT_MS: &str = "RUSTFS_PUT_COMMIT_NAMESPACE_LOCK_ACQUIRE_TIMEOUT_MS";
|
|
||||||
|
|
||||||
/// Default: PUT commit namespace lock acquire timeout override is disabled.
|
|
||||||
pub const DEFAULT_PUT_COMMIT_NAMESPACE_LOCK_ACQUIRE_TIMEOUT_MS: u64 = 0;
|
|
||||||
|
|
||||||
/// Environment variable for remote namespace lock RPC transport timeout in milliseconds.
|
/// Environment variable for remote namespace lock RPC transport timeout in milliseconds.
|
||||||
///
|
///
|
||||||
/// This timeout bounds the internode RPC call itself. It is intentionally
|
/// This timeout bounds the internode RPC call itself. It is intentionally
|
||||||
@@ -718,22 +649,4 @@ mod remote_version_state_tests {
|
|||||||
"RUSTFS_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED"
|
"RUSTFS_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn data_movement_part_checksum_gate_uses_stable_environment_names() {
|
|
||||||
assert_eq!(super::ENV_DATA_MOVEMENT_PART_CHECKSUMS_WRITE, "RUSTFS_DATA_MOVEMENT_PART_CHECKSUMS_WRITE");
|
|
||||||
assert_eq!(
|
|
||||||
super::ENV_DATA_MOVEMENT_PART_CHECKSUMS_FLEET_CONFIRMED,
|
|
||||||
"RUSTFS_DATA_MOVEMENT_PART_CHECKSUMS_FLEET_CONFIRMED"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn object_transaction_fencing_gate_uses_stable_environment_names() {
|
|
||||||
assert_eq!(super::ENV_OBJECT_TRANSACTION_FENCING_WRITE, "RUSTFS_OBJECT_TRANSACTION_FENCING_WRITE");
|
|
||||||
assert_eq!(
|
|
||||||
super::ENV_OBJECT_TRANSACTION_FENCING_FLEET_CONFIRMED,
|
|
||||||
"RUSTFS_OBJECT_TRANSACTION_FENCING_FLEET_CONFIRMED"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -81,9 +81,6 @@ pub const ENV_TEST_IAM_FAIL_INIT_ATTEMPTS: &str = "RUSTFS_TEST_IAM_FAIL_INIT_ATT
|
|||||||
pub const ENV_TEST_IAM_RETRY_INTERVAL_MS: &str = "RUSTFS_TEST_IAM_RETRY_INTERVAL_MS";
|
pub const ENV_TEST_IAM_RETRY_INTERVAL_MS: &str = "RUSTFS_TEST_IAM_RETRY_INTERVAL_MS";
|
||||||
/// Runtime env var controlling the transition worker count.
|
/// Runtime env var controlling the transition worker count.
|
||||||
pub const ENV_TRANSITION_WORKERS: &str = "RUSTFS_MAX_TRANSITION_WORKERS";
|
pub const ENV_TRANSITION_WORKERS: &str = "RUSTFS_MAX_TRANSITION_WORKERS";
|
||||||
/// Runtime env var controlling the ILM expiry worker count. A set, parsable,
|
|
||||||
/// non-zero value wins; anything else falls back to `min(cpus, 16)`.
|
|
||||||
pub const ENV_MAX_EXPIRY_WORKERS: &str = "RUSTFS_MAX_EXPIRY_WORKERS";
|
|
||||||
/// Runtime env var controlling the absolute maximum transition workers.
|
/// Runtime env var controlling the absolute maximum transition workers.
|
||||||
pub const ENV_TRANSITION_WORKERS_ABSOLUTE_MAX: &str = "RUSTFS_ABSOLUTE_MAX_WORKERS";
|
pub const ENV_TRANSITION_WORKERS_ABSOLUTE_MAX: &str = "RUSTFS_ABSOLUTE_MAX_WORKERS";
|
||||||
/// Runtime env var controlling the transition queue capacity.
|
/// Runtime env var controlling the transition queue capacity.
|
||||||
|
|||||||
@@ -228,6 +228,15 @@ pub const DEFAULT_SCANNER_MAX_CONCURRENT_DISK_SCANS: usize = 4;
|
|||||||
/// Default object interval for cooperative scanner yields.
|
/// Default object interval for cooperative scanner yields.
|
||||||
pub const DEFAULT_SCANNER_YIELD_EVERY_N_OBJECTS: u64 = 128;
|
pub const DEFAULT_SCANNER_YIELD_EVERY_N_OBJECTS: u64 = 128;
|
||||||
|
|
||||||
|
/// Compatibility flag kept for Patch 3 rollback windows.
|
||||||
|
///
|
||||||
|
/// Inline scanner heal execution has been removed in favor of heal-candidate enqueue.
|
||||||
|
/// When this flag is enabled, RustFS logs a warning and continues to use enqueue-based heal.
|
||||||
|
pub const ENV_SCANNER_INLINE_HEAL_ENABLE: &str = "RUSTFS_SCANNER_INLINE_HEAL_ENABLE";
|
||||||
|
|
||||||
|
/// Default inline scanner heal compatibility mode.
|
||||||
|
pub const DEFAULT_SCANNER_INLINE_HEAL_ENABLE: bool = false;
|
||||||
|
|
||||||
/// Scanner speed preset controlling throttling behavior.
|
/// Scanner speed preset controlling throttling behavior.
|
||||||
///
|
///
|
||||||
/// Each preset defines three parameters:
|
/// Each preset defines three parameters:
|
||||||
|
|||||||
@@ -92,11 +92,15 @@ pub const NOTIFY_SUB_SYSTEMS: &[&str] = &[
|
|||||||
pub const NOTIFY_KAFKA_SUB_SYS: &str = "notify_kafka";
|
pub const NOTIFY_KAFKA_SUB_SYS: &str = "notify_kafka";
|
||||||
pub const NOTIFY_MQTT_SUB_SYS: &str = "notify_mqtt";
|
pub const NOTIFY_MQTT_SUB_SYS: &str = "notify_mqtt";
|
||||||
pub const NOTIFY_MYSQL_SUB_SYS: &str = "notify_mysql";
|
pub const NOTIFY_MYSQL_SUB_SYS: &str = "notify_mysql";
|
||||||
|
#[allow(dead_code)]
|
||||||
pub const NOTIFY_NATS_SUB_SYS: &str = "notify_nats";
|
pub const NOTIFY_NATS_SUB_SYS: &str = "notify_nats";
|
||||||
|
#[allow(dead_code)]
|
||||||
pub const NOTIFY_NSQ_SUB_SYS: &str = "notify_nsq";
|
pub const NOTIFY_NSQ_SUB_SYS: &str = "notify_nsq";
|
||||||
|
#[allow(dead_code)]
|
||||||
pub const NOTIFY_ES_SUB_SYS: &str = "notify_elasticsearch";
|
pub const NOTIFY_ES_SUB_SYS: &str = "notify_elasticsearch";
|
||||||
pub const NOTIFY_AMQP_SUB_SYS: &str = "notify_amqp";
|
pub const NOTIFY_AMQP_SUB_SYS: &str = "notify_amqp";
|
||||||
pub const NOTIFY_POSTGRES_SUB_SYS: &str = "notify_postgres";
|
pub const NOTIFY_POSTGRES_SUB_SYS: &str = "notify_postgres";
|
||||||
|
#[allow(dead_code)]
|
||||||
pub const NOTIFY_REDIS_SUB_SYS: &str = "notify_redis";
|
pub const NOTIFY_REDIS_SUB_SYS: &str = "notify_redis";
|
||||||
pub const NOTIFY_REDIS_DEFAULT_CHANNEL: &str = "rustfs_notify_channel";
|
pub const NOTIFY_REDIS_DEFAULT_CHANNEL: &str = "rustfs_notify_channel";
|
||||||
pub const NOTIFY_PULSAR_SUB_SYS: &str = "notify_pulsar";
|
pub const NOTIFY_PULSAR_SUB_SYS: &str = "notify_pulsar";
|
||||||
|
|||||||
@@ -44,10 +44,6 @@ pub const ENV_OBS_METRICS_EXPORT_ENABLED: &str = "RUSTFS_OBS_METRICS_EXPORT_ENAB
|
|||||||
pub const ENV_OBS_LOGS_EXPORT_ENABLED: &str = "RUSTFS_OBS_LOGS_EXPORT_ENABLED";
|
pub const ENV_OBS_LOGS_EXPORT_ENABLED: &str = "RUSTFS_OBS_LOGS_EXPORT_ENABLED";
|
||||||
pub const ENV_OBS_PROFILING_EXPORT_ENABLED: &str = "RUSTFS_OBS_PROFILING_EXPORT_ENABLED";
|
pub const ENV_OBS_PROFILING_EXPORT_ENABLED: &str = "RUSTFS_OBS_PROFILING_EXPORT_ENABLED";
|
||||||
|
|
||||||
/// Enables detailed per-stage PUT metrics. Disabled by default because each
|
|
||||||
/// PUT records multiple timers and histograms when attribution is active.
|
|
||||||
pub const ENV_OBS_PUT_STAGE_METRICS_ENABLED: &str = "RUSTFS_OBS_PUT_STAGE_METRICS_ENABLED";
|
|
||||||
|
|
||||||
pub const ENV_OBS_LOGGER_LEVEL: &str = "RUSTFS_OBS_LOGGER_LEVEL";
|
pub const ENV_OBS_LOGGER_LEVEL: &str = "RUSTFS_OBS_LOGGER_LEVEL";
|
||||||
pub const ENV_OBS_LOG_STDOUT_ENABLED: &str = "RUSTFS_OBS_LOG_STDOUT_ENABLED";
|
pub const ENV_OBS_LOG_STDOUT_ENABLED: &str = "RUSTFS_OBS_LOG_STDOUT_ENABLED";
|
||||||
pub const ENV_OBS_LOG_DIRECTORY: &str = "RUSTFS_OBS_LOG_DIRECTORY";
|
pub const ENV_OBS_LOG_DIRECTORY: &str = "RUSTFS_OBS_LOG_DIRECTORY";
|
||||||
@@ -145,7 +141,6 @@ mod tests {
|
|||||||
assert_eq!(ENV_OBS_METRICS_EXPORT_ENABLED, "RUSTFS_OBS_METRICS_EXPORT_ENABLED");
|
assert_eq!(ENV_OBS_METRICS_EXPORT_ENABLED, "RUSTFS_OBS_METRICS_EXPORT_ENABLED");
|
||||||
assert_eq!(ENV_OBS_LOGS_EXPORT_ENABLED, "RUSTFS_OBS_LOGS_EXPORT_ENABLED");
|
assert_eq!(ENV_OBS_LOGS_EXPORT_ENABLED, "RUSTFS_OBS_LOGS_EXPORT_ENABLED");
|
||||||
assert_eq!(ENV_OBS_PROFILING_EXPORT_ENABLED, "RUSTFS_OBS_PROFILING_EXPORT_ENABLED");
|
assert_eq!(ENV_OBS_PROFILING_EXPORT_ENABLED, "RUSTFS_OBS_PROFILING_EXPORT_ENABLED");
|
||||||
assert_eq!(ENV_OBS_PUT_STAGE_METRICS_ENABLED, "RUSTFS_OBS_PUT_STAGE_METRICS_ENABLED");
|
|
||||||
// Test log cleanup related env keys
|
// Test log cleanup related env keys
|
||||||
assert_eq!(ENV_OBS_LOG_MAX_TOTAL_SIZE_BYTES, "RUSTFS_OBS_LOG_MAX_TOTAL_SIZE_BYTES");
|
assert_eq!(ENV_OBS_LOG_MAX_TOTAL_SIZE_BYTES, "RUSTFS_OBS_LOG_MAX_TOTAL_SIZE_BYTES");
|
||||||
assert_eq!(ENV_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES, "RUSTFS_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES");
|
assert_eq!(ENV_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES, "RUSTFS_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES");
|
||||||
|
|||||||
@@ -203,6 +203,14 @@ mod tests {
|
|||||||
assert!(result.is_err());
|
assert!(result.is_err());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_source_does_not_embed_private_key() {
|
||||||
|
let source = include_str!("license_token.rs");
|
||||||
|
let forbidden = ["BEGIN", "PRIVATE KEY"].join(" ");
|
||||||
|
|
||||||
|
assert!(!source.contains(&forbidden));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_parse_signed_license_token_rejects_invalid_token() {
|
fn test_parse_signed_license_token_rejects_invalid_token() {
|
||||||
let mut rng = rand::rng();
|
let mut rng = rand::rng();
|
||||||
|
|||||||
@@ -37,6 +37,7 @@ hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu", "rustfs-filemeta/hotpath-cpu"]
|
|||||||
hotpath.workspace = true
|
hotpath.workspace = true
|
||||||
serde = { workspace = true, features = ["derive"] }
|
serde = { workspace = true, features = ["derive"] }
|
||||||
rmp-serde = { workspace = true }
|
rmp-serde = { workspace = true }
|
||||||
|
async-trait = { workspace = true }
|
||||||
rustfs-filemeta = { workspace = true }
|
rustfs-filemeta = { workspace = true }
|
||||||
|
|
||||||
[lib]
|
[lib]
|
||||||
|
|||||||
@@ -317,15 +317,15 @@ pub struct SizeSummary {
|
|||||||
/// Number of delete markers
|
/// Number of delete markers
|
||||||
pub delete_markers: usize,
|
pub delete_markers: usize,
|
||||||
/// Replicated size
|
/// Replicated size
|
||||||
pub replicated_size: i64,
|
pub replicated_size: usize,
|
||||||
/// Replicated count
|
/// Replicated count
|
||||||
pub replicated_count: usize,
|
pub replicated_count: usize,
|
||||||
/// Pending size
|
/// Pending size
|
||||||
pub pending_size: i64,
|
pub pending_size: usize,
|
||||||
/// Failed size
|
/// Failed size
|
||||||
pub failed_size: i64,
|
pub failed_size: usize,
|
||||||
/// Replica size
|
/// Replica size
|
||||||
pub replica_size: i64,
|
pub replica_size: usize,
|
||||||
/// Replica count
|
/// Replica count
|
||||||
pub replica_count: usize,
|
pub replica_count: usize,
|
||||||
/// Pending count
|
/// Pending count
|
||||||
@@ -334,21 +334,19 @@ pub struct SizeSummary {
|
|||||||
pub failed_count: usize,
|
pub failed_count: usize,
|
||||||
/// Replication target stats
|
/// Replication target stats
|
||||||
pub repl_target_stats: HashMap<String, ReplTargetSizeSummary>,
|
pub repl_target_stats: HashMap<String, ReplTargetSizeSummary>,
|
||||||
/// Per-tier accounting, keyed by storage class or remote tier name
|
|
||||||
pub tier_stats: HashMap<String, TierStats>,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Replication target size summary
|
/// Replication target size summary
|
||||||
#[derive(Debug, Default, Clone)]
|
#[derive(Debug, Default, Clone)]
|
||||||
pub struct ReplTargetSizeSummary {
|
pub struct ReplTargetSizeSummary {
|
||||||
/// Replicated size
|
/// Replicated size
|
||||||
pub replicated_size: i64,
|
pub replicated_size: usize,
|
||||||
/// Replicated count
|
/// Replicated count
|
||||||
pub replicated_count: usize,
|
pub replicated_count: usize,
|
||||||
/// Pending size
|
/// Pending size
|
||||||
pub pending_size: i64,
|
pub pending_size: usize,
|
||||||
/// Failed size
|
/// Failed size
|
||||||
pub failed_size: i64,
|
pub failed_size: usize,
|
||||||
/// Pending count
|
/// Pending count
|
||||||
pub pending_count: usize,
|
pub pending_count: usize,
|
||||||
/// Failed count
|
/// Failed count
|
||||||
@@ -712,6 +710,28 @@ impl DataUsageEntry {
|
|||||||
self.children.insert(hash.key());
|
self.children.insert(hash.key());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn add_sizes(&mut self, summary: &SizeSummary) {
|
||||||
|
self.size += summary.total_size;
|
||||||
|
self.versions += summary.versions;
|
||||||
|
self.delete_markers += summary.delete_markers;
|
||||||
|
self.obj_sizes.add(summary.total_size as u64);
|
||||||
|
self.obj_versions.add(summary.versions as u64);
|
||||||
|
|
||||||
|
let replication_stats = self.replication_stats.get_or_insert_with(ReplicationAllStats::default);
|
||||||
|
replication_stats.replica_size += summary.replica_size as u64;
|
||||||
|
replication_stats.replica_count += summary.replica_count as u64;
|
||||||
|
|
||||||
|
for (arn, st) in &summary.repl_target_stats {
|
||||||
|
let tgt_stat = replication_stats.targets.entry(arn.to_string()).or_default();
|
||||||
|
tgt_stat.pending_size += st.pending_size as u64;
|
||||||
|
tgt_stat.failed_size += st.failed_size as u64;
|
||||||
|
tgt_stat.replicated_size += st.replicated_size as u64;
|
||||||
|
tgt_stat.replicated_count += st.replicated_count as u64;
|
||||||
|
tgt_stat.failed_count += st.failed_count as u64;
|
||||||
|
tgt_stat.pending_count += st.pending_count as u64;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub fn merge(&mut self, other: &DataUsageEntry) {
|
pub fn merge(&mut self, other: &DataUsageEntry) {
|
||||||
self.objects += other.objects;
|
self.objects += other.objects;
|
||||||
self.versions += other.versions;
|
self.versions += other.versions;
|
||||||
@@ -826,15 +846,8 @@ impl DataUsageEntry {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Read-only projection of the scanner's `.usage-cache.bin` info block.
|
/// Data usage cache info
|
||||||
///
|
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
|
||||||
/// The canonical wire format is written by the hand-written map-encoded
|
|
||||||
/// `Serialize` on the scanner-side `DataUsageCacheInfo`
|
|
||||||
/// (`crates/scanner/src/data_usage_define.rs`), which carries 16 fields.
|
|
||||||
/// This type decodes only the shared subset and is deliberately not
|
|
||||||
/// `Serialize`: a derived (array) encoding of this 6-field subset would
|
|
||||||
/// corrupt the cache for scanner readers, so no write path may exist here.
|
|
||||||
#[derive(Clone, Debug, Default, Deserialize)]
|
|
||||||
pub struct DataUsageCacheInfo {
|
pub struct DataUsageCacheInfo {
|
||||||
pub name: String,
|
pub name: String,
|
||||||
pub next_cycle: u64,
|
pub next_cycle: u64,
|
||||||
@@ -850,163 +863,8 @@ pub struct DataUsageCacheInfo {
|
|||||||
pub snapshot_complete: bool,
|
pub snapshot_complete: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Prefix-level usage over a raw entry map — the shared core behind
|
/// Data usage cache
|
||||||
/// [`DataUsageCache::prefix_usage`], usable by any cache-shaped reader (the
|
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
|
||||||
/// scanner's writer-side cache has the same map type).
|
|
||||||
///
|
|
||||||
/// Cache keys are cleaned literal paths (`bucket/pre/fix`), so sub-prefix
|
|
||||||
/// names come straight off the child keys — no reverse mapping exists or is
|
|
||||||
/// needed. A compacted prefix carries its aggregate but no children, which
|
|
||||||
/// the `compacted` flag reports so callers can say why the breakdown is
|
|
||||||
/// empty. `truncated` is set when the breakdown exceeded `max_entries` and
|
|
||||||
/// was cut (largest first).
|
|
||||||
pub fn prefix_usage_in_cache(
|
|
||||||
cache: &HashMap<String, DataUsageEntry>,
|
|
||||||
bucket: &str,
|
|
||||||
prefix: &str,
|
|
||||||
max_entries: usize,
|
|
||||||
) -> Option<PrefixUsageQuery> {
|
|
||||||
let prefix = prefix.trim_matches('/');
|
|
||||||
let root = if prefix.is_empty() {
|
|
||||||
bucket.to_string()
|
|
||||||
} else {
|
|
||||||
format!("{bucket}/{prefix}")
|
|
||||||
};
|
|
||||||
let entry = cache.get(&hash_path(&root).key())?.clone();
|
|
||||||
|
|
||||||
let usage = PrefixUsageSummary::from_entry(&flatten_entry(cache, &entry, 0)?);
|
|
||||||
|
|
||||||
let child_prefix = format!("{root}/");
|
|
||||||
let mut sub_prefixes: Vec<PrefixUsageEntry> = entry
|
|
||||||
.children
|
|
||||||
.iter()
|
|
||||||
.filter_map(|child_key| {
|
|
||||||
let child = cache.get(child_key)?;
|
|
||||||
let child_flat = flatten_entry(cache, child, 1)?;
|
|
||||||
// Child keys are literal `bucket/pre/name` paths; a trailing
|
|
||||||
// slash marks a directory object and is display-only here.
|
|
||||||
let name = child_key
|
|
||||||
.strip_prefix(child_prefix.as_str())
|
|
||||||
.unwrap_or(child_key.as_str())
|
|
||||||
.trim_end_matches('/')
|
|
||||||
.to_string();
|
|
||||||
Some(PrefixUsageEntry {
|
|
||||||
prefix: name,
|
|
||||||
usage: PrefixUsageSummary::from_entry(&child_flat),
|
|
||||||
})
|
|
||||||
})
|
|
||||||
.collect();
|
|
||||||
sub_prefixes.sort_by(|left, right| {
|
|
||||||
right
|
|
||||||
.usage
|
|
||||||
.size
|
|
||||||
.cmp(&left.usage.size)
|
|
||||||
.then_with(|| left.prefix.cmp(&right.prefix))
|
|
||||||
});
|
|
||||||
let truncated = sub_prefixes.len() > max_entries;
|
|
||||||
sub_prefixes.truncate(max_entries);
|
|
||||||
|
|
||||||
Some(PrefixUsageQuery {
|
|
||||||
usage,
|
|
||||||
compacted: entry.compacted,
|
|
||||||
truncated,
|
|
||||||
sub_prefixes,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Maximum subtree depth [`flatten_entry`] will walk before declaring the
|
|
||||||
/// cache corrupt — the same bound the scanner's checked flatten uses.
|
|
||||||
const PREFIX_USAGE_MAX_DEPTH: usize = 1024;
|
|
||||||
|
|
||||||
/// Flatten one entry's subtree into an aggregate: the free-function twin of
|
|
||||||
/// [`DataUsageCache::flatten`], carrying the scanner checked-flatten
|
|
||||||
/// hardening so a corrupt cache (cycles, over-deep trees, overflowing
|
|
||||||
/// counters) yields `None` instead of unbounded recursion or wrapped totals.
|
|
||||||
fn flatten_entry(cache: &HashMap<String, DataUsageEntry>, root: &DataUsageEntry, depth: usize) -> Option<DataUsageEntry> {
|
|
||||||
if depth > PREFIX_USAGE_MAX_DEPTH {
|
|
||||||
return None;
|
|
||||||
}
|
|
||||||
let mut flattened = DataUsageEntry::default();
|
|
||||||
if !flattened.checked_merge(root) {
|
|
||||||
return None;
|
|
||||||
}
|
|
||||||
flattened.compacted = root.compacted;
|
|
||||||
// The root itself is not pre-seeded: it is merged above, and a corrupt
|
|
||||||
// child edge pointing back at the root's own key is still terminated by
|
|
||||||
// the visited set on first encounter.
|
|
||||||
let mut visited: HashSet<&str> = HashSet::new();
|
|
||||||
let mut pending: Vec<(&String, usize)> = root.children.iter().map(|child| (child, depth + 1)).collect();
|
|
||||||
while let Some((key, child_depth)) = pending.pop() {
|
|
||||||
if child_depth > PREFIX_USAGE_MAX_DEPTH || !visited.insert(key.as_str()) {
|
|
||||||
return None;
|
|
||||||
}
|
|
||||||
let entry = cache.get(key)?;
|
|
||||||
if !flattened.checked_merge(entry) {
|
|
||||||
return None;
|
|
||||||
}
|
|
||||||
pending.extend(entry.children.iter().map(|child| (child, child_depth + 1)));
|
|
||||||
}
|
|
||||||
flattened.children.clear();
|
|
||||||
Some(flattened)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Flattened counters of one prefix subtree, as returned by
|
|
||||||
/// [`DataUsageCache::prefix_usage`].
|
|
||||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, serde::Serialize)]
|
|
||||||
#[serde(rename_all = "camelCase")]
|
|
||||||
pub struct PrefixUsageSummary {
|
|
||||||
pub size: u64,
|
|
||||||
pub objects: u64,
|
|
||||||
pub versions: u64,
|
|
||||||
pub delete_markers: u64,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl PrefixUsageSummary {
|
|
||||||
fn from_entry(entry: &DataUsageEntry) -> Self {
|
|
||||||
Self {
|
|
||||||
size: entry.size as u64,
|
|
||||||
objects: entry.objects as u64,
|
|
||||||
versions: entry.versions as u64,
|
|
||||||
delete_markers: entry.delete_markers as u64,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Add another set's counters into this one (entries are partitioned by
|
|
||||||
/// set, so per-set results sum).
|
|
||||||
pub fn merge(&mut self, other: &Self) {
|
|
||||||
self.size = self.size.saturating_add(other.size);
|
|
||||||
self.objects = self.objects.saturating_add(other.objects);
|
|
||||||
self.versions = self.versions.saturating_add(other.versions);
|
|
||||||
self.delete_markers = self.delete_markers.saturating_add(other.delete_markers);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// One first-level sub-prefix row of a [`PrefixUsageQuery`].
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize)]
|
|
||||||
pub struct PrefixUsageEntry {
|
|
||||||
pub prefix: String,
|
|
||||||
pub usage: PrefixUsageSummary,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Result of [`DataUsageCache::prefix_usage`].
|
|
||||||
#[derive(Clone, Debug, Default, PartialEq, Eq, serde::Serialize)]
|
|
||||||
#[serde(rename_all = "camelCase")]
|
|
||||||
pub struct PrefixUsageQuery {
|
|
||||||
pub usage: PrefixUsageSummary,
|
|
||||||
/// The prefix entry was compacted by the scanner: its aggregate is valid
|
|
||||||
/// but no sub-prefix breakdown exists on disk.
|
|
||||||
pub compacted: bool,
|
|
||||||
/// The breakdown had more entries than `max_entries`; the largest remain.
|
|
||||||
pub truncated: bool,
|
|
||||||
pub sub_prefixes: Vec<PrefixUsageEntry>,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Read-only projection of a scanner-written `.usage-cache.bin` file.
|
|
||||||
///
|
|
||||||
/// The scanner-side `DataUsageCache` (`crates/scanner/src/data_usage_define.rs`)
|
|
||||||
/// owns the persisted format; this type only decodes it (see
|
|
||||||
/// [`DataUsageCacheInfo`]) and must never grow a serialization path.
|
|
||||||
#[derive(Clone, Debug, Default, Deserialize)]
|
|
||||||
pub struct DataUsageCache {
|
pub struct DataUsageCache {
|
||||||
pub info: DataUsageCacheInfo,
|
pub info: DataUsageCacheInfo,
|
||||||
pub cache: HashMap<String, DataUsageEntry>,
|
pub cache: HashMap<String, DataUsageEntry>,
|
||||||
@@ -1128,21 +986,6 @@ impl DataUsageCache {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Prefix-level usage for one bucket subtree, plus the one-level
|
|
||||||
/// breakdown below it (rustfs/backlog#1872, MinIO
|
|
||||||
/// `loadPrefixUsageFromBackend` parity and beyond: arbitrary prefixes and
|
|
||||||
/// full counters instead of first-level sizes only).
|
|
||||||
///
|
|
||||||
/// Cache keys are cleaned literal paths (`bucket/pre/fix`), so sub-prefix
|
|
||||||
/// names come straight off the child keys — no reverse mapping exists or
|
|
||||||
/// is needed. A compacted prefix carries its aggregate but no children,
|
|
||||||
/// which the `compacted` flag reports so callers can say why the
|
|
||||||
/// breakdown is empty. `truncated` is set when the breakdown exceeded
|
|
||||||
/// `max_entries` and was cut (largest first).
|
|
||||||
pub fn prefix_usage(&self, bucket: &str, prefix: &str, max_entries: usize) -> Option<PrefixUsageQuery> {
|
|
||||||
prefix_usage_in_cache(&self.cache, bucket, prefix, max_entries)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn force_compact(&mut self, limit: usize) {
|
pub fn force_compact(&mut self, limit: usize) {
|
||||||
if self.cache.len() < limit {
|
if self.cache.len() < limit {
|
||||||
return;
|
return;
|
||||||
@@ -1343,10 +1186,31 @@ impl DataUsageCache {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn marshal_msg(&self) -> Result<Vec<u8>, Box<dyn std::error::Error + Send + Sync>> {
|
||||||
|
let mut buf = Vec::new();
|
||||||
|
self.serialize(&mut rmp_serde::Serializer::new(&mut buf))?;
|
||||||
|
Ok(buf)
|
||||||
|
}
|
||||||
|
|
||||||
pub fn unmarshal(buf: &[u8]) -> Result<Self, Box<dyn std::error::Error + Send + Sync>> {
|
pub fn unmarshal(buf: &[u8]) -> Result<Self, Box<dyn std::error::Error + Send + Sync>> {
|
||||||
let t: Self = rmp_serde::from_slice(buf)?;
|
let t: Self = rmp_serde::from_slice(buf)?;
|
||||||
Ok(t)
|
Ok(t)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Note: load and save methods are storage-specific and should be implemented
|
||||||
|
// in the ecstore crate where storage access is available
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Trait for storage-specific operations on DataUsageCache
|
||||||
|
#[async_trait::async_trait]
|
||||||
|
pub trait DataUsageCacheStorage {
|
||||||
|
/// Load data usage cache from backend storage
|
||||||
|
async fn load(store: &dyn std::any::Any, name: &str) -> Result<Self, Box<dyn std::error::Error + Send + Sync>>
|
||||||
|
where
|
||||||
|
Self: Sized;
|
||||||
|
|
||||||
|
/// Save data usage cache to backend storage
|
||||||
|
async fn save(&self, name: &str) -> Result<(), Box<dyn std::error::Error + Send + Sync>>;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Helper structs and functions for cache operations
|
// Helper structs and functions for cache operations
|
||||||
@@ -1702,6 +1566,14 @@ impl BucketUsageInfo {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Add size summary to this bucket usage
|
/// Add size summary to this bucket usage
|
||||||
|
pub fn add_size_summary(&mut self, summary: &SizeSummary) {
|
||||||
|
self.size += summary.total_size as u64;
|
||||||
|
self.versions_count += summary.versions as u64;
|
||||||
|
self.delete_markers_count += summary.delete_markers as u64;
|
||||||
|
self.replica_size += summary.replica_size as u64;
|
||||||
|
self.replica_count += summary.replica_count as u64;
|
||||||
|
}
|
||||||
|
|
||||||
/// Merge another BucketUsageInfo into this one
|
/// Merge another BucketUsageInfo into this one
|
||||||
pub fn merge(&mut self, other: &BucketUsageInfo) {
|
pub fn merge(&mut self, other: &BucketUsageInfo) {
|
||||||
self.size += other.size;
|
self.size += other.size;
|
||||||
@@ -1747,32 +1619,29 @@ impl SizeSummary {
|
|||||||
Self::default()
|
Self::default()
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Add another SizeSummary to this one.
|
/// Add another SizeSummary to this one
|
||||||
///
|
|
||||||
/// Saturating throughout: a scan that overflows a counter should report the
|
|
||||||
/// ceiling rather than panic in a debug build or wrap in a release one.
|
|
||||||
pub fn add(&mut self, other: &SizeSummary) {
|
pub fn add(&mut self, other: &SizeSummary) {
|
||||||
self.total_size = self.total_size.saturating_add(other.total_size);
|
self.total_size += other.total_size;
|
||||||
self.versions = self.versions.saturating_add(other.versions);
|
self.versions += other.versions;
|
||||||
self.delete_markers = self.delete_markers.saturating_add(other.delete_markers);
|
self.delete_markers += other.delete_markers;
|
||||||
self.replicated_size = self.replicated_size.saturating_add(other.replicated_size);
|
self.replicated_size += other.replicated_size;
|
||||||
self.replicated_count = self.replicated_count.saturating_add(other.replicated_count);
|
self.replicated_count += other.replicated_count;
|
||||||
self.pending_size = self.pending_size.saturating_add(other.pending_size);
|
self.pending_size += other.pending_size;
|
||||||
self.failed_size = self.failed_size.saturating_add(other.failed_size);
|
self.failed_size += other.failed_size;
|
||||||
self.replica_size = self.replica_size.saturating_add(other.replica_size);
|
self.replica_size += other.replica_size;
|
||||||
self.replica_count = self.replica_count.saturating_add(other.replica_count);
|
self.replica_count += other.replica_count;
|
||||||
self.pending_count = self.pending_count.saturating_add(other.pending_count);
|
self.pending_count += other.pending_count;
|
||||||
self.failed_count = self.failed_count.saturating_add(other.failed_count);
|
self.failed_count += other.failed_count;
|
||||||
|
|
||||||
// Merge replication target stats
|
// Merge replication target stats
|
||||||
for (target, stats) in &other.repl_target_stats {
|
for (target, stats) in &other.repl_target_stats {
|
||||||
let entry = self.repl_target_stats.entry(target.clone()).or_default();
|
let entry = self.repl_target_stats.entry(target.clone()).or_default();
|
||||||
entry.replicated_size = entry.replicated_size.saturating_add(stats.replicated_size);
|
entry.replicated_size += stats.replicated_size;
|
||||||
entry.replicated_count = entry.replicated_count.saturating_add(stats.replicated_count);
|
entry.replicated_count += stats.replicated_count;
|
||||||
entry.pending_size = entry.pending_size.saturating_add(stats.pending_size);
|
entry.pending_size += stats.pending_size;
|
||||||
entry.failed_size = entry.failed_size.saturating_add(stats.failed_size);
|
entry.failed_size += stats.failed_size;
|
||||||
entry.pending_count = entry.pending_count.saturating_add(stats.pending_count);
|
entry.pending_count += stats.pending_count;
|
||||||
entry.failed_count = entry.failed_count.saturating_add(stats.failed_count);
|
entry.failed_count += stats.failed_count;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1963,202 +1832,6 @@ mod tests {
|
|||||||
assert!(decoded.all_tier_stats.is_none());
|
assert!(decoded.all_tier_stats.is_none());
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Scanner-written `.usage-cache.bin` bytes: a 2-element array of the
|
|
||||||
/// canonical 16-field map-encoded info block and one map-encoded entry.
|
|
||||||
/// Captured from the canonical writer's `marshal_msg` — see
|
|
||||||
/// `usage_cache_wire_format_is_pinned` in
|
|
||||||
/// `crates/scanner/src/data_usage_define.rs`, which pins these exact
|
|
||||||
/// bytes and documents regeneration. Hardcoded here because a
|
|
||||||
/// dev-dependency on rustfs-scanner would pull the whole ecstore tree
|
|
||||||
/// into this crate's test build, and a fixture generated at test runtime
|
|
||||||
/// could not detect writer drift anyway.
|
|
||||||
const SCANNER_USAGE_CACHE_WIRE_FIXTURE: &[u8] = &[
|
|
||||||
0x92, 0xde, 0x00, 0x10, 0xa4, 0x6e, 0x61, 0x6d, 0x65, 0xab, 0x77, 0x69, 0x72, 0x65, 0x2d, 0x62, 0x75, 0x63, 0x6b, 0x65,
|
|
||||||
0x74, 0xaa, 0x6e, 0x65, 0x78, 0x74, 0x5f, 0x63, 0x79, 0x63, 0x6c, 0x65, 0x07, 0xac, 0x6c, 0x65, 0x61, 0x64, 0x65, 0x72,
|
|
||||||
0x5f, 0x65, 0x70, 0x6f, 0x63, 0x68, 0x09, 0xab, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x92,
|
|
||||||
0xce, 0x65, 0x53, 0xf1, 0x00, 0x00, 0xac, 0x73, 0x6b, 0x69, 0x70, 0x5f, 0x68, 0x65, 0x61, 0x6c, 0x69, 0x6e, 0x67, 0xc3,
|
|
||||||
0xa9, 0x6c, 0x69, 0x66, 0x65, 0x63, 0x79, 0x63, 0x6c, 0x65, 0xc0, 0xab, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74,
|
|
||||||
0x69, 0x6f, 0x6e, 0xc0, 0xae, 0x66, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x5f, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x81,
|
|
||||||
0xb0, 0x77, 0x69, 0x72, 0x65, 0x2d, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x2f, 0x6c, 0x6f, 0x73, 0x74, 0x0b, 0xb1, 0x73,
|
|
||||||
0x63, 0x61, 0x6e, 0x5f, 0x72, 0x65, 0x73, 0x75, 0x6d, 0x65, 0x5f, 0x61, 0x66, 0x74, 0x65, 0x72, 0xb2, 0x77, 0x69, 0x72,
|
|
||||||
0x65, 0x2d, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x2f, 0x72, 0x65, 0x73, 0x75, 0x6d, 0x65, 0xaf, 0x73, 0x63, 0x61, 0x6e,
|
|
||||||
0x5f, 0x63, 0x68, 0x65, 0x63, 0x6b, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0xc0, 0xad, 0x70, 0x65, 0x6e, 0x64, 0x69, 0x6e, 0x67,
|
|
||||||
0x5f, 0x68, 0x65, 0x61, 0x6c, 0x73, 0x91, 0x9a, 0xa6, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0xab, 0x77, 0x69, 0x72, 0x65,
|
|
||||||
0x2d, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0xa6, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x6e, 0xc0, 0x01, 0x64, 0xcc, 0xc8, 0x03,
|
|
||||||
0xa8, 0x64, 0x65, 0x66, 0x65, 0x72, 0x72, 0x65, 0x64, 0xa6, 0x62, 0x75, 0x64, 0x67, 0x65, 0x74, 0xab, 0x6f, 0x62, 0x6a,
|
|
||||||
0x65, 0x63, 0x74, 0x5f, 0x6c, 0x6f, 0x63, 0x6b, 0xc0, 0xa6, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x92, 0x01, 0x02, 0xb1,
|
|
||||||
0x73, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x5f, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0xc3, 0xb0, 0x73,
|
|
||||||
0x63, 0x61, 0x6e, 0x5f, 0x70, 0x6c, 0x61, 0x6e, 0x5f, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0xdc, 0x00, 0x20, 0x03, 0x03,
|
|
||||||
0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03,
|
|
||||||
0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0xb0, 0x63, 0x61, 0x63, 0x68, 0x65, 0x5f, 0x6b, 0x65, 0x79,
|
|
||||||
0x5f, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x01, 0x81, 0xab, 0x77, 0x69, 0x72, 0x65, 0x2d, 0x62, 0x75, 0x63, 0x6b, 0x65,
|
|
||||||
0x74, 0x8b, 0xa8, 0x63, 0x68, 0x69, 0x6c, 0x64, 0x72, 0x65, 0x6e, 0x90, 0xa4, 0x73, 0x69, 0x7a, 0x65, 0xcd, 0x10, 0x00,
|
|
||||||
0xa7, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x03, 0xa8, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x05, 0xae,
|
|
||||||
0x64, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x5f, 0x6d, 0x61, 0x72, 0x6b, 0x65, 0x72, 0x73, 0x01, 0xa9, 0x6f, 0x62, 0x6a, 0x5f,
|
|
||||||
0x73, 0x69, 0x7a, 0x65, 0x73, 0x9b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xac, 0x6f, 0x62,
|
|
||||||
0x6a, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x97, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb1, 0x72,
|
|
||||||
0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x73, 0xc0, 0xa9, 0x63, 0x6f,
|
|
||||||
0x6d, 0x70, 0x61, 0x63, 0x74, 0x65, 0x64, 0xc3, 0xae, 0x66, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x5f, 0x6f, 0x62, 0x6a, 0x65,
|
|
||||||
0x63, 0x74, 0x73, 0x02, 0xae, 0x61, 0x6c, 0x6c, 0x5f, 0x74, 0x69, 0x65, 0x72, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x73, 0x91,
|
|
||||||
0x81, 0xa4, 0x57, 0x41, 0x52, 0x4d, 0x93, 0xcd, 0x08, 0x00, 0x02, 0x01,
|
|
||||||
];
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn thin_usage_cache_decodes_scanner_wire_fixture() {
|
|
||||||
let decoded =
|
|
||||||
DataUsageCache::unmarshal(SCANNER_USAGE_CACHE_WIRE_FIXTURE).expect("thin projection decodes a scanner-written cache");
|
|
||||||
|
|
||||||
// The six fields shared with the scanner's 16-field info block; the
|
|
||||||
// remaining ten (lifecycle, replication, checkpoint, heals, ...) must
|
|
||||||
// be skipped, not error.
|
|
||||||
assert_eq!(decoded.info.name, "wire-bucket");
|
|
||||||
assert_eq!(decoded.info.next_cycle, 7);
|
|
||||||
assert_eq!(
|
|
||||||
decoded.info.last_update,
|
|
||||||
Some(SystemTime::UNIX_EPOCH + Duration::from_secs(1_700_000_000))
|
|
||||||
);
|
|
||||||
assert!(decoded.info.skip_healing);
|
|
||||||
assert_eq!(decoded.info.failed_objects.get("wire-bucket/lost"), Some(&11));
|
|
||||||
assert!(decoded.info.snapshot_complete);
|
|
||||||
|
|
||||||
// Entries use the shared canonical map-encoded type end to end.
|
|
||||||
let entry = decoded.cache.get("wire-bucket").expect("fixture entry decodes");
|
|
||||||
assert_eq!(entry.size, 4096);
|
|
||||||
assert_eq!(entry.objects, 3);
|
|
||||||
assert_eq!(entry.versions, 5);
|
|
||||||
assert_eq!(entry.delete_markers, 1);
|
|
||||||
assert!(entry.compacted);
|
|
||||||
assert_eq!(entry.failed_objects, 2);
|
|
||||||
assert_eq!(
|
|
||||||
entry.all_tier_stats.as_ref().and_then(|tiers| tiers.tiers.get("WARM")),
|
|
||||||
Some(&TierStats {
|
|
||||||
total_size: 2048,
|
|
||||||
num_versions: 2,
|
|
||||||
num_objects: 1,
|
|
||||||
})
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Build a cache shaped like `bucket/{a,b/{c,d}},bucket/loose` with
|
|
||||||
/// distinct counters so aggregation is observable.
|
|
||||||
fn prefix_usage_fixture_cache() -> DataUsageCache {
|
|
||||||
let mut cache = DataUsageCache::default();
|
|
||||||
let mut insert = |path: &str, parent: &str, size: usize, objects: usize, versions: usize, delete_markers: usize| {
|
|
||||||
cache.replace(
|
|
||||||
path,
|
|
||||||
parent,
|
|
||||||
DataUsageEntry {
|
|
||||||
size,
|
|
||||||
objects,
|
|
||||||
versions,
|
|
||||||
delete_markers,
|
|
||||||
..Default::default()
|
|
||||||
},
|
|
||||||
);
|
|
||||||
};
|
|
||||||
insert("bucket", "", 0, 0, 0, 0);
|
|
||||||
insert("bucket/a", "bucket", 100, 1, 1, 0);
|
|
||||||
insert("bucket/b", "bucket", 0, 0, 0, 0);
|
|
||||||
insert("bucket/b/c", "bucket/b", 200, 2, 2, 1);
|
|
||||||
insert("bucket/b/d", "bucket/b", 40, 1, 3, 0);
|
|
||||||
insert("bucket/loose", "bucket", 10, 1, 1, 1);
|
|
||||||
cache
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn prefix_usage_aggregates_bucket_root_and_one_level_below() {
|
|
||||||
let cache = prefix_usage_fixture_cache();
|
|
||||||
|
|
||||||
let root = cache
|
|
||||||
.prefix_usage("bucket", "", 100)
|
|
||||||
.expect("root query must find the bucket entry");
|
|
||||||
assert_eq!(root.usage.size, 350, "root aggregate flattens the whole subtree");
|
|
||||||
assert_eq!(root.usage.objects, 5);
|
|
||||||
assert_eq!(root.usage.versions, 7);
|
|
||||||
assert_eq!(root.usage.delete_markers, 2);
|
|
||||||
assert!(!root.compacted);
|
|
||||||
assert!(!root.truncated);
|
|
||||||
// Breakdown is one level: b (240) before a (100) before loose (10),
|
|
||||||
// each flattened to its own subtree total.
|
|
||||||
let names: Vec<(&str, u64)> = root
|
|
||||||
.sub_prefixes
|
|
||||||
.iter()
|
|
||||||
.map(|entry| (entry.prefix.as_str(), entry.usage.size))
|
|
||||||
.collect();
|
|
||||||
assert_eq!(names, vec![("b", 240), ("a", 100), ("loose", 10)]);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn prefix_usage_drills_into_arbitrary_prefixes() {
|
|
||||||
let cache = prefix_usage_fixture_cache();
|
|
||||||
|
|
||||||
let b = cache.prefix_usage("bucket", "b", 100).expect("nested prefix must resolve");
|
|
||||||
assert_eq!(b.usage.size, 240);
|
|
||||||
assert_eq!(b.usage.versions, 5);
|
|
||||||
let names: Vec<&str> = b.sub_prefixes.iter().map(|entry| entry.prefix.as_str()).collect();
|
|
||||||
assert_eq!(names, vec!["c", "d"]);
|
|
||||||
|
|
||||||
// Prefix slashes are normalized away.
|
|
||||||
let slashed = cache.prefix_usage("bucket", "/b/", 100).expect("slash-insensitive lookup");
|
|
||||||
assert_eq!(slashed.usage.size, 240);
|
|
||||||
|
|
||||||
assert!(cache.prefix_usage("bucket", "absent", 100).is_none(), "unknown prefix must be a miss");
|
|
||||||
assert!(cache.prefix_usage("other", "", 100).is_none(), "unknown bucket must be a miss");
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn prefix_usage_reports_and_respects_truncation() {
|
|
||||||
let cache = prefix_usage_fixture_cache();
|
|
||||||
let capped = cache.prefix_usage("bucket", "", 2).expect("root query");
|
|
||||||
assert!(capped.truncated, "three children capped to two must flag truncation");
|
|
||||||
let names: Vec<&str> = capped.sub_prefixes.iter().map(|entry| entry.prefix.as_str()).collect();
|
|
||||||
assert_eq!(names, vec!["b", "a"], "largest prefixes survive the cut");
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn prefix_usage_marks_compacted_entries() {
|
|
||||||
let mut cache = DataUsageCache::default();
|
|
||||||
cache.replace(
|
|
||||||
"bucket",
|
|
||||||
"",
|
|
||||||
DataUsageEntry {
|
|
||||||
size: 999,
|
|
||||||
objects: 9,
|
|
||||||
compacted: true,
|
|
||||||
..Default::default()
|
|
||||||
},
|
|
||||||
);
|
|
||||||
|
|
||||||
let compacted = cache.prefix_usage("bucket", "", 100).expect("compacted root resolves");
|
|
||||||
assert!(compacted.compacted, "compaction must be visible to callers");
|
|
||||||
assert_eq!(compacted.usage.size, 999);
|
|
||||||
assert!(compacted.sub_prefixes.is_empty(), "a compacted entry carries no children");
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn prefix_usage_rejects_cyclic_and_dangling_caches() {
|
|
||||||
// A self-referencing child (corrupt cache) must yield a miss for the
|
|
||||||
// whole query, not unbounded recursion.
|
|
||||||
let mut cache = prefix_usage_fixture_cache();
|
|
||||||
if let Some(entry) = cache.cache.get_mut("bucket/b") {
|
|
||||||
entry.children.insert("bucket/b".to_string());
|
|
||||||
}
|
|
||||||
assert!(cache.prefix_usage("bucket", "b", 100).is_none(), "a cyclic subtree must be rejected");
|
|
||||||
// The unaffected sibling still answers.
|
|
||||||
assert!(cache.prefix_usage("bucket", "a", 100).is_some());
|
|
||||||
|
|
||||||
// A child key with no entry (dangling link) is rejected rather than
|
|
||||||
// silently dropped: half a tree would under-report usage.
|
|
||||||
let mut dangling = prefix_usage_fixture_cache();
|
|
||||||
if let Some(entry) = dangling.cache.get_mut("bucket/b") {
|
|
||||||
entry.children.insert("bucket/b/ghost".to_string());
|
|
||||||
}
|
|
||||||
assert!(
|
|
||||||
dangling.prefix_usage("bucket", "b", 100).is_none(),
|
|
||||||
"a dangling child link must be rejected"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn hash_path_uses_portable_slash_semantics() {
|
fn hash_path_uses_portable_slash_semantics() {
|
||||||
for (input, expected) in [
|
for (input, expected) in [
|
||||||
@@ -2318,64 +1991,6 @@ mod tests {
|
|||||||
assert_eq!(usage1.versions_count, 15);
|
assert_eq!(usage1.versions_count, 15);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn size_summary_add_saturates_instead_of_overflowing() {
|
|
||||||
// The scanner folds one summary per object into a per-prefix total, so a
|
|
||||||
// counter at its ceiling must stay there rather than panic in a debug
|
|
||||||
// build or wrap in a release one (backlog#1828).
|
|
||||||
let mut summary = SizeSummary {
|
|
||||||
total_size: usize::MAX,
|
|
||||||
versions: usize::MAX,
|
|
||||||
replicated_size: i64::MAX,
|
|
||||||
pending_size: i64::MAX,
|
|
||||||
failed_size: i64::MAX,
|
|
||||||
replica_size: i64::MAX,
|
|
||||||
..Default::default()
|
|
||||||
};
|
|
||||||
summary.repl_target_stats.insert(
|
|
||||||
"arn".to_string(),
|
|
||||||
ReplTargetSizeSummary {
|
|
||||||
replicated_size: i64::MAX,
|
|
||||||
pending_size: i64::MAX,
|
|
||||||
failed_size: i64::MAX,
|
|
||||||
..Default::default()
|
|
||||||
},
|
|
||||||
);
|
|
||||||
|
|
||||||
let mut increment = SizeSummary {
|
|
||||||
total_size: 1,
|
|
||||||
versions: 1,
|
|
||||||
replicated_size: 1,
|
|
||||||
pending_size: 1,
|
|
||||||
failed_size: 1,
|
|
||||||
replica_size: 1,
|
|
||||||
..Default::default()
|
|
||||||
};
|
|
||||||
increment.repl_target_stats.insert(
|
|
||||||
"arn".to_string(),
|
|
||||||
ReplTargetSizeSummary {
|
|
||||||
replicated_size: 1,
|
|
||||||
pending_size: 1,
|
|
||||||
failed_size: 1,
|
|
||||||
..Default::default()
|
|
||||||
},
|
|
||||||
);
|
|
||||||
|
|
||||||
summary.add(&increment);
|
|
||||||
|
|
||||||
assert_eq!(summary.total_size, usize::MAX);
|
|
||||||
assert_eq!(summary.versions, usize::MAX);
|
|
||||||
assert_eq!(summary.replicated_size, i64::MAX);
|
|
||||||
assert_eq!(summary.pending_size, i64::MAX);
|
|
||||||
assert_eq!(summary.failed_size, i64::MAX);
|
|
||||||
assert_eq!(summary.replica_size, i64::MAX);
|
|
||||||
|
|
||||||
let target = summary.repl_target_stats.get("arn").expect("target survives the merge");
|
|
||||||
assert_eq!(target.replicated_size, i64::MAX);
|
|
||||||
assert_eq!(target.pending_size, i64::MAX);
|
|
||||||
assert_eq!(target.failed_size, i64::MAX);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_size_summary_add() {
|
fn test_size_summary_add() {
|
||||||
let mut summary1 = SizeSummary::new();
|
let mut summary1 = SizeSummary::new();
|
||||||
|
|||||||
+21
-26
@@ -48,14 +48,16 @@ cargo nextest run --profile e2e-smoke -p e2e_test
|
|||||||
cargo nextest run -j1 --run-ignored ignored-only -p rustfs-scanner -p rustfs \
|
cargo nextest run -j1 --run-ignored ignored-only -p rustfs-scanner -p rustfs \
|
||||||
-E 'binary(lifecycle_integration_test) or (package(rustfs) and test(lifecycle_transition_api_test))'
|
-E 'binary(lifecycle_integration_test) or (package(rustfs) and test(lifecycle_transition_api_test))'
|
||||||
|
|
||||||
|
# Protocols suite — fixed ports, MUST be single-threaded, gated by build features
|
||||||
|
RUSTFS_BUILD_FEATURES=ftps,webdav,sftp \
|
||||||
|
cargo test -p e2e_test test_protocol_core_suite -- --test-threads=1 --nocapture
|
||||||
```
|
```
|
||||||
|
|
||||||
The protocols suite has its own contract (fixed bind ports 9022–9301,
|
The protocols suite has its own contract (fixed bind ports 9022–9301,
|
||||||
single-worker execution, feature-gated scheduling) documented in
|
`--test-threads=1`, feature-gated scheduling) documented in
|
||||||
[`src/protocols/README.md`](src/protocols/README.md). `RUSTFS_BUILD_FEATURES`
|
[`src/protocols/README.md`](src/protocols/README.md). `RUSTFS_BUILD_FEATURES`
|
||||||
selects which features the spawned binary is built with; leave it unset to run
|
selects which features the spawned binary is built with; leave it unset to run
|
||||||
every protocol entry. Use the exact profile command under
|
every protocol entry.
|
||||||
[Troubleshooting](#troubleshooting) for CI-equivalent execution.
|
|
||||||
|
|
||||||
### `#[ignore]` semantics
|
### `#[ignore]` semantics
|
||||||
|
|
||||||
@@ -157,26 +159,27 @@ construction (random port + isolated temp dir) and need no serialization.
|
|||||||
## CI map
|
## CI map
|
||||||
|
|
||||||
`e2e_test` is **excluded** from the main `cargo nextest run --profile ci --all`
|
`e2e_test` is **excluded** from the main `cargo nextest run --profile ci --all`
|
||||||
pass (`--exclude e2e_test`) — the whole crate is too slow to gate every PR.
|
pass ([`.github/workflows/ci.yml`](../../.github/workflows/ci.yml) line 158,
|
||||||
Subsets join CI through nextest profiles; the fixed-port protocol suite uses
|
`--exclude e2e_test`) — the whole crate is too slow to gate every PR. Subsets
|
||||||
the same profile for membership and execution with one nightly worker.
|
join CI through the nextest profile system only (never as ad-hoc jobs):
|
||||||
|
|
||||||
| Suite | Runs where | Status |
|
| Suite | Runs where | Status |
|
||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
| Smoke subset (`e2e-smoke` profile) | `e2e-tests` job, every PR | **Active** (backlog#1149 ci-4) |
|
| Smoke subset (`e2e-smoke` profile) | `e2e-tests` job, every PR | **Active** (backlog#1149 ci-4) |
|
||||||
| Full single-node suite (`e2e-full` profile) | `e2e-full` job, merge queue + main | **Active** (backlog#1149 ci-5) |
|
|
||||||
| `s3s-e2e` black-box | `e2e-tests` + `e2e-tests-rio-v2` jobs | **Active** (external conformance tool) |
|
| `s3s-e2e` black-box | `e2e-tests` + `e2e-tests-rio-v2` jobs | **Active** (external conformance tool) |
|
||||||
| ILM / lifecycle (ignored) | `test-ilm-integration-serial` lane, `-j1` | **Active** (backlog#1148 ilm-1) |
|
| ILM / lifecycle (ignored) | `test-ilm-integration-serial` lane, `-j1` | **Active** (backlog#1148 ilm-1) |
|
||||||
| KMS suite | `e2e-full` job, merge queue + main | **Active** |
|
| KMS suite | — | Not in CI yet (backlog#1149 ci-5) |
|
||||||
| Cluster faults (`e2e-nightly` profile) | consolidated nightly workflow | **Active** (backlog#1149 ci-7) |
|
| Protocols (FTPS/WebDAV/SFTP) | — | Not in CI yet (backlog#1149 ci-7) |
|
||||||
| Protocols (FTPS/WebDAV/SFTP) | consolidated nightly workflow, serial | **Active** (backlog#1149 ci-7) |
|
|
||||||
| Replication (fast subset) | `e2e-smoke` profile, `e2e-tests` job, every PR | **Active** (backlog#1147 repl-1) |
|
| Replication (fast subset) | `e2e-smoke` profile, `e2e-tests` job, every PR | **Active** (backlog#1147 repl-1) |
|
||||||
| Replication (slow + multi-node) | `e2e-repl-nightly` profile, consolidated nightly workflow | **Active** (backlog#1147 repl-1) |
|
| Replication (slow + dual-node) | `e2e-repl-nightly` profile, scheduled workflow | **Active** (backlog#1147 repl-1) |
|
||||||
| `reliant/*` | 19 tests in PR smoke; remaining default tests in `e2e-full` | **Active** except `#[ignore]` |
|
| `reliant/*` (pre-started server) | — | Manual only |
|
||||||
|
|
||||||
The profile filters in [`.config/nextest.toml`](../../.config/nextest.toml) are
|
Links: [`ci.yml`](../../.github/workflows/ci.yml) `e2e-tests` (line 347),
|
||||||
the wiring source of truth. Committed test-ID digests under
|
`test-ilm-integration-serial` (line 196). The `e2e-smoke` `default-filter` in
|
||||||
`.config/e2e-*-selection.txt` make every membership change explicit.
|
[`.config/nextest.toml`](../../.config/nextest.toml) is the **single wiring
|
||||||
|
mechanism** — extend that filter (or add a sibling profile) to admit more
|
||||||
|
tests; do not add e2e jobs to `ci.yml`. repl-1 / ilm-3 are landing in parallel
|
||||||
|
and may add lanes; keep the table above easy to extend.
|
||||||
|
|
||||||
## Troubleshooting
|
## Troubleshooting
|
||||||
|
|
||||||
@@ -185,15 +188,9 @@ the wiring source of truth. Committed test-ID digests under
|
|||||||
```bash
|
```bash
|
||||||
# Smoke (e2e-tests job) — includes the 20 fast replication tests
|
# Smoke (e2e-tests job) — includes the 20 fast replication tests
|
||||||
cargo nextest run --profile e2e-smoke -p e2e_test
|
cargo nextest run --profile e2e-smoke -p e2e_test
|
||||||
# Full single-node merge/main lane
|
# Replication nightly lane (16 slow + dual-node tests; install awscurl for the
|
||||||
cargo nextest run --profile e2e-full -p e2e_test
|
# STS dual-node test, else it skips gracefully)
|
||||||
# Cluster fault nightly lane
|
|
||||||
cargo nextest run --profile e2e-nightly -p e2e_test
|
|
||||||
# Replication nightly lane; install awscurl so STS paths do not skip
|
|
||||||
cargo nextest run --profile e2e-repl-nightly -p e2e_test
|
cargo nextest run --profile e2e-repl-nightly -p e2e_test
|
||||||
# Fixed-port protocol nightly lane
|
|
||||||
RUSTFS_BUILD_FEATURES=ftps,webdav,sftp \
|
|
||||||
cargo nextest run -j 1 --profile e2e-protocols -p e2e_test --no-capture
|
|
||||||
# ILM serial lane
|
# ILM serial lane
|
||||||
cargo nextest run -j1 --run-ignored ignored-only -p rustfs-scanner -p rustfs \
|
cargo nextest run -j1 --run-ignored ignored-only -p rustfs-scanner -p rustfs \
|
||||||
-E 'binary(lifecycle_integration_test) or (package(rustfs) and test(lifecycle_transition_api_test))'
|
-E 'binary(lifecycle_integration_test) or (package(rustfs) and test(lifecycle_transition_api_test))'
|
||||||
@@ -276,6 +273,4 @@ current subset is.
|
|||||||
`docs/testing/e2e-suite-inventory.md` records the per-module test counts as
|
`docs/testing/e2e-suite-inventory.md` records the per-module test counts as
|
||||||
listed by `cargo nextest list -p e2e_test`. Regenerate it when adding or
|
listed by `cargo nextest list -p e2e_test`. Regenerate it when adding or
|
||||||
moving e2e tests so acceptance numbers in the test-strategy issues
|
moving e2e tests so acceptance numbers in the test-strategy issues
|
||||||
(backlog#1147–#1155) stay auditable. When a profile membership change is
|
(backlog#1147–#1155) stay auditable.
|
||||||
intentional, review its JSON listing before updating the matching
|
|
||||||
`.config/e2e-*-selection.txt` test-ID digest.
|
|
||||||
|
|||||||
@@ -38,6 +38,7 @@ mod tests {
|
|||||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||||
use rustfs_signer::sign_v4;
|
use rustfs_signer::sign_v4;
|
||||||
use s3s::Body;
|
use s3s::Body;
|
||||||
|
use serial_test::serial;
|
||||||
use std::error::Error;
|
use std::error::Error;
|
||||||
use std::io::Read;
|
use std::io::Read;
|
||||||
use std::process::{Command, Stdio};
|
use std::process::{Command, Stdio};
|
||||||
@@ -161,6 +162,7 @@ mod tests {
|
|||||||
/// A fully authenticated but non-admin credential must be rejected with
|
/// A fully authenticated but non-admin credential must be rejected with
|
||||||
/// `403 AccessDenied` on an admin API, while the root credential succeeds.
|
/// `403 AccessDenied` on an admin API, while the root credential succeeds.
|
||||||
#[tokio::test(flavor = "multi_thread")]
|
#[tokio::test(flavor = "multi_thread")]
|
||||||
|
#[serial]
|
||||||
async fn non_admin_credential_denied_on_admin_api() -> Result<(), Box<dyn Error + Send + Sync>> {
|
async fn non_admin_credential_denied_on_admin_api() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||||
init_logging();
|
init_logging();
|
||||||
let mut env = RustFSTestEnvironment::new().await?;
|
let mut env = RustFSTestEnvironment::new().await?;
|
||||||
@@ -196,6 +198,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test(flavor = "multi_thread")]
|
#[tokio::test(flavor = "multi_thread")]
|
||||||
|
#[serial]
|
||||||
async fn non_admin_credential_denied_on_manual_transition_run() -> Result<(), Box<dyn Error + Send + Sync>> {
|
async fn non_admin_credential_denied_on_manual_transition_run() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||||
init_logging();
|
init_logging();
|
||||||
let mut env = RustFSTestEnvironment::new().await?;
|
let mut env = RustFSTestEnvironment::new().await?;
|
||||||
@@ -323,6 +326,7 @@ mod tests {
|
|||||||
/// credential is accepted and the old one is rejected, on both the S3 data
|
/// credential is accepted and the old one is rejected, on both the S3 data
|
||||||
/// plane and the admin plane.
|
/// plane and the admin plane.
|
||||||
#[tokio::test(flavor = "multi_thread")]
|
#[tokio::test(flavor = "multi_thread")]
|
||||||
|
#[serial]
|
||||||
async fn root_credential_rotation_takes_effect() -> Result<(), Box<dyn Error + Send + Sync>> {
|
async fn root_credential_rotation_takes_effect() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||||
init_logging();
|
init_logging();
|
||||||
let mut env = RustFSTestEnvironment::new().await?;
|
let mut env = RustFSTestEnvironment::new().await?;
|
||||||
@@ -385,6 +389,7 @@ mod tests {
|
|||||||
/// runtime. We capture the child's stdout/stderr directly (the shared
|
/// runtime. We capture the child's stdout/stderr directly (the shared
|
||||||
/// harness inherits stdio) and poll for the warning until it appears.
|
/// harness inherits stdio) and poll for the warning until it appears.
|
||||||
#[tokio::test(flavor = "multi_thread")]
|
#[tokio::test(flavor = "multi_thread")]
|
||||||
|
#[serial]
|
||||||
async fn default_credentials_emit_startup_warning() -> Result<(), Box<dyn Error + Send + Sync>> {
|
async fn default_credentials_emit_startup_warning() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||||
init_logging();
|
init_logging();
|
||||||
|
|
||||||
|
|||||||
@@ -33,6 +33,7 @@ use aws_sdk_s3::config::{Credentials, Region};
|
|||||||
use aws_sdk_s3::primitives::ByteStream;
|
use aws_sdk_s3::primitives::ByteStream;
|
||||||
use aws_sdk_s3::{Client, Config};
|
use aws_sdk_s3::{Client, Config};
|
||||||
use reqwest::StatusCode;
|
use reqwest::StatusCode;
|
||||||
|
use serial_test::serial;
|
||||||
use std::error::Error;
|
use std::error::Error;
|
||||||
use tokio::time::{Duration, sleep};
|
use tokio::time::{Duration, sleep};
|
||||||
|
|
||||||
@@ -177,6 +178,7 @@ async fn assert_admin_status(
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn test_update_service_account_enforces_owner_and_parent_scope() -> TestResult {
|
async fn test_update_service_account_enforces_owner_and_parent_scope() -> TestResult {
|
||||||
init_logging();
|
init_logging();
|
||||||
|
|
||||||
@@ -346,6 +348,7 @@ async fn test_update_service_account_enforces_owner_and_parent_scope() -> TestRe
|
|||||||
/// Full user -> policy -> service-account lifecycle, proving each management
|
/// Full user -> policy -> service-account lifecycle, proving each management
|
||||||
/// call takes effect on the data plane, not just that the endpoint answers 200.
|
/// call takes effect on the data plane, not just that the endpoint answers 200.
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn test_admin_user_policy_service_account_crud_lifecycle() -> TestResult {
|
async fn test_admin_user_policy_service_account_crud_lifecycle() -> TestResult {
|
||||||
init_logging();
|
init_logging();
|
||||||
|
|
||||||
@@ -570,6 +573,7 @@ async fn test_admin_user_policy_service_account_crud_lifecycle() -> TestResult {
|
|||||||
/// non-admin credential with 403 AccessDenied (sec-4 assertion pattern; the
|
/// non-admin credential with 403 AccessDenied (sec-4 assertion pattern; the
|
||||||
/// gate implementation itself is owned by sec-4 / admin_auth_test).
|
/// gate implementation itself is owned by sec-4 / admin_auth_test).
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn test_admin_iam_endpoints_deny_non_admin_credential() -> TestResult {
|
async fn test_admin_iam_endpoints_deny_non_admin_credential() -> TestResult {
|
||||||
init_logging();
|
init_logging();
|
||||||
|
|
||||||
|
|||||||
@@ -21,6 +21,7 @@ use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
|||||||
use rustfs_signer::sign_v4;
|
use rustfs_signer::sign_v4;
|
||||||
use s3s::Body;
|
use s3s::Body;
|
||||||
use serde::Deserialize;
|
use serde::Deserialize;
|
||||||
|
use serial_test::serial;
|
||||||
use std::error::Error;
|
use std::error::Error;
|
||||||
use std::process::Command;
|
use std::process::Command;
|
||||||
use tokio::time::{Duration, sleep, timeout};
|
use tokio::time::{Duration, sleep, timeout};
|
||||||
@@ -99,6 +100,7 @@ fn offline_server_count(info: &InfoMessage) -> usize {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test(flavor = "multi_thread")]
|
#[tokio::test(flavor = "multi_thread")]
|
||||||
|
#[serial]
|
||||||
async fn test_single_admin_timeout_does_not_immediately_mark_peer_offline() -> Result<(), Box<dyn Error + Send + Sync>> {
|
async fn test_single_admin_timeout_does_not_immediately_mark_peer_offline() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||||
init_logging();
|
init_logging();
|
||||||
|
|
||||||
|
|||||||
@@ -18,6 +18,7 @@
|
|||||||
|
|
||||||
use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
|
use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
|
||||||
use aws_sdk_s3::types::PublicAccessBlockConfiguration;
|
use aws_sdk_s3::types::PublicAccessBlockConfiguration;
|
||||||
|
use serial_test::serial;
|
||||||
use tracing::info;
|
use tracing::info;
|
||||||
|
|
||||||
async fn setup_public_bucket(
|
async fn setup_public_bucket(
|
||||||
@@ -72,6 +73,7 @@ async fn anonymous_get_object(
|
|||||||
/// Issue #2036: Anonymous GetObject should succeed when bucket policy allows it
|
/// Issue #2036: Anonymous GetObject should succeed when bucket policy allows it
|
||||||
/// and no PublicAccessBlock configuration exists (ConfigNotFound).
|
/// and no PublicAccessBlock configuration exists (ConfigNotFound).
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn test_anonymous_access_allowed_when_public_access_block_missing() -> Result<(), Box<dyn std::error::Error + Send + Sync>>
|
async fn test_anonymous_access_allowed_when_public_access_block_missing() -> Result<(), Box<dyn std::error::Error + Send + Sync>>
|
||||||
{
|
{
|
||||||
init_logging();
|
init_logging();
|
||||||
@@ -98,6 +100,7 @@ async fn test_anonymous_access_allowed_when_public_access_block_missing() -> Res
|
|||||||
|
|
||||||
/// Anonymous GetObject should be denied when RestrictPublicBuckets is true.
|
/// Anonymous GetObject should be denied when RestrictPublicBuckets is true.
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn test_anonymous_access_denied_when_restrict_public_buckets_enabled()
|
async fn test_anonymous_access_denied_when_restrict_public_buckets_enabled()
|
||||||
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||||
init_logging();
|
init_logging();
|
||||||
@@ -134,6 +137,7 @@ async fn test_anonymous_access_denied_when_restrict_public_buckets_enabled()
|
|||||||
/// Anonymous GetObject should succeed when PublicAccessBlock exists but
|
/// Anonymous GetObject should succeed when PublicAccessBlock exists but
|
||||||
/// RestrictPublicBuckets is explicitly false.
|
/// RestrictPublicBuckets is explicitly false.
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn test_anonymous_access_allowed_when_restrict_public_buckets_disabled()
|
async fn test_anonymous_access_allowed_when_restrict_public_buckets_disabled()
|
||||||
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||||
init_logging();
|
init_logging();
|
||||||
@@ -172,6 +176,7 @@ async fn test_anonymous_access_allowed_when_restrict_public_buckets_disabled()
|
|||||||
/// reaches authorization through a fallback branch, and that branch has to apply the
|
/// reaches authorization through a fallback branch, and that branch has to apply the
|
||||||
/// same public-access gate as a direct grant.
|
/// same public-access gate as a direct grant.
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn ghsa_x298_anonymous_list_object_versions_denied_when_restrict_public_buckets_enabled()
|
async fn ghsa_x298_anonymous_list_object_versions_denied_when_restrict_public_buckets_enabled()
|
||||||
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||||
init_logging();
|
init_logging();
|
||||||
|
|||||||
@@ -18,11 +18,13 @@
|
|||||||
//! completely inert with default configuration.
|
//! completely inert with default configuration.
|
||||||
|
|
||||||
use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
|
use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
|
||||||
|
use serial_test::serial;
|
||||||
use tracing::info;
|
use tracing::info;
|
||||||
|
|
||||||
type TestResult = Result<(), Box<dyn std::error::Error + Send + Sync>>;
|
type TestResult = Result<(), Box<dyn std::error::Error + Send + Sync>>;
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn api_rate_limit_enforces_429_with_retry_after_when_enabled() -> TestResult {
|
async fn api_rate_limit_enforces_429_with_retry_after_when_enabled() -> TestResult {
|
||||||
init_logging();
|
init_logging();
|
||||||
let mut env = RustFSTestEnvironment::new().await?;
|
let mut env = RustFSTestEnvironment::new().await?;
|
||||||
@@ -85,6 +87,7 @@ async fn api_rate_limit_enforces_429_with_retry_after_when_enabled() -> TestResu
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn api_rate_limit_bucket_dimension_throttles_per_bucket() -> TestResult {
|
async fn api_rate_limit_bucket_dimension_throttles_per_bucket() -> TestResult {
|
||||||
init_logging();
|
init_logging();
|
||||||
let mut env = RustFSTestEnvironment::new().await?;
|
let mut env = RustFSTestEnvironment::new().await?;
|
||||||
@@ -130,6 +133,7 @@ async fn api_rate_limit_bucket_dimension_throttles_per_bucket() -> TestResult {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn api_rate_limit_stays_inert_by_default() -> TestResult {
|
async fn api_rate_limit_stays_inert_by_default() -> TestResult {
|
||||||
init_logging();
|
init_logging();
|
||||||
let mut env = RustFSTestEnvironment::new().await?;
|
let mut env = RustFSTestEnvironment::new().await?;
|
||||||
|
|||||||
@@ -24,6 +24,7 @@ mod tests {
|
|||||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||||
use rustfs_signer::{pre_sign_v4, sign_v4};
|
use rustfs_signer::{pre_sign_v4, sign_v4};
|
||||||
use s3s::Body;
|
use s3s::Body;
|
||||||
|
use serial_test::serial;
|
||||||
use sha2::{Digest, Sha256};
|
use sha2::{Digest, Sha256};
|
||||||
use std::error::Error;
|
use std::error::Error;
|
||||||
use std::io::{Cursor, Write};
|
use std::io::{Cursor, Write};
|
||||||
@@ -338,6 +339,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn test_archive_put_allows_content_encoding_by_default() -> Result<(), Box<dyn Error + Send + Sync>> {
|
async fn test_archive_put_allows_content_encoding_by_default() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||||
init_logging();
|
init_logging();
|
||||||
let mut env = RustFSTestEnvironment::new().await?;
|
let mut env = RustFSTestEnvironment::new().await?;
|
||||||
@@ -365,6 +367,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn test_archive_put_rejects_content_encoding_when_strict_mode_enabled() -> Result<(), Box<dyn Error + Send + Sync>> {
|
async fn test_archive_put_rejects_content_encoding_when_strict_mode_enabled() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||||
init_logging();
|
init_logging();
|
||||||
let mut env = RustFSTestEnvironment::new().await?;
|
let mut env = RustFSTestEnvironment::new().await?;
|
||||||
@@ -388,6 +391,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn test_archive_put_with_aws_chunked_does_not_persist_content_encoding_by_default()
|
async fn test_archive_put_with_aws_chunked_does_not_persist_content_encoding_by_default()
|
||||||
-> Result<(), Box<dyn Error + Send + Sync>> {
|
-> Result<(), Box<dyn Error + Send + Sync>> {
|
||||||
init_logging();
|
init_logging();
|
||||||
@@ -423,6 +427,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn test_archive_put_with_aws_chunked_and_effective_encoding_roundtrips_by_default()
|
async fn test_archive_put_with_aws_chunked_and_effective_encoding_roundtrips_by_default()
|
||||||
-> Result<(), Box<dyn Error + Send + Sync>> {
|
-> Result<(), Box<dyn Error + Send + Sync>> {
|
||||||
init_logging();
|
init_logging();
|
||||||
@@ -458,6 +463,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn test_archive_put_with_aws_chunked_allowed_when_strict_mode_enabled() -> Result<(), Box<dyn Error + Send + Sync>> {
|
async fn test_archive_put_with_aws_chunked_allowed_when_strict_mode_enabled() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||||
init_logging();
|
init_logging();
|
||||||
let mut env = RustFSTestEnvironment::new().await?;
|
let mut env = RustFSTestEnvironment::new().await?;
|
||||||
@@ -492,6 +498,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn test_archive_put_with_aws_chunked_and_effective_encoding_rejects_when_strict_mode_enabled()
|
async fn test_archive_put_with_aws_chunked_and_effective_encoding_rejects_when_strict_mode_enabled()
|
||||||
-> Result<(), Box<dyn Error + Send + Sync>> {
|
-> Result<(), Box<dyn Error + Send + Sync>> {
|
||||||
init_logging();
|
init_logging();
|
||||||
@@ -522,6 +529,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn test_archive_download_roundtrip_with_http_compression_enabled() -> Result<(), Box<dyn Error + Send + Sync>> {
|
async fn test_archive_download_roundtrip_with_http_compression_enabled() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||||
init_logging();
|
init_logging();
|
||||||
let mut env = RustFSTestEnvironment::new().await?;
|
let mut env = RustFSTestEnvironment::new().await?;
|
||||||
@@ -583,6 +591,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn test_archive_multipart_roundtrip_preserves_bytes() -> Result<(), Box<dyn Error + Send + Sync>> {
|
async fn test_archive_multipart_roundtrip_preserves_bytes() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||||
init_logging();
|
init_logging();
|
||||||
let mut env = RustFSTestEnvironment::new().await?;
|
let mut env = RustFSTestEnvironment::new().await?;
|
||||||
@@ -678,6 +687,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn test_multipart_get_ignores_empty_conditional_etag_headers() -> Result<(), Box<dyn Error + Send + Sync>> {
|
async fn test_multipart_get_ignores_empty_conditional_etag_headers() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||||
init_logging();
|
init_logging();
|
||||||
let mut env = RustFSTestEnvironment::new().await?;
|
let mut env = RustFSTestEnvironment::new().await?;
|
||||||
@@ -713,6 +723,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn test_archive_multipart_with_aws_chunked_and_effective_encoding_roundtrips_by_default()
|
async fn test_archive_multipart_with_aws_chunked_and_effective_encoding_roundtrips_by_default()
|
||||||
-> Result<(), Box<dyn Error + Send + Sync>> {
|
-> Result<(), Box<dyn Error + Send + Sync>> {
|
||||||
init_logging();
|
init_logging();
|
||||||
@@ -742,6 +753,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn test_archive_multipart_with_aws_chunked_allowed_when_strict_mode_enabled() -> Result<(), Box<dyn Error + Send + Sync>>
|
async fn test_archive_multipart_with_aws_chunked_allowed_when_strict_mode_enabled() -> Result<(), Box<dyn Error + Send + Sync>>
|
||||||
{
|
{
|
||||||
init_logging();
|
init_logging();
|
||||||
@@ -771,6 +783,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn test_archive_multipart_with_aws_chunked_and_effective_encoding_rejects_when_strict_mode_enabled()
|
async fn test_archive_multipart_with_aws_chunked_and_effective_encoding_rejects_when_strict_mode_enabled()
|
||||||
-> Result<(), Box<dyn Error + Send + Sync>> {
|
-> Result<(), Box<dyn Error + Send + Sync>> {
|
||||||
init_logging();
|
init_logging();
|
||||||
@@ -803,6 +816,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn test_presigned_get_and_reverse_proxy_preserve_multipart_bytes() -> Result<(), Box<dyn Error + Send + Sync>> {
|
async fn test_presigned_get_and_reverse_proxy_preserve_multipart_bytes() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||||
init_logging();
|
init_logging();
|
||||||
let mut env = RustFSTestEnvironment::new().await?;
|
let mut env = RustFSTestEnvironment::new().await?;
|
||||||
|
|||||||
@@ -24,9 +24,11 @@ mod tests {
|
|||||||
};
|
};
|
||||||
use http::Method;
|
use http::Method;
|
||||||
use http::header::CONTENT_TYPE;
|
use http::header::CONTENT_TYPE;
|
||||||
|
use serial_test::serial;
|
||||||
use tracing::info;
|
use tracing::info;
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn test_dummy_bucket_compatibility_endpoints() {
|
async fn test_dummy_bucket_compatibility_endpoints() {
|
||||||
init_logging();
|
init_logging();
|
||||||
info!("Starting test: dummy-compat bucket APIs should match S3-compatible behavior");
|
info!("Starting test: dummy-compat bucket APIs should match S3-compatible behavior");
|
||||||
@@ -234,6 +236,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn test_dummy_bucket_compatibility_endpoints_no_such_bucket() {
|
async fn test_dummy_bucket_compatibility_endpoints_no_such_bucket() {
|
||||||
init_logging();
|
init_logging();
|
||||||
info!("Starting test: dummy-compat bucket APIs should return NoSuchBucket for missing bucket");
|
info!("Starting test: dummy-compat bucket APIs should return NoSuchBucket for missing bucket");
|
||||||
@@ -389,6 +392,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn test_dummy_bucket_endpoints_http_contracts() {
|
async fn test_dummy_bucket_endpoints_http_contracts() {
|
||||||
init_logging();
|
init_logging();
|
||||||
info!("Starting test: dummy-compat bucket API HTTP contracts");
|
info!("Starting test: dummy-compat bucket API HTTP contracts");
|
||||||
|
|||||||
@@ -18,6 +18,7 @@
|
|||||||
use crate::common::{RustFSTestEnvironment, init_logging};
|
use crate::common::{RustFSTestEnvironment, init_logging};
|
||||||
use aws_sdk_s3::config::{Credentials, Region};
|
use aws_sdk_s3::config::{Credentials, Region};
|
||||||
use aws_sdk_s3::{Client, Config};
|
use aws_sdk_s3::{Client, Config};
|
||||||
|
use serial_test::serial;
|
||||||
use tracing::info;
|
use tracing::info;
|
||||||
|
|
||||||
async fn create_user(
|
async fn create_user(
|
||||||
@@ -50,6 +51,7 @@ fn create_user_client(env: &RustFSTestEnvironment, access_key: &str, secret_key:
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn test_bucket_policy_authenticated_user() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
async fn test_bucket_policy_authenticated_user() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||||
init_logging();
|
init_logging();
|
||||||
if !crate::common::awscurl_available() {
|
if !crate::common::awscurl_available() {
|
||||||
|
|||||||
@@ -35,6 +35,7 @@ mod tests {
|
|||||||
use aws_sdk_s3::primitives::ByteStream;
|
use aws_sdk_s3::primitives::ByteStream;
|
||||||
use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
|
use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
|
||||||
use rustfs_data_usage::DataUsageInfo;
|
use rustfs_data_usage::DataUsageInfo;
|
||||||
|
use serial_test::serial;
|
||||||
use std::error::Error;
|
use std::error::Error;
|
||||||
use tokio::time::{Duration, sleep};
|
use tokio::time::{Duration, sleep};
|
||||||
use tracing::info;
|
use tracing::info;
|
||||||
@@ -58,6 +59,7 @@ mod tests {
|
|||||||
/// 3. Query admin data usage API
|
/// 3. Query admin data usage API
|
||||||
/// 4. Verify object count > 0
|
/// 4. Verify object count > 0
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn test_bucket_object_count_updates_after_put() -> TestResult {
|
async fn test_bucket_object_count_updates_after_put() -> TestResult {
|
||||||
init_logging();
|
init_logging();
|
||||||
info!("RT-09: bucket object count updates after PUT");
|
info!("RT-09: bucket object count updates after PUT");
|
||||||
@@ -124,6 +126,7 @@ mod tests {
|
|||||||
/// Regression pattern: stats remain unchanged after objects are deleted
|
/// Regression pattern: stats remain unchanged after objects are deleted
|
||||||
/// (rustfs#5615).
|
/// (rustfs#5615).
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn test_bucket_object_count_updates_after_delete() -> TestResult {
|
async fn test_bucket_object_count_updates_after_delete() -> TestResult {
|
||||||
init_logging();
|
init_logging();
|
||||||
info!("RT-09b: bucket object count updates after DELETE");
|
info!("RT-09b: bucket object count updates after DELETE");
|
||||||
@@ -217,6 +220,7 @@ mod tests {
|
|||||||
/// Regression pattern: DataUsageInfo undercounts versioned bucket versions
|
/// Regression pattern: DataUsageInfo undercounts versioned bucket versions
|
||||||
/// and delete markers (rustfs#3898).
|
/// and delete markers (rustfs#3898).
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn test_versioned_bucket_stats_count_all_versions() -> TestResult {
|
async fn test_versioned_bucket_stats_count_all_versions() -> TestResult {
|
||||||
init_logging();
|
init_logging();
|
||||||
info!("RT-09c: versioned bucket stats count all versions");
|
info!("RT-09c: versioned bucket stats count all versions");
|
||||||
|
|||||||
+18
-102
@@ -40,8 +40,7 @@ use http::header::{CONTENT_TYPE, HOST};
|
|||||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||||
use rustfs_signer::sign_v4;
|
use rustfs_signer::sign_v4;
|
||||||
use s3s::Body;
|
use s3s::Body;
|
||||||
use sha2::{Digest, Sha256};
|
use std::collections::BTreeSet;
|
||||||
use std::collections::{BTreeMap, BTreeSet};
|
|
||||||
use std::error::Error;
|
use std::error::Error;
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
use tracing::info;
|
use tracing::info;
|
||||||
@@ -60,26 +59,13 @@ pub(crate) struct VersionShardCensus {
|
|||||||
pub version_id: Option<String>,
|
pub version_id: Option<String>,
|
||||||
pub has_xl_meta: bool,
|
pub has_xl_meta: bool,
|
||||||
pub data_dir: Option<String>,
|
pub data_dir: Option<String>,
|
||||||
pub erasure_index: Option<usize>,
|
|
||||||
pub expected_part_numbers: BTreeSet<usize>,
|
pub expected_part_numbers: BTreeSet<usize>,
|
||||||
pub present_part_fingerprints: BTreeMap<usize, PartShardFingerprint>,
|
pub present_part_numbers: BTreeSet<usize>,
|
||||||
pub inline_data_fingerprint: Option<PartShardFingerprint>,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
|
||||||
pub(crate) struct PartShardFingerprint {
|
|
||||||
pub size: u64,
|
|
||||||
pub sha256: String,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
impl VersionShardCensus {
|
impl VersionShardCensus {
|
||||||
pub(crate) fn is_complete(&self) -> bool {
|
pub(crate) fn is_complete(&self) -> bool {
|
||||||
self.has_xl_meta
|
self.has_xl_meta && self.expected_part_numbers == self.present_part_numbers
|
||||||
&& self.expected_part_numbers.len() == self.present_part_fingerprints.len()
|
|
||||||
&& self
|
|
||||||
.expected_part_numbers
|
|
||||||
.iter()
|
|
||||||
.all(|part_number| self.present_part_fingerprints.contains_key(part_number))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn matches_manifest(&self, manifest: &Self) -> bool {
|
pub(crate) fn matches_manifest(&self, manifest: &Self) -> bool {
|
||||||
@@ -87,25 +73,10 @@ impl VersionShardCensus {
|
|||||||
&& self.is_complete()
|
&& self.is_complete()
|
||||||
&& manifest.is_complete()
|
&& manifest.is_complete()
|
||||||
&& self.data_dir == manifest.data_dir
|
&& self.data_dir == manifest.data_dir
|
||||||
&& self.erasure_index == manifest.erasure_index
|
|
||||||
&& self.expected_part_numbers == manifest.expected_part_numbers
|
&& self.expected_part_numbers == manifest.expected_part_numbers
|
||||||
&& self.present_part_fingerprints == manifest.present_part_fingerprints
|
|
||||||
&& self.inline_data_fingerprint == manifest.inline_data_fingerprint
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn sha256_hex(data: &[u8]) -> String {
|
|
||||||
let digest = Sha256::digest(data);
|
|
||||||
digest.iter().map(|byte| format!("{byte:02x}")).collect()
|
|
||||||
}
|
|
||||||
|
|
||||||
fn shard_fingerprint(data: &[u8]) -> ChaosResult<PartShardFingerprint> {
|
|
||||||
Ok(PartShardFingerprint {
|
|
||||||
size: u64::try_from(data.len())?,
|
|
||||||
sha256: sha256_hex(data),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Single-node RustFS server with `disk_count` local volume directories that
|
/// Single-node RustFS server with `disk_count` local volume directories that
|
||||||
/// can be faulted individually while the server is running.
|
/// can be faulted individually while the server is running.
|
||||||
pub struct DiskFaultHarness {
|
pub struct DiskFaultHarness {
|
||||||
@@ -312,10 +283,8 @@ pub(crate) fn census_object_version_on_disk(
|
|||||||
version_id,
|
version_id,
|
||||||
has_xl_meta: false,
|
has_xl_meta: false,
|
||||||
data_dir: None,
|
data_dir: None,
|
||||||
erasure_index: None,
|
|
||||||
expected_part_numbers: BTreeSet::new(),
|
expected_part_numbers: BTreeSet::new(),
|
||||||
present_part_fingerprints: BTreeMap::new(),
|
present_part_numbers: BTreeSet::new(),
|
||||||
inline_data_fingerprint: None,
|
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -327,31 +296,20 @@ pub(crate) fn census_object_version_on_disk(
|
|||||||
file_info.parts.iter().map(|part| part.number).collect()
|
file_info.parts.iter().map(|part| part.number).collect()
|
||||||
};
|
};
|
||||||
let data_dir = file_info.data_dir.map(|id| id.to_string());
|
let data_dir = file_info.data_dir.map(|id| id.to_string());
|
||||||
let erasure_index = Some(file_info.erasure.index);
|
|
||||||
let inline_data_fingerprint = file_info.data.as_deref().map(shard_fingerprint).transpose()?;
|
|
||||||
let part_dir = data_dir.as_ref().map_or_else(|| object_dir.clone(), |id| object_dir.join(id));
|
let part_dir = data_dir.as_ref().map_or_else(|| object_dir.clone(), |id| object_dir.join(id));
|
||||||
let present_part_fingerprints = match std::fs::read_dir(&part_dir) {
|
let present_part_numbers = match std::fs::read_dir(&part_dir) {
|
||||||
Ok(entries) => {
|
Ok(entries) => entries
|
||||||
let mut fingerprints = BTreeMap::new();
|
.filter_map(Result::ok)
|
||||||
for entry in entries {
|
.filter_map(|entry| {
|
||||||
let entry = entry?;
|
entry
|
||||||
if !entry.file_type()?.is_file() {
|
.file_type()
|
||||||
continue;
|
.ok()
|
||||||
}
|
.filter(|kind| kind.is_file())
|
||||||
let file_name = entry.file_name();
|
.and_then(|_| entry.file_name().to_str().map(str::to_owned))
|
||||||
let Some(part_number) = file_name
|
})
|
||||||
.to_str()
|
.filter_map(|name| name.strip_prefix("part.").and_then(|number| number.parse::<usize>().ok()))
|
||||||
.and_then(|name| name.strip_prefix("part."))
|
.collect(),
|
||||||
.and_then(|number| number.parse::<usize>().ok())
|
Err(error) if error.kind() == std::io::ErrorKind::NotFound => BTreeSet::new(),
|
||||||
else {
|
|
||||||
continue;
|
|
||||||
};
|
|
||||||
let data = std::fs::read(entry.path())?;
|
|
||||||
fingerprints.insert(part_number, shard_fingerprint(&data)?);
|
|
||||||
}
|
|
||||||
fingerprints
|
|
||||||
}
|
|
||||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => BTreeMap::new(),
|
|
||||||
Err(error) => return Err(error.into()),
|
Err(error) => return Err(error.into()),
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -359,10 +317,8 @@ pub(crate) fn census_object_version_on_disk(
|
|||||||
version_id,
|
version_id,
|
||||||
has_xl_meta: true,
|
has_xl_meta: true,
|
||||||
data_dir,
|
data_dir,
|
||||||
erasure_index,
|
|
||||||
expected_part_numbers,
|
expected_part_numbers,
|
||||||
present_part_fingerprints,
|
present_part_numbers,
|
||||||
inline_data_fingerprint,
|
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -402,43 +358,3 @@ pub async fn signed_admin_post(url: &str, body: Option<&str>, access_key: &str,
|
|||||||
|
|
||||||
Ok(body)
|
Ok(body)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
|
||||||
mod tests {
|
|
||||||
use super::*;
|
|
||||||
|
|
||||||
fn complete_census() -> VersionShardCensus {
|
|
||||||
VersionShardCensus {
|
|
||||||
version_id: Some("version".to_string()),
|
|
||||||
has_xl_meta: true,
|
|
||||||
data_dir: Some("data-dir".to_string()),
|
|
||||||
erasure_index: Some(3),
|
|
||||||
expected_part_numbers: BTreeSet::from([1]),
|
|
||||||
present_part_fingerprints: BTreeMap::from([(1, shard_fingerprint(b"part").unwrap())]),
|
|
||||||
inline_data_fingerprint: None,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn shard_fingerprint_uses_physical_length_and_sha256() {
|
|
||||||
assert_eq!(
|
|
||||||
shard_fingerprint(b"abc").unwrap(),
|
|
||||||
PartShardFingerprint {
|
|
||||||
size: 3,
|
|
||||||
sha256: "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad".to_string(),
|
|
||||||
}
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn manifest_requires_matching_inline_payload() {
|
|
||||||
let mut expected = complete_census();
|
|
||||||
expected.expected_part_numbers.clear();
|
|
||||||
expected.present_part_fingerprints.clear();
|
|
||||||
expected.inline_data_fingerprint = Some(shard_fingerprint(b"expected").unwrap());
|
|
||||||
let mut changed = expected.clone();
|
|
||||||
changed.inline_data_fingerprint = Some(shard_fingerprint(b"changed").unwrap());
|
|
||||||
assert!(expected.matches_manifest(&expected));
|
|
||||||
assert!(!changed.matches_manifest(&expected));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -26,6 +26,7 @@ mod tests {
|
|||||||
use base64::Engine;
|
use base64::Engine;
|
||||||
use md5::{Digest as Md5Digest, Md5};
|
use md5::{Digest as Md5Digest, Md5};
|
||||||
use rustfs_rio::{Checksum, ChecksumType as RioChecksumType};
|
use rustfs_rio::{Checksum, ChecksumType as RioChecksumType};
|
||||||
|
use serial_test::serial;
|
||||||
use sha2::Sha256;
|
use sha2::Sha256;
|
||||||
use tracing::info;
|
use tracing::info;
|
||||||
|
|
||||||
@@ -89,6 +90,7 @@ mod tests {
|
|||||||
|
|
||||||
/// PutObject with Content-MD5: upload succeeds and GetObject returns same content.
|
/// PutObject with Content-MD5: upload succeeds and GetObject returns same content.
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn test_put_object_with_content_md5() {
|
async fn test_put_object_with_content_md5() {
|
||||||
init_logging();
|
init_logging();
|
||||||
info!("TEST: PutObject with Content-MD5");
|
info!("TEST: PutObject with Content-MD5");
|
||||||
@@ -124,6 +126,7 @@ mod tests {
|
|||||||
|
|
||||||
/// PutObject with x-amz-checksum-sha256: upload succeeds and GetObject returns same content.
|
/// PutObject with x-amz-checksum-sha256: upload succeeds and GetObject returns same content.
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn test_put_object_with_checksum_sha256() {
|
async fn test_put_object_with_checksum_sha256() {
|
||||||
init_logging();
|
init_logging();
|
||||||
info!("TEST: PutObject with x-amz-checksum-sha256");
|
info!("TEST: PutObject with x-amz-checksum-sha256");
|
||||||
@@ -161,6 +164,7 @@ mod tests {
|
|||||||
/// PutObject with a SHA256 checksum that does NOT match the body must be
|
/// PutObject with a SHA256 checksum that does NOT match the body must be
|
||||||
/// rejected (BadDigest / checksum mismatch), NOT accepted with HTTP 200.
|
/// rejected (BadDigest / checksum mismatch), NOT accepted with HTTP 200.
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn test_put_object_rejects_mismatched_sha256() {
|
async fn test_put_object_rejects_mismatched_sha256() {
|
||||||
init_logging();
|
init_logging();
|
||||||
info!("TEST: PutObject rejects mismatched x-amz-checksum-sha256 (issue #4341)");
|
info!("TEST: PutObject rejects mismatched x-amz-checksum-sha256 (issue #4341)");
|
||||||
@@ -208,6 +212,7 @@ mod tests {
|
|||||||
/// After PutObject with a correct SHA256 checksum, HeadObject with
|
/// After PutObject with a correct SHA256 checksum, HeadObject with
|
||||||
/// ChecksumMode=ENABLED must return that stored base64 SHA256 digest.
|
/// ChecksumMode=ENABLED must return that stored base64 SHA256 digest.
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn test_head_object_returns_stored_sha256() {
|
async fn test_head_object_returns_stored_sha256() {
|
||||||
init_logging();
|
init_logging();
|
||||||
info!("TEST: HeadObject returns stored SHA256 with ChecksumMode=ENABLED (issue #4341)");
|
info!("TEST: HeadObject returns stored SHA256 with ChecksumMode=ENABLED (issue #4341)");
|
||||||
@@ -253,6 +258,7 @@ mod tests {
|
|||||||
/// Multipart upload with checksum: CreateMultipartUpload, UploadPart(s) with checksum_sha256, CompleteMultipartUpload; then GetObject verifies content.
|
/// Multipart upload with checksum: CreateMultipartUpload, UploadPart(s) with checksum_sha256, CompleteMultipartUpload; then GetObject verifies content.
|
||||||
/// Uses part size >= 5MB (server minimum) for two parts.
|
/// Uses part size >= 5MB (server minimum) for two parts.
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn test_multipart_upload_with_checksum() {
|
async fn test_multipart_upload_with_checksum() {
|
||||||
init_logging();
|
init_logging();
|
||||||
info!("TEST: MultipartUpload with checksum (checksum_sha256 on parts)");
|
info!("TEST: MultipartUpload with checksum (checksum_sha256 on parts)");
|
||||||
@@ -350,6 +356,7 @@ mod tests {
|
|||||||
/// Regression test for issue #2282:
|
/// Regression test for issue #2282:
|
||||||
/// CRC64NVME full-object checksum should match between direct PutObject and multipart upload.
|
/// CRC64NVME full-object checksum should match between direct PutObject and multipart upload.
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn test_crc64nvme_matches_between_put_object_and_multipart_upload() {
|
async fn test_crc64nvme_matches_between_put_object_and_multipart_upload() {
|
||||||
init_logging();
|
init_logging();
|
||||||
info!("TEST: CRC64NVME matches between direct PutObject and multipart upload");
|
info!("TEST: CRC64NVME matches between direct PutObject and multipart upload");
|
||||||
@@ -485,6 +492,7 @@ mod tests {
|
|||||||
/// value is rejected with BadDigest and nothing is stored. Full HEAD/GET header
|
/// value is rejected with BadDigest and nothing is stored. Full HEAD/GET header
|
||||||
/// echo round-trip is additionally exercised by the boto3+awscrt e2e.
|
/// echo round-trip is additionally exercised by the boto3+awscrt e2e.
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn test_additional_checksums_verify_on_write() {
|
async fn test_additional_checksums_verify_on_write() {
|
||||||
init_logging();
|
init_logging();
|
||||||
info!("TEST: additional checksums (XXHash3/64/128, SHA-512, MD5) verify-on-write");
|
info!("TEST: additional checksums (XXHash3/64/128, SHA-512, MD5) verify-on-write");
|
||||||
|
|||||||
@@ -16,6 +16,7 @@ use crate::common::RustFSTestClusterEnvironment;
|
|||||||
use aws_sdk_s3::Client;
|
use aws_sdk_s3::Client;
|
||||||
use aws_sdk_s3::error::SdkError;
|
use aws_sdk_s3::error::SdkError;
|
||||||
use bytes::Bytes;
|
use bytes::Bytes;
|
||||||
|
use serial_test::serial;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use tokio::sync::Barrier;
|
use tokio::sync::Barrier;
|
||||||
use tracing::{info, warn};
|
use tracing::{info, warn};
|
||||||
@@ -134,6 +135,7 @@ async fn run_race_iteration(
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn test_conditional_put_race_cluster() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
async fn test_conditional_put_race_cluster() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||||
crate::common::init_logging();
|
crate::common::init_logging();
|
||||||
info!("Starting conditional PUT race test with auto cluster");
|
info!("Starting conditional PUT race test with auto cluster");
|
||||||
@@ -190,6 +192,7 @@ async fn test_conditional_put_race_cluster() -> Result<(), Box<dyn std::error::E
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn test_conditional_put_basic_cluster() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
async fn test_conditional_put_basic_cluster() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||||
crate::common::init_logging();
|
crate::common::init_logging();
|
||||||
info!("Starting basic conditional PUT test with auto cluster");
|
info!("Starting basic conditional PUT test with auto cluster");
|
||||||
|
|||||||
@@ -31,6 +31,7 @@
|
|||||||
//! (toxiproxy / socket proxy) and 5GiB large-object budgets.
|
//! (toxiproxy / socket proxy) and 5GiB large-object budgets.
|
||||||
|
|
||||||
use crate::common::{ClusterTopology, RustFSTestClusterEnvironment};
|
use crate::common::{ClusterTopology, RustFSTestClusterEnvironment};
|
||||||
|
use serial_test::serial;
|
||||||
|
|
||||||
type TestResult = Result<(), Box<dyn std::error::Error + Send + Sync>>;
|
type TestResult = Result<(), Box<dyn std::error::Error + Send + Sync>>;
|
||||||
|
|
||||||
@@ -57,6 +58,7 @@ async fn put_get_roundtrip(cluster: &RustFSTestClusterEnvironment, key: &str, pa
|
|||||||
|
|
||||||
/// 4 nodes x 2 drives, single pool: the multi-drive layout boots and round-trips.
|
/// 4 nodes x 2 drives, single pool: the multi-drive layout boots and round-trips.
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn cluster_multidrive_single_pool_smoke() -> TestResult {
|
async fn cluster_multidrive_single_pool_smoke() -> TestResult {
|
||||||
crate::common::init_logging();
|
crate::common::init_logging();
|
||||||
|
|
||||||
@@ -79,6 +81,7 @@ async fn cluster_multidrive_single_pool_smoke() -> TestResult {
|
|||||||
/// Two single-node pools, 2 drives each: the multi-pool layout boots and
|
/// Two single-node pools, 2 drives each: the multi-pool layout boots and
|
||||||
/// round-trips. Every pool is a distinct erasure pool (`pool_idx` 0 and 1).
|
/// round-trips. Every pool is a distinct erasure pool (`pool_idx` 0 and 1).
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn cluster_two_pool_smoke() -> TestResult {
|
async fn cluster_two_pool_smoke() -> TestResult {
|
||||||
crate::common::init_logging();
|
crate::common::init_logging();
|
||||||
|
|
||||||
|
|||||||
+11
-124
@@ -32,7 +32,6 @@ use rustfs_signer::sign_v4;
|
|||||||
use s3s::Body;
|
use s3s::Body;
|
||||||
use std::ffi::OsStr;
|
use std::ffi::OsStr;
|
||||||
use std::fs as stdfs;
|
use std::fs as stdfs;
|
||||||
use std::io::ErrorKind;
|
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
use std::process::{Child, Command, Stdio};
|
use std::process::{Child, Command, Stdio};
|
||||||
use std::sync::Once;
|
use std::sync::Once;
|
||||||
@@ -52,12 +51,6 @@ pub(crate) const FAST_DATA_USAGE_SCANNER_ENV: &[(&str, &str)] =
|
|||||||
&[("RUSTFS_SCANNER_CYCLE", "1"), ("RUSTFS_SCANNER_START_DELAY_SECS", "0")];
|
&[("RUSTFS_SCANNER_CYCLE", "1"), ("RUSTFS_SCANNER_START_DELAY_SECS", "0")];
|
||||||
pub const TEST_BUCKET: &str = "e2e-test-bucket";
|
pub const TEST_BUCKET: &str = "e2e-test-bucket";
|
||||||
const RUSTFS_FULL_FEATURE: &str = "full";
|
const RUSTFS_FULL_FEATURE: &str = "full";
|
||||||
const TEST_PORT_MIN: u16 = 20_000;
|
|
||||||
// Keep allocator ports below the ephemeral range used by bind(..., 0) test helpers.
|
|
||||||
const TEST_PORT_RANGE: u16 = 10_000;
|
|
||||||
const TEST_PORT_COUNTER_PATH: &str = "/tmp/rustfs_e2e_next_port";
|
|
||||||
const TEST_PORT_LOCK_DIR: &str = "/tmp/rustfs_e2e_port_allocator.lock";
|
|
||||||
const TEST_PORT_LOCK_STALE_AFTER: Duration = Duration::from_secs(30);
|
|
||||||
|
|
||||||
fn capture_log_path(log_dir: &Path, temp_dir: &str) -> Option<PathBuf> {
|
fn capture_log_path(log_dir: &Path, temp_dir: &str) -> Option<PathBuf> {
|
||||||
let temp_name = Path::new(temp_dir).file_name()?.to_string_lossy();
|
let temp_name = Path::new(temp_dir).file_name()?.to_string_lossy();
|
||||||
@@ -74,77 +67,6 @@ fn configured_capture_log_path(temp_dir: &str) -> Option<String> {
|
|||||||
capture_log_path(Path::new(&log_dir), temp_dir).map(|path| path.to_string_lossy().into_owned())
|
capture_log_path(Path::new(&log_dir), temp_dir).map(|path| path.to_string_lossy().into_owned())
|
||||||
}
|
}
|
||||||
|
|
||||||
struct PortAllocatorGuard;
|
|
||||||
|
|
||||||
impl PortAllocatorGuard {
|
|
||||||
async fn acquire() -> Result<Self, Box<dyn std::error::Error + Send + Sync>> {
|
|
||||||
loop {
|
|
||||||
match stdfs::create_dir(TEST_PORT_LOCK_DIR) {
|
|
||||||
Ok(()) => return Ok(Self),
|
|
||||||
Err(err) if err.kind() == ErrorKind::AlreadyExists => {
|
|
||||||
remove_stale_port_allocator_lock();
|
|
||||||
sleep(Duration::from_millis(10)).await;
|
|
||||||
}
|
|
||||||
Err(err) => return Err(err.into()),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Drop for PortAllocatorGuard {
|
|
||||||
fn drop(&mut self) {
|
|
||||||
let _ = stdfs::remove_dir(TEST_PORT_LOCK_DIR);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn advance_test_port(port: u16) -> u16 {
|
|
||||||
let offset = (port - TEST_PORT_MIN + 1) % TEST_PORT_RANGE;
|
|
||||||
TEST_PORT_MIN + offset
|
|
||||||
}
|
|
||||||
|
|
||||||
fn seeded_test_port() -> u16 {
|
|
||||||
let offset = (Uuid::new_v4().as_u128() % u128::from(TEST_PORT_RANGE)) as u16;
|
|
||||||
TEST_PORT_MIN + offset
|
|
||||||
}
|
|
||||||
|
|
||||||
fn read_next_test_port() -> u16 {
|
|
||||||
stdfs::read_to_string(TEST_PORT_COUNTER_PATH)
|
|
||||||
.ok()
|
|
||||||
.and_then(|value| value.trim().parse::<u16>().ok())
|
|
||||||
.filter(|port| (TEST_PORT_MIN..TEST_PORT_MIN + TEST_PORT_RANGE).contains(port))
|
|
||||||
.unwrap_or_else(seeded_test_port)
|
|
||||||
}
|
|
||||||
|
|
||||||
fn remove_stale_port_allocator_lock() {
|
|
||||||
let Ok(metadata) = stdfs::metadata(TEST_PORT_LOCK_DIR) else {
|
|
||||||
return;
|
|
||||||
};
|
|
||||||
let Ok(modified) = metadata.modified() else {
|
|
||||||
return;
|
|
||||||
};
|
|
||||||
if modified.elapsed().is_ok_and(|elapsed| elapsed > TEST_PORT_LOCK_STALE_AFTER) {
|
|
||||||
let _ = stdfs::remove_dir(TEST_PORT_LOCK_DIR);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn write_next_test_port(port: u16) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
|
||||||
stdfs::write(TEST_PORT_COUNTER_PATH, port.to_string())?;
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn capture_command_logs(
|
|
||||||
command: &mut Command,
|
|
||||||
log_path: Option<&str>,
|
|
||||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
|
||||||
let Some(log_path) = log_path else {
|
|
||||||
return Ok(());
|
|
||||||
};
|
|
||||||
let file = stdfs::OpenOptions::new().create(true).append(true).open(log_path)?;
|
|
||||||
let stderr_file = file.try_clone()?;
|
|
||||||
command.stdout(Stdio::from(file)).stderr(Stdio::from(stderr_file));
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn build_test_s3_config(
|
pub(crate) fn build_test_s3_config(
|
||||||
endpoint_url: &str,
|
endpoint_url: &str,
|
||||||
access_key: &str,
|
access_key: &str,
|
||||||
@@ -573,21 +495,10 @@ impl RustFSTestEnvironment {
|
|||||||
/// Find an available port for the test
|
/// Find an available port for the test
|
||||||
pub async fn find_available_port() -> Result<u16, Box<dyn std::error::Error + Send + Sync>> {
|
pub async fn find_available_port() -> Result<u16, Box<dyn std::error::Error + Send + Sync>> {
|
||||||
use std::net::TcpListener;
|
use std::net::TcpListener;
|
||||||
let _guard = PortAllocatorGuard::acquire().await?;
|
let listener = TcpListener::bind("127.0.0.1:0")?;
|
||||||
let mut next_port = read_next_test_port();
|
let port = listener.local_addr()?.port();
|
||||||
|
drop(listener);
|
||||||
for _ in 0..TEST_PORT_RANGE {
|
Ok(port)
|
||||||
let port = next_port;
|
|
||||||
next_port = advance_test_port(next_port);
|
|
||||||
write_next_test_port(next_port)?;
|
|
||||||
|
|
||||||
if let Ok(listener) = TcpListener::bind(("127.0.0.1", port)) {
|
|
||||||
drop(listener);
|
|
||||||
return Ok(port);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
Err("no available E2E test port found".into())
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Kill any existing RustFS processes
|
/// Kill any existing RustFS processes
|
||||||
@@ -646,7 +557,13 @@ impl RustFSTestEnvironment {
|
|||||||
for (key, value) in extra_env {
|
for (key, value) in extra_env {
|
||||||
command.env(key, value);
|
command.env(key, value);
|
||||||
}
|
}
|
||||||
capture_command_logs(&mut command, self.capture_log_path.as_deref())?;
|
// Optionally capture the child's stdout+stderr to a file so the test can
|
||||||
|
// grep server logs (e.g. to confirm which GET reader path was taken).
|
||||||
|
if let Some(log_path) = &self.capture_log_path {
|
||||||
|
let file = stdfs::OpenOptions::new().create(true).append(true).open(log_path)?;
|
||||||
|
let stderr_file = file.try_clone()?;
|
||||||
|
command.stdout(Stdio::from(file)).stderr(Stdio::from(stderr_file));
|
||||||
|
}
|
||||||
let process = command.args(&args).spawn()?;
|
let process = command.args(&args).spawn()?;
|
||||||
|
|
||||||
self.process = Some(process);
|
self.process = Some(process);
|
||||||
@@ -1134,7 +1051,6 @@ pub struct RustFSTestClusterEnvironment {
|
|||||||
pub secret_key: String,
|
pub secret_key: String,
|
||||||
pub extra_env: Vec<(String, String)>,
|
pub extra_env: Vec<(String, String)>,
|
||||||
pub node_extra_env: Vec<Vec<(String, String)>>,
|
pub node_extra_env: Vec<Vec<(String, String)>>,
|
||||||
pub node_capture_log_paths: Vec<Option<String>>,
|
|
||||||
pub topology: ClusterTopology,
|
pub topology: ClusterTopology,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1234,7 +1150,6 @@ impl RustFSTestClusterEnvironment {
|
|||||||
secret_key: "rustfs-cluster-test-secret".to_string(),
|
secret_key: "rustfs-cluster-test-secret".to_string(),
|
||||||
extra_env,
|
extra_env,
|
||||||
node_extra_env: vec![Vec::new(); topology.node_count],
|
node_extra_env: vec![Vec::new(); topology.node_count],
|
||||||
node_capture_log_paths: vec![None; topology.node_count],
|
|
||||||
topology,
|
topology,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -1264,20 +1179,6 @@ impl RustFSTestClusterEnvironment {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Capture stdout+stderr for a single cluster node process.
|
|
||||||
pub fn set_node_capture_log_path<P>(
|
|
||||||
&mut self,
|
|
||||||
node_idx: usize,
|
|
||||||
path: P,
|
|
||||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>>
|
|
||||||
where
|
|
||||||
P: Into<String>,
|
|
||||||
{
|
|
||||||
self.ensure_node_index(node_idx)?;
|
|
||||||
self.node_capture_log_paths[node_idx] = Some(path.into());
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
fn ensure_node_index(&self, node_idx: usize) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
fn ensure_node_index(&self, node_idx: usize) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||||
if node_idx >= self.nodes.len() {
|
if node_idx >= self.nodes.len() {
|
||||||
return Err(format!("node_idx {node_idx} is invalid").into());
|
return Err(format!("node_idx {node_idx} is invalid").into());
|
||||||
@@ -1367,7 +1268,6 @@ impl RustFSTestClusterEnvironment {
|
|||||||
for (key, value) in &self.node_extra_env[i] {
|
for (key, value) in &self.node_extra_env[i] {
|
||||||
command.env(key, value);
|
command.env(key, value);
|
||||||
}
|
}
|
||||||
capture_command_logs(&mut command, self.node_capture_log_paths[i].as_deref())?;
|
|
||||||
|
|
||||||
let process = command.current_dir(&node.data_dir).spawn()?;
|
let process = command.current_dir(&node.data_dir).spawn()?;
|
||||||
|
|
||||||
@@ -1394,7 +1294,6 @@ impl RustFSTestClusterEnvironment {
|
|||||||
|
|
||||||
let binary_path = rustfs_binary_path();
|
let binary_path = rustfs_binary_path();
|
||||||
let volumes_arg = self.build_volumes_arg();
|
let volumes_arg = self.build_volumes_arg();
|
||||||
let log_path = self.node_capture_log_paths[node_idx].clone();
|
|
||||||
let node = &mut self.nodes[node_idx];
|
let node = &mut self.nodes[node_idx];
|
||||||
info!("Starting cluster node {} on {}", node_idx, node.address);
|
info!("Starting cluster node {} on {}", node_idx, node.address);
|
||||||
|
|
||||||
@@ -1413,7 +1312,6 @@ impl RustFSTestClusterEnvironment {
|
|||||||
for (key, value) in &self.node_extra_env[node_idx] {
|
for (key, value) in &self.node_extra_env[node_idx] {
|
||||||
command.env(key, value);
|
command.env(key, value);
|
||||||
}
|
}
|
||||||
capture_command_logs(&mut command, log_path.as_deref())?;
|
|
||||||
|
|
||||||
let process = command.current_dir(&node.data_dir).spawn()?;
|
let process = command.current_dir(&node.data_dir).spawn()?;
|
||||||
node.process = Some(process);
|
node.process = Some(process);
|
||||||
@@ -1665,7 +1563,6 @@ mod tests {
|
|||||||
secret_key: DEFAULT_SECRET_KEY.to_string(),
|
secret_key: DEFAULT_SECRET_KEY.to_string(),
|
||||||
extra_env: Vec::new(),
|
extra_env: Vec::new(),
|
||||||
node_extra_env: vec![Vec::new(); topology.node_count],
|
node_extra_env: vec![Vec::new(); topology.node_count],
|
||||||
node_capture_log_paths: vec![None; topology.node_count],
|
|
||||||
topology,
|
topology,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1761,16 +1658,6 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn cluster_node_log_capture_supports_per_node_paths() {
|
|
||||||
let mut env = fake_cluster(ClusterTopology::single_pool(3));
|
|
||||||
env.set_node_capture_log_path(1, "/tmp/node1.log").unwrap();
|
|
||||||
assert_eq!(env.node_capture_log_paths[0], None);
|
|
||||||
assert_eq!(env.node_capture_log_paths[1], Some("/tmp/node1.log".to_string()));
|
|
||||||
assert_eq!(env.node_capture_log_paths[2], None);
|
|
||||||
assert!(env.set_node_capture_log_path(3, "/tmp/invalid.log").is_err());
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn cluster_node_env_rejects_invalid_index() {
|
fn cluster_node_env_rejects_invalid_index() {
|
||||||
let mut env = fake_cluster(ClusterTopology::single_pool(4));
|
let mut env = fake_cluster(ClusterTopology::single_pool(4));
|
||||||
|
|||||||
@@ -2,10 +2,13 @@
|
|||||||
|
|
||||||
use crate::common::{RustFSTestEnvironment, init_logging, rustfs_binary_path};
|
use crate::common::{RustFSTestEnvironment, init_logging, rustfs_binary_path};
|
||||||
use aws_sdk_s3::primitives::ByteStream;
|
use aws_sdk_s3::primitives::ByteStream;
|
||||||
use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
|
use serial_test::serial;
|
||||||
use std::fs;
|
use std::fs;
|
||||||
use std::path::PathBuf;
|
use std::path::PathBuf;
|
||||||
use std::process::Command;
|
use std::process::Command;
|
||||||
|
use std::time::Duration;
|
||||||
|
use tokio::net::TcpStream;
|
||||||
|
use tokio::time::sleep;
|
||||||
use tracing::info;
|
use tracing::info;
|
||||||
|
|
||||||
const COMPRESSION_TEST_BUCKET: &str = "compression-test-bucket";
|
const COMPRESSION_TEST_BUCKET: &str = "compression-test-bucket";
|
||||||
@@ -22,15 +25,6 @@ fn generate_compressible_data(size: usize) -> Vec<u8> {
|
|||||||
data
|
data
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Deterministic 2048-byte-period binary pattern that compresses extremely well: every part
|
|
||||||
/// yields many compressed blocks, which is exactly the shape that reproduced the mid-payload
|
|
||||||
/// Pending truncation (rustfs/rustfs#5957).
|
|
||||||
fn generate_high_ratio_binary_data(size: usize, seed: u8) -> Vec<u8> {
|
|
||||||
(0..size)
|
|
||||||
.map(|i| ((i as u64).wrapping_mul(2_654_435_761).wrapping_add(seed as u64) >> 3) as u8)
|
|
||||||
.collect()
|
|
||||||
}
|
|
||||||
|
|
||||||
fn find_part_files(temp_dir: &str, bucket: &str, object_key: &str) -> Vec<PathBuf> {
|
fn find_part_files(temp_dir: &str, bucket: &str, object_key: &str) -> Vec<PathBuf> {
|
||||||
let bucket_path = PathBuf::from(temp_dir).join(bucket);
|
let bucket_path = PathBuf::from(temp_dir).join(bucket);
|
||||||
let mut part_files = Vec::new();
|
let mut part_files = Vec::new();
|
||||||
@@ -61,14 +55,9 @@ async fn start_rustfs_with_compression(env: &mut RustFSTestEnvironment) -> Resul
|
|||||||
env.cleanup_existing_processes().await?;
|
env.cleanup_existing_processes().await?;
|
||||||
|
|
||||||
let binary_path = rustfs_binary_path();
|
let binary_path = rustfs_binary_path();
|
||||||
// Route the child's stdout/stderr through the shared RUSTFS_E2E_LOG_DIR
|
let process = Command::new(&binary_path)
|
||||||
// capture (survives the temp-dir cleanup on Drop and is uploaded as a CI
|
|
||||||
// artifact); without the env var the child inherits stdio as before.
|
|
||||||
let mut command = Command::new(&binary_path);
|
|
||||||
command
|
|
||||||
.env("RUSTFS_CONSOLE_ENABLE", "false")
|
.env("RUSTFS_CONSOLE_ENABLE", "false")
|
||||||
.env("RUSTFS_COMPRESSION_ENABLED", "true")
|
.env("RUSTFS_COMPRESSION_ENABLED", "true")
|
||||||
.env("RUSTFS_COMPRESSION_MULTIPART_ENABLED", "true")
|
|
||||||
.args([
|
.args([
|
||||||
"--address",
|
"--address",
|
||||||
&env.address,
|
&env.address,
|
||||||
@@ -77,17 +66,27 @@ async fn start_rustfs_with_compression(env: &mut RustFSTestEnvironment) -> Resul
|
|||||||
"--secret-key",
|
"--secret-key",
|
||||||
&env.secret_key,
|
&env.secret_key,
|
||||||
&env.temp_dir,
|
&env.temp_dir,
|
||||||
]);
|
])
|
||||||
crate::common::capture_command_logs(&mut command, env.capture_log_path.as_deref())?;
|
.spawn()?;
|
||||||
let process = command.spawn()?;
|
|
||||||
|
|
||||||
env.process = Some(process);
|
env.process = Some(process);
|
||||||
|
|
||||||
info!("Waiting for RustFS server with compression enabled on {}", env.address);
|
info!("Waiting for RustFS server with compression enabled on {}", env.address);
|
||||||
env.wait_for_server_ready().await
|
for i in 0..30 {
|
||||||
|
if TcpStream::connect(&env.address).await.is_ok() {
|
||||||
|
info!("RustFS server is ready after {} attempts", i + 1);
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
if i == 29 {
|
||||||
|
return Err("RustFS server failed to become ready".into());
|
||||||
|
}
|
||||||
|
sleep(Duration::from_secs(1)).await;
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn test_compression_roundtrip() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
async fn test_compression_roundtrip() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||||
init_logging();
|
init_logging();
|
||||||
info!("Starting compression roundtrip test");
|
info!("Starting compression roundtrip test");
|
||||||
@@ -155,632 +154,3 @@ async fn test_compression_roundtrip() -> Result<(), Box<dyn std::error::Error +
|
|||||||
env.stop_server();
|
env.stop_server();
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
const MULTIPART_COMPRESSION_BUCKET: &str = "compression-multipart-bucket";
|
|
||||||
const MPU_PART1_SIZE: usize = 5 * 1024 * 1024;
|
|
||||||
const MPU_PART2_SIZE: usize = 1024 * 1024;
|
|
||||||
|
|
||||||
async fn multipart_upload(
|
|
||||||
client: &aws_sdk_s3::Client,
|
|
||||||
bucket: &str,
|
|
||||||
key: &str,
|
|
||||||
parts: &[&[u8]],
|
|
||||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
|
||||||
let create = client.create_multipart_upload().bucket(bucket).key(key).send().await?;
|
|
||||||
let upload_id = create.upload_id().ok_or("missing upload id")?.to_string();
|
|
||||||
|
|
||||||
let mut completed_parts = Vec::with_capacity(parts.len());
|
|
||||||
for (i, part) in parts.iter().enumerate() {
|
|
||||||
let part_number = (i + 1) as i32;
|
|
||||||
let upload = client
|
|
||||||
.upload_part()
|
|
||||||
.bucket(bucket)
|
|
||||||
.key(key)
|
|
||||||
.upload_id(&upload_id)
|
|
||||||
.part_number(part_number)
|
|
||||||
.body(ByteStream::from(part.to_vec()))
|
|
||||||
.send()
|
|
||||||
.await?;
|
|
||||||
completed_parts.push(
|
|
||||||
CompletedPart::builder()
|
|
||||||
.part_number(part_number)
|
|
||||||
.e_tag(upload.e_tag().unwrap_or_default())
|
|
||||||
.build(),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
client
|
|
||||||
.complete_multipart_upload()
|
|
||||||
.bucket(bucket)
|
|
||||||
.key(key)
|
|
||||||
.upload_id(&upload_id)
|
|
||||||
.multipart_upload(CompletedMultipartUpload::builder().set_parts(Some(completed_parts)).build())
|
|
||||||
.send()
|
|
||||||
.await?;
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn fetch_range(
|
|
||||||
client: &aws_sdk_s3::Client,
|
|
||||||
bucket: &str,
|
|
||||||
key: &str,
|
|
||||||
range: &str,
|
|
||||||
) -> Result<Vec<u8>, Box<dyn std::error::Error + Send + Sync>> {
|
|
||||||
let response = client.get_object().bucket(bucket).key(key).range(range).send().await?;
|
|
||||||
Ok(response.body.collect().await?.into_bytes().to_vec())
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Multipart disk compression roundtrip: parts are written as independent
|
|
||||||
/// compressed streams and every GET shape must reassemble the original bytes
|
|
||||||
/// (rustfs/rustfs#5957: multipart uploads previously bypassed disk compression
|
|
||||||
/// entirely).
|
|
||||||
#[tokio::test]
|
|
||||||
async fn test_compression_multipart_roundtrip() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
|
||||||
init_logging();
|
|
||||||
info!("Starting multipart compression roundtrip test");
|
|
||||||
|
|
||||||
let mut env = RustFSTestEnvironment::new().await?;
|
|
||||||
start_rustfs_with_compression(&mut env).await?;
|
|
||||||
|
|
||||||
let client = env.create_s3_client();
|
|
||||||
env.create_test_bucket(MULTIPART_COMPRESSION_BUCKET).await?;
|
|
||||||
|
|
||||||
let object_key = "multipart-compressible.txt";
|
|
||||||
let part1 = generate_compressible_data(MPU_PART1_SIZE);
|
|
||||||
let part2 = generate_compressible_data(MPU_PART2_SIZE);
|
|
||||||
let mut original_data = part1.clone();
|
|
||||||
original_data.extend_from_slice(&part2);
|
|
||||||
let total_size = original_data.len();
|
|
||||||
|
|
||||||
multipart_upload(&client, MULTIPART_COMPRESSION_BUCKET, object_key, &[&part1, &part2]).await?;
|
|
||||||
|
|
||||||
let head_response = client
|
|
||||||
.head_object()
|
|
||||||
.bucket(MULTIPART_COMPRESSION_BUCKET)
|
|
||||||
.key(object_key)
|
|
||||||
.send()
|
|
||||||
.await?;
|
|
||||||
assert_eq!(
|
|
||||||
head_response.content_length().unwrap_or(0) as usize,
|
|
||||||
total_size,
|
|
||||||
"Content-Length should be the logical object size"
|
|
||||||
);
|
|
||||||
|
|
||||||
let part_files = find_part_files(&env.temp_dir, MULTIPART_COMPRESSION_BUCKET, object_key);
|
|
||||||
assert!(!part_files.is_empty(), "expected on-disk part files for the multipart object");
|
|
||||||
let total_physical_size: u64 = part_files.iter().filter_map(|p| fs::metadata(p).ok()).map(|m| m.len()).sum();
|
|
||||||
assert!(
|
|
||||||
total_physical_size < (total_size / 2) as u64,
|
|
||||||
"Physical size {total_physical_size} should be well below original size {total_size} (multipart compression applied)"
|
|
||||||
);
|
|
||||||
info!("Multipart physical storage size: {total_physical_size} bytes (compressed from {total_size} bytes)");
|
|
||||||
|
|
||||||
// Full GET must reassemble both independently compressed parts.
|
|
||||||
let get_response = client
|
|
||||||
.get_object()
|
|
||||||
.bucket(MULTIPART_COMPRESSION_BUCKET)
|
|
||||||
.key(object_key)
|
|
||||||
.send()
|
|
||||||
.await?;
|
|
||||||
let downloaded = get_response.body.collect().await?.into_bytes();
|
|
||||||
assert_eq!(downloaded.len(), total_size);
|
|
||||||
assert_eq!(&downloaded[..], &original_data[..], "full GET data mismatch");
|
|
||||||
|
|
||||||
// Range fully inside part 1.
|
|
||||||
let range_inside_part1 = fetch_range(&client, MULTIPART_COMPRESSION_BUCKET, object_key, "bytes=1024-999423").await?;
|
|
||||||
assert_eq!(&range_inside_part1[..], &original_data[1024..999424], "part-1 range mismatch");
|
|
||||||
|
|
||||||
// Range crossing the part boundary.
|
|
||||||
let boundary_start = MPU_PART1_SIZE - 128 * 1024;
|
|
||||||
let boundary_end = MPU_PART1_SIZE + 128 * 1024 - 1;
|
|
||||||
let range_crossing = fetch_range(
|
|
||||||
&client,
|
|
||||||
MULTIPART_COMPRESSION_BUCKET,
|
|
||||||
object_key,
|
|
||||||
&format!("bytes={boundary_start}-{boundary_end}"),
|
|
||||||
)
|
|
||||||
.await?;
|
|
||||||
assert_eq!(
|
|
||||||
&range_crossing[..],
|
|
||||||
&original_data[boundary_start..boundary_end + 1],
|
|
||||||
"boundary-crossing range mismatch"
|
|
||||||
);
|
|
||||||
|
|
||||||
// Range fully inside part 2.
|
|
||||||
let part2_start = MPU_PART1_SIZE + 4096;
|
|
||||||
let part2_end = MPU_PART1_SIZE + 256 * 1024 - 1;
|
|
||||||
let range_inside_part2 = fetch_range(
|
|
||||||
&client,
|
|
||||||
MULTIPART_COMPRESSION_BUCKET,
|
|
||||||
object_key,
|
|
||||||
&format!("bytes={part2_start}-{part2_end}"),
|
|
||||||
)
|
|
||||||
.await?;
|
|
||||||
assert_eq!(
|
|
||||||
&range_inside_part2[..],
|
|
||||||
&original_data[part2_start..part2_end + 1],
|
|
||||||
"part-2 range mismatch"
|
|
||||||
);
|
|
||||||
|
|
||||||
// Suffix range (last 128 KiB, entirely in part 2).
|
|
||||||
let suffix_len = 128 * 1024;
|
|
||||||
let suffix = fetch_range(&client, MULTIPART_COMPRESSION_BUCKET, object_key, &format!("bytes=-{suffix_len}")).await?;
|
|
||||||
assert_eq!(&suffix[..], &original_data[total_size - suffix_len..], "suffix range mismatch");
|
|
||||||
|
|
||||||
// partNumber GETs must return each original part.
|
|
||||||
for (part_number, expected) in [(1, &part1), (2, &part2)] {
|
|
||||||
let response = client
|
|
||||||
.get_object()
|
|
||||||
.bucket(MULTIPART_COMPRESSION_BUCKET)
|
|
||||||
.key(object_key)
|
|
||||||
.part_number(part_number)
|
|
||||||
.send()
|
|
||||||
.await?;
|
|
||||||
let body = response.body.collect().await?.into_bytes();
|
|
||||||
assert_eq!(&body[..], &expected[..], "partNumber={part_number} GET mismatch");
|
|
||||||
}
|
|
||||||
|
|
||||||
info!("Multipart compression roundtrip test passed");
|
|
||||||
env.delete_test_bucket(MULTIPART_COMPRESSION_BUCKET).await?;
|
|
||||||
env.stop_server();
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
const MPU_HIGH_RATIO_BUCKET: &str = "compression-mpu-high-ratio-bucket";
|
|
||||||
|
|
||||||
/// High-ratio binary multipart payload: the object key is on the compression allow-list, so the
|
|
||||||
/// disk-compression path runs and each part is stored as many compressed blocks — the shape that
|
|
||||||
/// reproduced the mid-payload Pending truncation (rustfs/rustfs#5957). Every GET shape must return
|
|
||||||
/// the exact original bytes, and the stored size must show the data really was compressed.
|
|
||||||
#[tokio::test]
|
|
||||||
async fn test_compression_multipart_high_ratio_binary_roundtrip() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
|
||||||
init_logging();
|
|
||||||
info!("Starting multipart high-ratio binary compression roundtrip test");
|
|
||||||
|
|
||||||
let mut env = RustFSTestEnvironment::new().await?;
|
|
||||||
start_rustfs_with_compression(&mut env).await?;
|
|
||||||
|
|
||||||
let client = env.create_s3_client();
|
|
||||||
env.create_test_bucket(MPU_HIGH_RATIO_BUCKET).await?;
|
|
||||||
|
|
||||||
let object_key = "multipart-high-ratio.txt";
|
|
||||||
let part1 = generate_high_ratio_binary_data(MPU_PART1_SIZE, 7);
|
|
||||||
let part2 = generate_high_ratio_binary_data(MPU_PART2_SIZE, 61);
|
|
||||||
let mut original_data = part1.clone();
|
|
||||||
original_data.extend_from_slice(&part2);
|
|
||||||
let total_size = original_data.len();
|
|
||||||
|
|
||||||
multipart_upload(&client, MPU_HIGH_RATIO_BUCKET, object_key, &[&part1, &part2]).await?;
|
|
||||||
|
|
||||||
let head_response = client
|
|
||||||
.head_object()
|
|
||||||
.bucket(MPU_HIGH_RATIO_BUCKET)
|
|
||||||
.key(object_key)
|
|
||||||
.send()
|
|
||||||
.await?;
|
|
||||||
assert_eq!(
|
|
||||||
head_response.content_length().unwrap_or(0) as usize,
|
|
||||||
total_size,
|
|
||||||
"Content-Length should be the logical object size"
|
|
||||||
);
|
|
||||||
|
|
||||||
// This pattern compresses to roughly 1/50 of its logical size, so a comfortably loose 2x
|
|
||||||
// margin still proves the parts were stored compressed rather than raw or double-encoded.
|
|
||||||
let part_files = find_part_files(&env.temp_dir, MPU_HIGH_RATIO_BUCKET, object_key);
|
|
||||||
assert!(!part_files.is_empty(), "expected on-disk part files for the multipart object");
|
|
||||||
let total_physical_size: u64 = part_files.iter().filter_map(|p| fs::metadata(p).ok()).map(|m| m.len()).sum();
|
|
||||||
assert!(
|
|
||||||
total_physical_size < (total_size as u64) / 2,
|
|
||||||
"Physical size {total_physical_size} should be far below the logical size {total_size} for high-ratio data"
|
|
||||||
);
|
|
||||||
info!("High-ratio multipart physical storage size: {total_physical_size} bytes (logical {total_size} bytes)");
|
|
||||||
|
|
||||||
info!("step: full GET");
|
|
||||||
let get_response = client
|
|
||||||
.get_object()
|
|
||||||
.bucket(MPU_HIGH_RATIO_BUCKET)
|
|
||||||
.key(object_key)
|
|
||||||
.send()
|
|
||||||
.await?;
|
|
||||||
let downloaded = get_response.body.collect().await?.into_bytes();
|
|
||||||
assert_eq!(downloaded.len(), total_size);
|
|
||||||
assert_eq!(&downloaded[..], &original_data[..], "full GET data mismatch");
|
|
||||||
|
|
||||||
// Range crossing the part boundary.
|
|
||||||
info!("step: boundary range GET");
|
|
||||||
let boundary_start = MPU_PART1_SIZE - 128 * 1024;
|
|
||||||
let boundary_end = MPU_PART1_SIZE + 128 * 1024 - 1;
|
|
||||||
let range_crossing = fetch_range(
|
|
||||||
&client,
|
|
||||||
MPU_HIGH_RATIO_BUCKET,
|
|
||||||
object_key,
|
|
||||||
&format!("bytes={boundary_start}-{boundary_end}"),
|
|
||||||
)
|
|
||||||
.await?;
|
|
||||||
assert_eq!(
|
|
||||||
&range_crossing[..],
|
|
||||||
&original_data[boundary_start..boundary_end + 1],
|
|
||||||
"boundary-crossing range mismatch"
|
|
||||||
);
|
|
||||||
|
|
||||||
// partNumber GET for the trailing part.
|
|
||||||
info!("step: partNumber GET");
|
|
||||||
let part2_response = client
|
|
||||||
.get_object()
|
|
||||||
.bucket(MPU_HIGH_RATIO_BUCKET)
|
|
||||||
.key(object_key)
|
|
||||||
.part_number(2)
|
|
||||||
.send()
|
|
||||||
.await?;
|
|
||||||
let part2_body = part2_response.body.collect().await?.into_bytes();
|
|
||||||
assert_eq!(&part2_body[..], &part2[..], "partNumber=2 GET mismatch");
|
|
||||||
|
|
||||||
info!("Multipart high-ratio binary compression roundtrip test passed");
|
|
||||||
env.delete_test_bucket(MPU_HIGH_RATIO_BUCKET).await?;
|
|
||||||
env.stop_server();
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
const MPU_COPY_COMPRESSION_BUCKET: &str = "compression-mpu-copy-bucket";
|
|
||||||
const MPU_COPY_SOURCE_SIZE: usize = 6 * 1024 * 1024;
|
|
||||||
const MPU_COPY_RANGE_LEN: usize = 5 * 1024 * 1024;
|
|
||||||
|
|
||||||
/// UploadPartCopy feeds a part from an already stored (and already compressed) object. The copied
|
|
||||||
/// range must be decompressed on read and re-compressed into the destination part, so the final
|
|
||||||
/// object has to match "source prefix + uploaded tail" byte for byte.
|
|
||||||
#[tokio::test]
|
|
||||||
async fn test_compression_multipart_upload_part_copy_roundtrip() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
|
||||||
init_logging();
|
|
||||||
info!("Starting multipart upload-part-copy compression roundtrip test");
|
|
||||||
|
|
||||||
let mut env = RustFSTestEnvironment::new().await?;
|
|
||||||
start_rustfs_with_compression(&mut env).await?;
|
|
||||||
|
|
||||||
let client = env.create_s3_client();
|
|
||||||
env.create_test_bucket(MPU_COPY_COMPRESSION_BUCKET).await?;
|
|
||||||
|
|
||||||
// Source object: a plain PUT that goes through the single-stream compression path.
|
|
||||||
let source_key = "copy-source.txt";
|
|
||||||
let source_data = generate_compressible_data(MPU_COPY_SOURCE_SIZE);
|
|
||||||
client
|
|
||||||
.put_object()
|
|
||||||
.bucket(MPU_COPY_COMPRESSION_BUCKET)
|
|
||||||
.key(source_key)
|
|
||||||
.body(ByteStream::from(source_data.clone()))
|
|
||||||
.send()
|
|
||||||
.await?;
|
|
||||||
|
|
||||||
// Destination object: part 1 copied from the source, part 2 uploaded directly.
|
|
||||||
let target_key = "copy-target.txt";
|
|
||||||
let part2 = generate_compressible_data(MPU_PART2_SIZE);
|
|
||||||
let mut expected_data = source_data[..MPU_COPY_RANGE_LEN].to_vec();
|
|
||||||
expected_data.extend_from_slice(&part2);
|
|
||||||
let total_size = expected_data.len();
|
|
||||||
|
|
||||||
let create = client
|
|
||||||
.create_multipart_upload()
|
|
||||||
.bucket(MPU_COPY_COMPRESSION_BUCKET)
|
|
||||||
.key(target_key)
|
|
||||||
.send()
|
|
||||||
.await?;
|
|
||||||
let upload_id = create.upload_id().ok_or("missing upload id")?.to_string();
|
|
||||||
|
|
||||||
let copy_part = client
|
|
||||||
.upload_part_copy()
|
|
||||||
.bucket(MPU_COPY_COMPRESSION_BUCKET)
|
|
||||||
.key(target_key)
|
|
||||||
.upload_id(&upload_id)
|
|
||||||
.part_number(1)
|
|
||||||
.copy_source(format!("{MPU_COPY_COMPRESSION_BUCKET}/{source_key}"))
|
|
||||||
.copy_source_range(format!("bytes=0-{}", MPU_COPY_RANGE_LEN - 1))
|
|
||||||
.send()
|
|
||||||
.await?;
|
|
||||||
let copy_etag = copy_part
|
|
||||||
.copy_part_result()
|
|
||||||
.and_then(|r| r.e_tag())
|
|
||||||
.ok_or("missing copy part etag")?
|
|
||||||
.to_string();
|
|
||||||
|
|
||||||
let uploaded_part = client
|
|
||||||
.upload_part()
|
|
||||||
.bucket(MPU_COPY_COMPRESSION_BUCKET)
|
|
||||||
.key(target_key)
|
|
||||||
.upload_id(&upload_id)
|
|
||||||
.part_number(2)
|
|
||||||
.body(ByteStream::from(part2.clone()))
|
|
||||||
.send()
|
|
||||||
.await?;
|
|
||||||
|
|
||||||
client
|
|
||||||
.complete_multipart_upload()
|
|
||||||
.bucket(MPU_COPY_COMPRESSION_BUCKET)
|
|
||||||
.key(target_key)
|
|
||||||
.upload_id(&upload_id)
|
|
||||||
.multipart_upload(
|
|
||||||
CompletedMultipartUpload::builder()
|
|
||||||
.parts(CompletedPart::builder().part_number(1).e_tag(copy_etag).build())
|
|
||||||
.parts(
|
|
||||||
CompletedPart::builder()
|
|
||||||
.part_number(2)
|
|
||||||
.e_tag(uploaded_part.e_tag().unwrap_or_default())
|
|
||||||
.build(),
|
|
||||||
)
|
|
||||||
.build(),
|
|
||||||
)
|
|
||||||
.send()
|
|
||||||
.await?;
|
|
||||||
|
|
||||||
let head_response = client
|
|
||||||
.head_object()
|
|
||||||
.bucket(MPU_COPY_COMPRESSION_BUCKET)
|
|
||||||
.key(target_key)
|
|
||||||
.send()
|
|
||||||
.await?;
|
|
||||||
assert_eq!(
|
|
||||||
head_response.content_length().unwrap_or(0) as usize,
|
|
||||||
total_size,
|
|
||||||
"Content-Length should be the logical object size"
|
|
||||||
);
|
|
||||||
|
|
||||||
let part_files = find_part_files(&env.temp_dir, MPU_COPY_COMPRESSION_BUCKET, target_key);
|
|
||||||
assert!(!part_files.is_empty(), "expected on-disk part files for the copied object");
|
|
||||||
let total_physical_size: u64 = part_files.iter().filter_map(|p| fs::metadata(p).ok()).map(|m| m.len()).sum();
|
|
||||||
assert!(
|
|
||||||
total_physical_size < (total_size / 2) as u64,
|
|
||||||
"Physical size {total_physical_size} should be well below original size {total_size} (copied part compression applied)"
|
|
||||||
);
|
|
||||||
|
|
||||||
let get_response = client
|
|
||||||
.get_object()
|
|
||||||
.bucket(MPU_COPY_COMPRESSION_BUCKET)
|
|
||||||
.key(target_key)
|
|
||||||
.send()
|
|
||||||
.await?;
|
|
||||||
let downloaded = get_response.body.collect().await?.into_bytes();
|
|
||||||
assert_eq!(downloaded.len(), total_size);
|
|
||||||
assert_eq!(&downloaded[..], &expected_data[..], "copied multipart GET data mismatch");
|
|
||||||
|
|
||||||
info!("Multipart upload-part-copy compression roundtrip test passed");
|
|
||||||
env.delete_test_bucket(MPU_COPY_COMPRESSION_BUCKET).await?;
|
|
||||||
env.stop_server();
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
const MPU_THREE_PARTS_BUCKET: &str = "compression-mpu-three-parts-bucket";
|
|
||||||
const MPU_THREE_PARTS_TAIL_SIZE: usize = 512 * 1024;
|
|
||||||
|
|
||||||
/// Three-part upload with uneven part sizes: each partNumber GET must map back to exactly one
|
|
||||||
/// compressed part stream, and a suffix range must resolve inside the trailing part.
|
|
||||||
#[tokio::test]
|
|
||||||
async fn test_compression_multipart_three_parts_part_number_gets() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
|
||||||
init_logging();
|
|
||||||
info!("Starting three-part multipart compression partNumber test");
|
|
||||||
|
|
||||||
let mut env = RustFSTestEnvironment::new().await?;
|
|
||||||
start_rustfs_with_compression(&mut env).await?;
|
|
||||||
|
|
||||||
let client = env.create_s3_client();
|
|
||||||
env.create_test_bucket(MPU_THREE_PARTS_BUCKET).await?;
|
|
||||||
|
|
||||||
let object_key = "multipart-three-parts.txt";
|
|
||||||
let part1 = generate_compressible_data(MPU_PART1_SIZE);
|
|
||||||
let part2 = generate_compressible_data(MPU_PART1_SIZE);
|
|
||||||
let part3 = generate_compressible_data(MPU_THREE_PARTS_TAIL_SIZE);
|
|
||||||
let mut original_data = part1.clone();
|
|
||||||
original_data.extend_from_slice(&part2);
|
|
||||||
original_data.extend_from_slice(&part3);
|
|
||||||
let total_size = original_data.len();
|
|
||||||
|
|
||||||
multipart_upload(&client, MPU_THREE_PARTS_BUCKET, object_key, &[&part1, &part2, &part3]).await?;
|
|
||||||
|
|
||||||
let head_response = client
|
|
||||||
.head_object()
|
|
||||||
.bucket(MPU_THREE_PARTS_BUCKET)
|
|
||||||
.key(object_key)
|
|
||||||
.send()
|
|
||||||
.await?;
|
|
||||||
assert_eq!(
|
|
||||||
head_response.content_length().unwrap_or(0) as usize,
|
|
||||||
total_size,
|
|
||||||
"Content-Length should be the logical object size"
|
|
||||||
);
|
|
||||||
|
|
||||||
let part_files = find_part_files(&env.temp_dir, MPU_THREE_PARTS_BUCKET, object_key);
|
|
||||||
assert!(!part_files.is_empty(), "expected on-disk part files for the multipart object");
|
|
||||||
let total_physical_size: u64 = part_files.iter().filter_map(|p| fs::metadata(p).ok()).map(|m| m.len()).sum();
|
|
||||||
assert!(
|
|
||||||
total_physical_size < (total_size / 2) as u64,
|
|
||||||
"Physical size {total_physical_size} should be well below original size {total_size} (multipart compression applied)"
|
|
||||||
);
|
|
||||||
|
|
||||||
// Every partNumber GET must return exactly the bytes of the corresponding uploaded part.
|
|
||||||
for (part_number, expected) in [(1, &part1), (2, &part2), (3, &part3)] {
|
|
||||||
let response = client
|
|
||||||
.get_object()
|
|
||||||
.bucket(MPU_THREE_PARTS_BUCKET)
|
|
||||||
.key(object_key)
|
|
||||||
.part_number(part_number)
|
|
||||||
.send()
|
|
||||||
.await?;
|
|
||||||
let body = response.body.collect().await?.into_bytes();
|
|
||||||
assert_eq!(&body[..], &expected[..], "partNumber={part_number} GET mismatch");
|
|
||||||
}
|
|
||||||
|
|
||||||
// Suffix range (last 64 KiB) resolves inside the trailing part.
|
|
||||||
let suffix_len = 64 * 1024;
|
|
||||||
let suffix = fetch_range(&client, MPU_THREE_PARTS_BUCKET, object_key, &format!("bytes=-{suffix_len}")).await?;
|
|
||||||
assert_eq!(&suffix[..], &original_data[total_size - suffix_len..], "suffix range mismatch");
|
|
||||||
|
|
||||||
info!("Three-part multipart compression partNumber test passed");
|
|
||||||
env.delete_test_bucket(MPU_THREE_PARTS_BUCKET).await?;
|
|
||||||
env.stop_server();
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
const MPU_SSE_COMPRESSION_BUCKET: &str = "compression-mpu-sse-bucket";
|
|
||||||
|
|
||||||
async fn start_rustfs_with_compression_and_sse(
|
|
||||||
env: &mut RustFSTestEnvironment,
|
|
||||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
|
||||||
use base64::Engine;
|
|
||||||
env.cleanup_existing_processes().await?;
|
|
||||||
|
|
||||||
let binary_path = rustfs_binary_path();
|
|
||||||
let master_key = base64::engine::general_purpose::STANDARD.encode([0x42u8; 32]);
|
|
||||||
// Server output goes to a file inside the per-test temp dir so a failing
|
|
||||||
// run can be diagnosed from the child's logs.
|
|
||||||
let server_log = std::fs::File::create(format!("{}/server.log", env.temp_dir))?;
|
|
||||||
let server_log_err = server_log.try_clone()?;
|
|
||||||
let process = Command::new(&binary_path)
|
|
||||||
.env("RUSTFS_CONSOLE_ENABLE", "false")
|
|
||||||
.env("RUSTFS_COMPRESSION_ENABLED", "true")
|
|
||||||
.env("RUSTFS_COMPRESSION_MULTIPART_ENABLED", "true")
|
|
||||||
.env("RUSTFS_SSE_S3_MASTER_KEY", master_key)
|
|
||||||
.env("RUST_LOG", "rustfs=info,rustfs_ecstore=info")
|
|
||||||
.stdout(std::process::Stdio::from(server_log))
|
|
||||||
.stderr(std::process::Stdio::from(server_log_err))
|
|
||||||
.args([
|
|
||||||
"--address",
|
|
||||||
&env.address,
|
|
||||||
"--access-key",
|
|
||||||
&env.access_key,
|
|
||||||
"--secret-key",
|
|
||||||
&env.secret_key,
|
|
||||||
&env.temp_dir,
|
|
||||||
])
|
|
||||||
.spawn()?;
|
|
||||||
|
|
||||||
env.process = Some(process);
|
|
||||||
|
|
||||||
info!("Waiting for RustFS server with compression + SSE-S3 enabled on {}", env.address);
|
|
||||||
env.wait_for_server_ready().await
|
|
||||||
}
|
|
||||||
|
|
||||||
/// SSE-S3 + disk compression multipart: each part is compressed and then encrypted, and every GET
|
|
||||||
/// shape must still return the original plaintext bytes. Physical size must shrink because the
|
|
||||||
/// compression runs before encryption.
|
|
||||||
#[tokio::test]
|
|
||||||
async fn test_compression_multipart_sse_s3_roundtrip() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
|
||||||
use aws_sdk_s3::types::ServerSideEncryption;
|
|
||||||
|
|
||||||
init_logging();
|
|
||||||
info!("Starting SSE-S3 multipart compression roundtrip test");
|
|
||||||
|
|
||||||
let mut env = RustFSTestEnvironment::new().await?;
|
|
||||||
start_rustfs_with_compression_and_sse(&mut env).await?;
|
|
||||||
|
|
||||||
let client = env.create_s3_client();
|
|
||||||
env.create_test_bucket(MPU_SSE_COMPRESSION_BUCKET).await?;
|
|
||||||
|
|
||||||
let object_key = "multipart-sse-compressible.txt";
|
|
||||||
let part1 = generate_compressible_data(MPU_PART1_SIZE);
|
|
||||||
let part2 = generate_compressible_data(MPU_PART2_SIZE);
|
|
||||||
let mut original_data = part1.clone();
|
|
||||||
original_data.extend_from_slice(&part2);
|
|
||||||
let total_size = original_data.len();
|
|
||||||
|
|
||||||
let create = client
|
|
||||||
.create_multipart_upload()
|
|
||||||
.bucket(MPU_SSE_COMPRESSION_BUCKET)
|
|
||||||
.key(object_key)
|
|
||||||
.server_side_encryption(ServerSideEncryption::Aes256)
|
|
||||||
.send()
|
|
||||||
.await?;
|
|
||||||
let upload_id = create.upload_id().ok_or("missing upload id")?.to_string();
|
|
||||||
|
|
||||||
let mut completed_parts = Vec::new();
|
|
||||||
for (i, part) in [&part1, &part2].into_iter().enumerate() {
|
|
||||||
let part_number = (i + 1) as i32;
|
|
||||||
let upload = client
|
|
||||||
.upload_part()
|
|
||||||
.bucket(MPU_SSE_COMPRESSION_BUCKET)
|
|
||||||
.key(object_key)
|
|
||||||
.upload_id(&upload_id)
|
|
||||||
.part_number(part_number)
|
|
||||||
.body(ByteStream::from(part.clone()))
|
|
||||||
.send()
|
|
||||||
.await?;
|
|
||||||
completed_parts.push(
|
|
||||||
CompletedPart::builder()
|
|
||||||
.part_number(part_number)
|
|
||||||
.e_tag(upload.e_tag().unwrap_or_default())
|
|
||||||
.build(),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
client
|
|
||||||
.complete_multipart_upload()
|
|
||||||
.bucket(MPU_SSE_COMPRESSION_BUCKET)
|
|
||||||
.key(object_key)
|
|
||||||
.upload_id(&upload_id)
|
|
||||||
.multipart_upload(CompletedMultipartUpload::builder().set_parts(Some(completed_parts)).build())
|
|
||||||
.send()
|
|
||||||
.await?;
|
|
||||||
|
|
||||||
let head_response = client
|
|
||||||
.head_object()
|
|
||||||
.bucket(MPU_SSE_COMPRESSION_BUCKET)
|
|
||||||
.key(object_key)
|
|
||||||
.send()
|
|
||||||
.await?;
|
|
||||||
assert_eq!(
|
|
||||||
head_response.content_length().unwrap_or(0) as usize,
|
|
||||||
total_size,
|
|
||||||
"Content-Length should be the logical object size"
|
|
||||||
);
|
|
||||||
assert_eq!(
|
|
||||||
head_response.server_side_encryption(),
|
|
||||||
Some(&ServerSideEncryption::Aes256),
|
|
||||||
"HEAD must report SSE-S3"
|
|
||||||
);
|
|
||||||
|
|
||||||
let part_files = find_part_files(&env.temp_dir, MPU_SSE_COMPRESSION_BUCKET, object_key);
|
|
||||||
assert!(!part_files.is_empty(), "expected on-disk part files for the multipart object");
|
|
||||||
let total_physical_size: u64 = part_files.iter().filter_map(|p| fs::metadata(p).ok()).map(|m| m.len()).sum();
|
|
||||||
assert!(
|
|
||||||
total_physical_size < (total_size / 2) as u64,
|
|
||||||
"Physical size {total_physical_size} should be well below original size {total_size} (compress-then-encrypt applied)"
|
|
||||||
);
|
|
||||||
|
|
||||||
let get_response = client
|
|
||||||
.get_object()
|
|
||||||
.bucket(MPU_SSE_COMPRESSION_BUCKET)
|
|
||||||
.key(object_key)
|
|
||||||
.send()
|
|
||||||
.await?;
|
|
||||||
let downloaded = get_response.body.collect().await?.into_bytes();
|
|
||||||
assert_eq!(downloaded.len(), total_size);
|
|
||||||
assert_eq!(&downloaded[..], &original_data[..], "SSE-S3 multipart full GET data mismatch");
|
|
||||||
|
|
||||||
// Range crossing the part boundary must decrypt and decompress across parts.
|
|
||||||
let boundary_start = MPU_PART1_SIZE - 64 * 1024;
|
|
||||||
let boundary_end = MPU_PART1_SIZE + 64 * 1024 - 1;
|
|
||||||
let range_crossing = fetch_range(
|
|
||||||
&client,
|
|
||||||
MPU_SSE_COMPRESSION_BUCKET,
|
|
||||||
object_key,
|
|
||||||
&format!("bytes={boundary_start}-{boundary_end}"),
|
|
||||||
)
|
|
||||||
.await?;
|
|
||||||
assert_eq!(
|
|
||||||
&range_crossing[..],
|
|
||||||
&original_data[boundary_start..boundary_end + 1],
|
|
||||||
"SSE-S3 boundary-crossing range mismatch"
|
|
||||||
);
|
|
||||||
|
|
||||||
// partNumber GET for the trailing part.
|
|
||||||
let part2_response = client
|
|
||||||
.get_object()
|
|
||||||
.bucket(MPU_SSE_COMPRESSION_BUCKET)
|
|
||||||
.key(object_key)
|
|
||||||
.part_number(2)
|
|
||||||
.send()
|
|
||||||
.await?;
|
|
||||||
let part2_body = part2_response.body.collect().await?.into_bytes();
|
|
||||||
assert_eq!(&part2_body[..], &part2[..], "SSE-S3 partNumber=2 GET mismatch");
|
|
||||||
|
|
||||||
info!("SSE-S3 multipart compression roundtrip test passed");
|
|
||||||
env.delete_test_bucket(MPU_SSE_COMPRESSION_BUCKET).await?;
|
|
||||||
env.stop_server();
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -18,6 +18,7 @@
|
|||||||
//! concurrency — a queued connection is served only after a held one closes.
|
//! concurrency — a queued connection is served only after a held one closes.
|
||||||
|
|
||||||
use crate::common::{RustFSTestEnvironment, init_logging};
|
use crate::common::{RustFSTestEnvironment, init_logging};
|
||||||
|
use serial_test::serial;
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||||
use tokio::net::TcpStream;
|
use tokio::net::TcpStream;
|
||||||
@@ -56,6 +57,7 @@ async fn read_response_head(stream: &mut TcpStream, dur: Duration) -> Option<Str
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn connection_cap_releases_permits_on_close() -> TestResult {
|
async fn connection_cap_releases_permits_on_close() -> TestResult {
|
||||||
init_logging();
|
init_logging();
|
||||||
let mut env = RustFSTestEnvironment::new().await?;
|
let mut env = RustFSTestEnvironment::new().await?;
|
||||||
@@ -87,6 +89,7 @@ async fn open_and_stall(addr: &str) -> std::io::Result<TcpStream> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn connection_cap_blocks_excess_connections_until_permits_free() -> TestResult {
|
async fn connection_cap_blocks_excess_connections_until_permits_free() -> TestResult {
|
||||||
init_logging();
|
init_logging();
|
||||||
let mut env = RustFSTestEnvironment::new().await?;
|
let mut env = RustFSTestEnvironment::new().await?;
|
||||||
|
|||||||
@@ -33,6 +33,7 @@
|
|||||||
//! serve the unauthenticated console endpoints at all.
|
//! serve the unauthenticated console endpoints at all.
|
||||||
|
|
||||||
use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
|
use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
|
||||||
|
use serial_test::serial;
|
||||||
use std::error::Error;
|
use std::error::Error;
|
||||||
use tokio::time::{Duration, sleep};
|
use tokio::time::{Duration, sleep};
|
||||||
|
|
||||||
@@ -57,6 +58,7 @@ async fn wait_for_console_ready(console_base: &str) -> Result<reqwest::Response,
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn test_console_over_the_wire_smoke() -> TestResult {
|
async fn test_console_over_the_wire_smoke() -> TestResult {
|
||||||
init_logging();
|
init_logging();
|
||||||
|
|
||||||
|
|||||||
@@ -22,10 +22,12 @@
|
|||||||
mod tests {
|
mod tests {
|
||||||
use crate::common::{RustFSTestEnvironment, init_logging};
|
use crate::common::{RustFSTestEnvironment, init_logging};
|
||||||
use aws_sdk_s3::primitives::ByteStream;
|
use aws_sdk_s3::primitives::ByteStream;
|
||||||
|
use serial_test::serial;
|
||||||
use tracing::info;
|
use tracing::info;
|
||||||
|
|
||||||
/// Verify Content-Encoding header roundtrips through PUT, GET, and HEAD operations
|
/// Verify Content-Encoding header roundtrips through PUT, GET, and HEAD operations
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn test_content_encoding_roundtrip() {
|
async fn test_content_encoding_roundtrip() {
|
||||||
init_logging();
|
init_logging();
|
||||||
info!("Starting Content-Encoding roundtrip test");
|
info!("Starting Content-Encoding roundtrip test");
|
||||||
@@ -103,6 +105,7 @@ mod tests {
|
|||||||
/// Issue #1857: Content-Encoding "aws-chunked" is used by SigV4 streaming clients and must
|
/// Issue #1857: Content-Encoding "aws-chunked" is used by SigV4 streaming clients and must
|
||||||
/// not be stored or returned. Upload with aws-chunked and verify GET/HEAD do not return it.
|
/// not be stored or returned. Upload with aws-chunked and verify GET/HEAD do not return it.
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn test_content_encoding_aws_chunked_not_returned_issue_1857() {
|
async fn test_content_encoding_aws_chunked_not_returned_issue_1857() {
|
||||||
init_logging();
|
init_logging();
|
||||||
info!("Issue #1857: aws-chunked must not be persisted or returned");
|
info!("Issue #1857: aws-chunked must not be persisted or returned");
|
||||||
@@ -158,6 +161,7 @@ mod tests {
|
|||||||
/// Issue #2475 / Route A: when aws-chunked is combined with an effective object encoding,
|
/// Issue #2475 / Route A: when aws-chunked is combined with an effective object encoding,
|
||||||
/// only the effective encoding should roundtrip through GET/HEAD.
|
/// only the effective encoding should roundtrip through GET/HEAD.
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn test_content_encoding_aws_chunked_with_effective_encoding_roundtrip() {
|
async fn test_content_encoding_aws_chunked_with_effective_encoding_roundtrip() {
|
||||||
init_logging();
|
init_logging();
|
||||||
info!("aws-chunked,gzip should persist only gzip");
|
info!("aws-chunked,gzip should persist only gzip");
|
||||||
|
|||||||
@@ -30,6 +30,7 @@ mod tests {
|
|||||||
use base64::Engine as _;
|
use base64::Engine as _;
|
||||||
use base64::engine::general_purpose::STANDARD as BASE64;
|
use base64::engine::general_purpose::STANDARD as BASE64;
|
||||||
use rustfs_rio::{Checksum, ChecksumType as RioChecksumType};
|
use rustfs_rio::{Checksum, ChecksumType as RioChecksumType};
|
||||||
|
use serial_test::serial;
|
||||||
use sha2::{Digest, Sha256};
|
use sha2::{Digest, Sha256};
|
||||||
use tracing::info;
|
use tracing::info;
|
||||||
|
|
||||||
@@ -113,6 +114,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn test_copy_supports_all_checksum_algorithms() {
|
async fn test_copy_supports_all_checksum_algorithms() {
|
||||||
init_logging();
|
init_logging();
|
||||||
|
|
||||||
@@ -194,6 +196,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn test_copy_without_algorithm_preserves_every_supported_source_checksum() {
|
async fn test_copy_without_algorithm_preserves_every_supported_source_checksum() {
|
||||||
init_logging();
|
init_logging();
|
||||||
|
|
||||||
@@ -259,6 +262,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn test_copy_without_algorithm_preserves_composite_checksum_type() {
|
async fn test_copy_without_algorithm_preserves_composite_checksum_type() {
|
||||||
init_logging();
|
init_logging();
|
||||||
|
|
||||||
@@ -348,6 +352,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn test_copy_rejects_unknown_algorithm_without_destination_mutation() {
|
async fn test_copy_rejects_unknown_algorithm_without_destination_mutation() {
|
||||||
init_logging();
|
init_logging();
|
||||||
|
|
||||||
@@ -448,6 +453,7 @@ mod tests {
|
|||||||
/// bytes, return it in `CopyObjectResult.ChecksumSHA256`, and persist it so a checksum-mode
|
/// bytes, return it in `CopyObjectResult.ChecksumSHA256`, and persist it so a checksum-mode
|
||||||
/// HEAD on the destination returns the identical value.
|
/// HEAD on the destination returns the identical value.
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn test_copy_with_checksum_algorithm_returns_and_persists_sha256() {
|
async fn test_copy_with_checksum_algorithm_returns_and_persists_sha256() {
|
||||||
init_logging();
|
init_logging();
|
||||||
info!("Issue #4996: CopyObject with ChecksumAlgorithm=SHA256 must return and persist the checksum");
|
info!("Issue #4996: CopyObject with ChecksumAlgorithm=SHA256 must return and persist the checksum");
|
||||||
@@ -517,6 +523,7 @@ mod tests {
|
|||||||
/// No algorithm requested: when the source object already carries a checksum, the copy must
|
/// No algorithm requested: when the source object already carries a checksum, the copy must
|
||||||
/// preserve it on the destination (AWS default), visible via a checksum-mode HEAD.
|
/// preserve it on the destination (AWS default), visible via a checksum-mode HEAD.
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn test_copy_without_algorithm_preserves_source_checksum() {
|
async fn test_copy_without_algorithm_preserves_source_checksum() {
|
||||||
init_logging();
|
init_logging();
|
||||||
info!("Issue #4996: CopyObject without ChecksumAlgorithm must preserve the source object's checksum");
|
info!("Issue #4996: CopyObject without ChecksumAlgorithm must preserve the source object's checksum");
|
||||||
@@ -596,6 +603,7 @@ mod tests {
|
|||||||
/// checksum-not-inherited path, and exercises the CRC32 code path (a different branch of
|
/// checksum-not-inherited path, and exercises the CRC32 code path (a different branch of
|
||||||
/// ChecksumType::from_string than SHA256).
|
/// ChecksumType::from_string than SHA256).
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn test_copy_requested_algorithm_overrides_source_checksum() {
|
async fn test_copy_requested_algorithm_overrides_source_checksum() {
|
||||||
init_logging();
|
init_logging();
|
||||||
info!("Issue #4996: a requested CopyObject checksum algorithm must override the source object's algorithm");
|
info!("Issue #4996: a requested CopyObject checksum algorithm must override the source object's algorithm");
|
||||||
|
|||||||
@@ -22,9 +22,11 @@ mod tests {
|
|||||||
use aws_sdk_s3::types::{
|
use aws_sdk_s3::types::{
|
||||||
BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, MetadataDirective, StorageClass, VersioningConfiguration,
|
BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, MetadataDirective, StorageClass, VersioningConfiguration,
|
||||||
};
|
};
|
||||||
|
use serial_test::serial;
|
||||||
use tracing::info;
|
use tracing::info;
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn copy_object_standard_metadata_copy_replace_and_clear() {
|
async fn copy_object_standard_metadata_copy_replace_and_clear() {
|
||||||
init_logging();
|
init_logging();
|
||||||
info!("Issue #2789: self-copy metadata replacement must preserve object data");
|
info!("Issue #2789: self-copy metadata replacement must preserve object data");
|
||||||
@@ -298,6 +300,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn copy_object_replace_accepts_each_standard_field_independently() {
|
async fn copy_object_replace_accepts_each_standard_field_independently() {
|
||||||
init_logging();
|
init_logging();
|
||||||
let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
|
let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
|
||||||
@@ -413,6 +416,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn copy_object_replace_handles_versioned_multipart_source() {
|
async fn copy_object_replace_handles_versioned_multipart_source() {
|
||||||
init_logging();
|
init_logging();
|
||||||
let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
|
let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
|
||||||
@@ -526,6 +530,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn invalid_replacement_metadata_does_not_mutate_destination() {
|
async fn invalid_replacement_metadata_does_not_mutate_destination() {
|
||||||
init_logging();
|
init_logging();
|
||||||
let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
|
let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
|
||||||
|
|||||||
@@ -21,6 +21,7 @@ mod tests {
|
|||||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||||
use aws_sdk_s3::primitives::ByteStream;
|
use aws_sdk_s3::primitives::ByteStream;
|
||||||
use aws_sdk_s3::types::{BucketVersioningStatus, MetadataDirective, TaggingDirective, VersioningConfiguration};
|
use aws_sdk_s3::types::{BucketVersioningStatus, MetadataDirective, TaggingDirective, VersioningConfiguration};
|
||||||
|
use serial_test::serial;
|
||||||
use std::collections::BTreeMap;
|
use std::collections::BTreeMap;
|
||||||
|
|
||||||
async fn object_tags(client: &Client, bucket: &str, key: &str) -> BTreeMap<String, String> {
|
async fn object_tags(client: &Client, bucket: &str, key: &str) -> BTreeMap<String, String> {
|
||||||
@@ -38,6 +39,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn copy_object_applies_copy_replace_and_empty_tagging_directives() {
|
async fn copy_object_applies_copy_replace_and_empty_tagging_directives() {
|
||||||
init_logging();
|
init_logging();
|
||||||
let mut env = RustFSTestEnvironment::new()
|
let mut env = RustFSTestEnvironment::new()
|
||||||
@@ -303,6 +305,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn copy_object_tag_replacement_honors_request_tag_policy_denial() -> Result<(), Box<dyn std::error::Error + Send + Sync>>
|
async fn copy_object_tag_replacement_honors_request_tag_policy_denial() -> Result<(), Box<dyn std::error::Error + Send + Sync>>
|
||||||
{
|
{
|
||||||
init_logging();
|
init_logging();
|
||||||
|
|||||||
@@ -21,9 +21,11 @@ mod tests {
|
|||||||
use crate::common::{RustFSTestEnvironment, init_logging};
|
use crate::common::{RustFSTestEnvironment, init_logging};
|
||||||
use aws_sdk_s3::primitives::ByteStream;
|
use aws_sdk_s3::primitives::ByteStream;
|
||||||
use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
|
use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
|
||||||
|
use serial_test::serial;
|
||||||
use tracing::info;
|
use tracing::info;
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn test_self_copy_of_historical_version_restores_data_and_metadata() {
|
async fn test_self_copy_of_historical_version_restores_data_and_metadata() {
|
||||||
init_logging();
|
init_logging();
|
||||||
info!("Issue #4238: self-copy of a historical version must be allowed and preserve metadata");
|
info!("Issue #4238: self-copy of a historical version must be allowed and preserve metadata");
|
||||||
@@ -163,6 +165,7 @@ mod tests {
|
|||||||
/// version copied via `x-amz-copy-source-version-id` (SDK `CopySourceVersionId`), kept distinct
|
/// version copied via `x-amz-copy-source-version-id` (SDK `CopySourceVersionId`), kept distinct
|
||||||
/// from the newly created destination `x-amz-version-id`.
|
/// from the newly created destination `x-amz-version-id`.
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn test_copy_of_non_latest_source_version_returns_copy_source_version_id() {
|
async fn test_copy_of_non_latest_source_version_returns_copy_source_version_id() {
|
||||||
init_logging();
|
init_logging();
|
||||||
info!("Issue #4976: versioned CopyObject must return x-amz-copy-source-version-id for the exact source version");
|
info!("Issue #4976: versioned CopyObject must return x-amz-copy-source-version-id for the exact source version");
|
||||||
|
|||||||
@@ -47,6 +47,7 @@ mod tests {
|
|||||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||||
use rustfs_signer::sign_v4;
|
use rustfs_signer::sign_v4;
|
||||||
use s3s::Body;
|
use s3s::Body;
|
||||||
|
use serial_test::serial;
|
||||||
use std::error::Error;
|
use std::error::Error;
|
||||||
|
|
||||||
/// Signed raw `PUT` copy request with an explicit copy-source conditional
|
/// Signed raw `PUT` copy request with an explicit copy-source conditional
|
||||||
@@ -83,6 +84,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn test_copy_source_if_unmodified_since_valid_and_invalid() -> Result<(), Box<dyn Error + Send + Sync>> {
|
async fn test_copy_source_if_unmodified_since_valid_and_invalid() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||||
init_logging();
|
init_logging();
|
||||||
let mut env = RustFSTestEnvironment::new().await?;
|
let mut env = RustFSTestEnvironment::new().await?;
|
||||||
|
|||||||
@@ -25,11 +25,13 @@
|
|||||||
mod tests {
|
mod tests {
|
||||||
use crate::common::{RustFSTestEnvironment, init_logging};
|
use crate::common::{RustFSTestEnvironment, init_logging};
|
||||||
use aws_sdk_s3::types::{BucketLocationConstraint, CreateBucketConfiguration};
|
use aws_sdk_s3::types::{BucketLocationConstraint, CreateBucketConfiguration};
|
||||||
|
use serial_test::serial;
|
||||||
use std::error::Error;
|
use std::error::Error;
|
||||||
|
|
||||||
/// `CreateBucket` with a `LocationConstraint` body must pass SigV4 validation
|
/// `CreateBucket` with a `LocationConstraint` body must pass SigV4 validation
|
||||||
/// and create the bucket, mirroring `minio-go` `MakeBucket(bucket, "us-east-1")`.
|
/// and create the bucket, mirroring `minio-go` `MakeBucket(bucket, "us-east-1")`.
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn test_create_bucket_with_us_east_1_location_constraint() -> Result<(), Box<dyn Error + Send + Sync>> {
|
async fn test_create_bucket_with_us_east_1_location_constraint() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||||
init_logging();
|
init_logging();
|
||||||
let mut env = RustFSTestEnvironment::new().await?;
|
let mut env = RustFSTestEnvironment::new().await?;
|
||||||
@@ -60,6 +62,7 @@ mod tests {
|
|||||||
/// A plain `CreateBucket` (no body) must also succeed; guards against a
|
/// A plain `CreateBucket` (no body) must also succeed; guards against a
|
||||||
/// regression where an empty body would be hashed incorrectly during SigV4.
|
/// regression where an empty body would be hashed incorrectly during SigV4.
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn test_create_bucket_without_location_constraint() -> Result<(), Box<dyn Error + Send + Sync>> {
|
async fn test_create_bucket_without_location_constraint() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||||
init_logging();
|
init_logging();
|
||||||
let mut env = RustFSTestEnvironment::new().await?;
|
let mut env = RustFSTestEnvironment::new().await?;
|
||||||
|
|||||||
@@ -15,6 +15,7 @@
|
|||||||
use aws_sdk_s3::primitives::ByteStream;
|
use aws_sdk_s3::primitives::ByteStream;
|
||||||
use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
|
use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
|
||||||
use rustfs_data_usage::DataUsageInfo;
|
use rustfs_data_usage::DataUsageInfo;
|
||||||
|
use serial_test::serial;
|
||||||
use tokio::time::{Duration, sleep};
|
use tokio::time::{Duration, sleep};
|
||||||
|
|
||||||
use crate::common::{FAST_DATA_USAGE_SCANNER_ENV, RustFSTestEnvironment, TEST_BUCKET, awscurl_get, init_logging};
|
use crate::common::{FAST_DATA_USAGE_SCANNER_ENV, RustFSTestEnvironment, TEST_BUCKET, awscurl_get, init_logging};
|
||||||
@@ -59,6 +60,7 @@ where
|
|||||||
/// Regression test for data usage accuracy (issue #1012).
|
/// Regression test for data usage accuracy (issue #1012).
|
||||||
/// Launches rustfs, writes 1000 objects, then asserts admin data usage reports the full count.
|
/// Launches rustfs, writes 1000 objects, then asserts admin data usage reports the full count.
|
||||||
#[tokio::test(flavor = "multi_thread")]
|
#[tokio::test(flavor = "multi_thread")]
|
||||||
|
#[serial]
|
||||||
#[ignore = "Starts a rustfs server and requires awscurl; enable when running full E2E"]
|
#[ignore = "Starts a rustfs server and requires awscurl; enable when running full E2E"]
|
||||||
async fn data_usage_reports_all_objects() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
async fn data_usage_reports_all_objects() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||||
init_logging();
|
init_logging();
|
||||||
@@ -116,6 +118,7 @@ async fn data_usage_reports_all_objects() -> Result<(), Box<dyn std::error::Erro
|
|||||||
/// Regression test for issue #3898.
|
/// Regression test for issue #3898.
|
||||||
/// Versioned buckets should expose versions and delete markers through admin data usage.
|
/// Versioned buckets should expose versions and delete markers through admin data usage.
|
||||||
#[tokio::test(flavor = "multi_thread")]
|
#[tokio::test(flavor = "multi_thread")]
|
||||||
|
#[serial]
|
||||||
#[ignore = "Starts a rustfs server and requires awscurl; enable when running full E2E"]
|
#[ignore = "Starts a rustfs server and requires awscurl; enable when running full E2E"]
|
||||||
async fn data_usage_reports_versioned_objects_and_delete_markers() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
async fn data_usage_reports_versioned_objects_and_delete_markers() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||||
init_logging();
|
init_logging();
|
||||||
|
|||||||
@@ -18,6 +18,7 @@ mod tests {
|
|||||||
use aws_sdk_s3::Client;
|
use aws_sdk_s3::Client;
|
||||||
use aws_sdk_s3::primitives::ByteStream;
|
use aws_sdk_s3::primitives::ByteStream;
|
||||||
use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
|
use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
|
||||||
|
use serial_test::serial;
|
||||||
|
|
||||||
async fn create_versioned_bucket(client: &Client, bucket: &str) {
|
async fn create_versioned_bucket(client: &Client, bucket: &str) {
|
||||||
client
|
client
|
||||||
@@ -71,6 +72,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn test_versioning_only_delete_marker_has_minio_compatible_visibility_for_migration_proof() {
|
async fn test_versioning_only_delete_marker_has_minio_compatible_visibility_for_migration_proof() {
|
||||||
init_logging();
|
init_logging();
|
||||||
let mut env = RustFSTestEnvironment::new().await.expect("create test environment");
|
let mut env = RustFSTestEnvironment::new().await.expect("create test environment");
|
||||||
@@ -111,6 +113,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn test_versioning_delete_marker_plus_history_remains_visible_for_migration_proof() {
|
async fn test_versioning_delete_marker_plus_history_remains_visible_for_migration_proof() {
|
||||||
init_logging();
|
init_logging();
|
||||||
let mut env = RustFSTestEnvironment::new().await.expect("create test environment");
|
let mut env = RustFSTestEnvironment::new().await.expect("create test environment");
|
||||||
|
|||||||
@@ -24,6 +24,7 @@ mod tests {
|
|||||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||||
use rustfs_signer::sign_v4;
|
use rustfs_signer::sign_v4;
|
||||||
use s3s::Body;
|
use s3s::Body;
|
||||||
|
use serial_test::serial;
|
||||||
use std::error::Error;
|
use std::error::Error;
|
||||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||||
use tokio::net::TcpStream;
|
use tokio::net::TcpStream;
|
||||||
@@ -91,6 +92,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn test_delete_object_version_without_content_length_succeeds() -> Result<(), Box<dyn Error + Send + Sync>> {
|
async fn test_delete_object_version_without_content_length_succeeds() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||||
init_logging();
|
init_logging();
|
||||||
info!("🧪 TEST: signed DELETE Object?versionId succeeds without Content-Length");
|
info!("🧪 TEST: signed DELETE Object?versionId succeeds without Content-Length");
|
||||||
|
|||||||
@@ -29,6 +29,7 @@ mod tests {
|
|||||||
use crate::common::{RustFSTestEnvironment, init_logging};
|
use crate::common::{RustFSTestEnvironment, init_logging};
|
||||||
use aws_sdk_s3::Client;
|
use aws_sdk_s3::Client;
|
||||||
use aws_sdk_s3::types::{BucketVersioningStatus, Delete, ObjectIdentifier, VersioningConfiguration};
|
use aws_sdk_s3::types::{BucketVersioningStatus, Delete, ObjectIdentifier, VersioningConfiguration};
|
||||||
|
use serial_test::serial;
|
||||||
use tracing::info;
|
use tracing::info;
|
||||||
|
|
||||||
fn create_s3_client(env: &RustFSTestEnvironment) -> Client {
|
fn create_s3_client(env: &RustFSTestEnvironment) -> Client {
|
||||||
@@ -41,6 +42,7 @@ mod tests {
|
|||||||
/// a versioned bucket, calling `list_object_versions` **immediately** (with
|
/// a versioned bucket, calling `list_object_versions` **immediately** (with
|
||||||
/// no sleep) returns the newly-created DeleteMarker with `is_latest = true`.
|
/// no sleep) returns the newly-created DeleteMarker with `is_latest = true`.
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn test_delete_objects_delete_marker_immediately_visible() {
|
async fn test_delete_objects_delete_marker_immediately_visible() {
|
||||||
init_logging();
|
init_logging();
|
||||||
info!("🧪 TEST: DeleteMarker from delete_objects is immediately visible via list_object_versions");
|
info!("🧪 TEST: DeleteMarker from delete_objects is immediately visible via list_object_versions");
|
||||||
@@ -188,6 +190,7 @@ mod tests {
|
|||||||
/// a single `delete_objects` call all have their delete markers visible
|
/// a single `delete_objects` call all have their delete markers visible
|
||||||
/// immediately afterwards.
|
/// immediately afterwards.
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn test_delete_objects_multiple_keys_delete_markers_immediately_visible() {
|
async fn test_delete_objects_multiple_keys_delete_markers_immediately_visible() {
|
||||||
init_logging();
|
init_logging();
|
||||||
info!("🧪 TEST: Multiple delete markers from delete_objects are immediately visible");
|
info!("🧪 TEST: Multiple delete markers from delete_objects are immediately visible");
|
||||||
|
|||||||
@@ -33,6 +33,7 @@ mod tests {
|
|||||||
use crate::common::{RustFSTestEnvironment, init_logging};
|
use crate::common::{RustFSTestEnvironment, init_logging};
|
||||||
use aws_sdk_s3::primitives::ByteStream;
|
use aws_sdk_s3::primitives::ByteStream;
|
||||||
use aws_sdk_s3::types::{BucketVersioningStatus, Delete, ObjectIdentifier, VersioningConfiguration};
|
use aws_sdk_s3::types::{BucketVersioningStatus, Delete, ObjectIdentifier, VersioningConfiguration};
|
||||||
|
use serial_test::serial;
|
||||||
use std::error::Error;
|
use std::error::Error;
|
||||||
use tracing::info;
|
use tracing::info;
|
||||||
|
|
||||||
@@ -50,6 +51,7 @@ mod tests {
|
|||||||
/// 4. Verify the object is NOT in LIST
|
/// 4. Verify the object is NOT in LIST
|
||||||
/// 5. Verify HEAD returns 404
|
/// 5. Verify HEAD returns 404
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn test_delete_removes_object_from_list() -> TestResult {
|
async fn test_delete_removes_object_from_list() -> TestResult {
|
||||||
init_logging();
|
init_logging();
|
||||||
info!("RT-05: delete removes object from list");
|
info!("RT-05: delete removes object from list");
|
||||||
@@ -130,6 +132,7 @@ mod tests {
|
|||||||
/// Regression pattern: batch delete returns success but some objects
|
/// Regression pattern: batch delete returns success but some objects
|
||||||
/// remain in LIST.
|
/// remain in LIST.
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn test_batch_delete_removes_all_objects() -> TestResult {
|
async fn test_batch_delete_removes_all_objects() -> TestResult {
|
||||||
init_logging();
|
init_logging();
|
||||||
info!("RT-05c: batch delete removes all objects");
|
info!("RT-05c: batch delete removes all objects");
|
||||||
@@ -209,6 +212,7 @@ mod tests {
|
|||||||
/// Covers the pattern where permanent deletion of a specific version
|
/// Covers the pattern where permanent deletion of a specific version
|
||||||
/// fails with FileAccessDenied (rustfs#4978).
|
/// fails with FileAccessDenied (rustfs#4978).
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn test_versioned_permanent_delete() -> TestResult {
|
async fn test_versioned_permanent_delete() -> TestResult {
|
||||||
init_logging();
|
init_logging();
|
||||||
info!("RT-05d: versioned permanent delete");
|
info!("RT-05d: versioned permanent delete");
|
||||||
@@ -279,6 +283,7 @@ mod tests {
|
|||||||
/// Covers the pattern where creating a delete marker and then listing
|
/// Covers the pattern where creating a delete marker and then listing
|
||||||
/// versions shows incorrect state (rustfs#760).
|
/// versions shows incorrect state (rustfs#760).
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn test_versioned_delete_marker_and_list_consistency() -> TestResult {
|
async fn test_versioned_delete_marker_and_list_consistency() -> TestResult {
|
||||||
init_logging();
|
init_logging();
|
||||||
info!("RT-05e: versioned delete marker and list consistency");
|
info!("RT-05e: versioned delete marker and list consistency");
|
||||||
@@ -374,6 +379,7 @@ mod tests {
|
|||||||
/// Regression pattern: after delete, the object data files remain on disk
|
/// Regression pattern: after delete, the object data files remain on disk
|
||||||
/// (rustfs#5029: Node Does Not Remove Files After Reconnect).
|
/// (rustfs#5029: Node Does Not Remove Files After Reconnect).
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn test_delete_removes_object_head_returns_404() -> TestResult {
|
async fn test_delete_removes_object_head_returns_404() -> TestResult {
|
||||||
init_logging();
|
init_logging();
|
||||||
info!("RT-05f: delete → HEAD 404 consistency");
|
info!("RT-05f: delete → HEAD 404 consistency");
|
||||||
|
|||||||
@@ -32,6 +32,7 @@
|
|||||||
mod tests {
|
mod tests {
|
||||||
use crate::common::{RustFSTestClusterEnvironment, init_logging};
|
use crate::common::{RustFSTestClusterEnvironment, init_logging};
|
||||||
use aws_sdk_s3::primitives::ByteStream;
|
use aws_sdk_s3::primitives::ByteStream;
|
||||||
|
use serial_test::serial;
|
||||||
use std::error::Error;
|
use std::error::Error;
|
||||||
use tokio::time::{Duration, sleep};
|
use tokio::time::{Duration, sleep};
|
||||||
use tracing::info;
|
use tracing::info;
|
||||||
@@ -49,6 +50,7 @@ mod tests {
|
|||||||
/// 3. Verify all nodes report healthy
|
/// 3. Verify all nodes report healthy
|
||||||
/// 4. Verify S3 operations work through any node
|
/// 4. Verify S3 operations work through any node
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn test_four_node_cluster_startup_and_health() -> TestResult {
|
async fn test_four_node_cluster_startup_and_health() -> TestResult {
|
||||||
init_logging();
|
init_logging();
|
||||||
info!("RT-10: 4-node cluster startup and health");
|
info!("RT-10: 4-node cluster startup and health");
|
||||||
@@ -101,6 +103,7 @@ mod tests {
|
|||||||
/// Regression pattern: after a node restart, it cannot rejoin the cluster
|
/// Regression pattern: after a node restart, it cannot rejoin the cluster
|
||||||
/// or enters a faulty state (rustfs#2601).
|
/// or enters a faulty state (rustfs#2601).
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn test_cluster_survives_node_restart() -> TestResult {
|
async fn test_cluster_survives_node_restart() -> TestResult {
|
||||||
init_logging();
|
init_logging();
|
||||||
info!("RT-10b: cluster survives node restart");
|
info!("RT-10b: cluster survives node restart");
|
||||||
@@ -165,6 +168,7 @@ mod tests {
|
|||||||
/// Regression pattern: bucket metadata is not replicated to all nodes,
|
/// Regression pattern: bucket metadata is not replicated to all nodes,
|
||||||
/// causing NoSuchBucket errors on some nodes (rustfs#3191).
|
/// causing NoSuchBucket errors on some nodes (rustfs#3191).
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn test_bucket_visible_from_all_nodes() -> TestResult {
|
async fn test_bucket_visible_from_all_nodes() -> TestResult {
|
||||||
init_logging();
|
init_logging();
|
||||||
info!("RT-10c: bucket visible from all nodes");
|
info!("RT-10c: bucket visible from all nodes");
|
||||||
|
|||||||
@@ -23,6 +23,7 @@ use aws_sdk_s3::config::{Credentials, Region};
|
|||||||
use aws_sdk_s3::primitives::ByteStream;
|
use aws_sdk_s3::primitives::ByteStream;
|
||||||
use aws_sdk_s3::types::{Delete, ObjectIdentifier, Tag, Tagging};
|
use aws_sdk_s3::types::{Delete, ObjectIdentifier, Tag, Tagging};
|
||||||
use aws_sdk_s3::{Client, Config};
|
use aws_sdk_s3::{Client, Config};
|
||||||
|
use serial_test::serial;
|
||||||
use tracing::info;
|
use tracing::info;
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
|
|
||||||
@@ -173,6 +174,7 @@ async fn cleanup_bucket_and_object(admin: &Client, bucket: &str, key: &str) {
|
|||||||
|
|
||||||
/// IAM identity policy: GetObject allowed only when `s3:ExistingObjectTag/security` == `public`.
|
/// IAM identity policy: GetObject allowed only when `s3:ExistingObjectTag/security` == `public`.
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn test_e2e_iam_policy_existing_object_tag_get_object() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
async fn test_e2e_iam_policy_existing_object_tag_get_object() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||||
init_logging();
|
init_logging();
|
||||||
if !awscurl_available() {
|
if !awscurl_available() {
|
||||||
@@ -231,6 +233,7 @@ async fn test_e2e_iam_policy_existing_object_tag_get_object() -> Result<(), Box<
|
|||||||
|
|
||||||
/// Bucket policy: same `ExistingObjectTag` condition; user has no canned IAM policy attached.
|
/// Bucket policy: same `ExistingObjectTag` condition; user has no canned IAM policy attached.
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn test_e2e_bucket_policy_existing_object_tag_get_object() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
async fn test_e2e_bucket_policy_existing_object_tag_get_object() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||||
init_logging();
|
init_logging();
|
||||||
if !awscurl_available() {
|
if !awscurl_available() {
|
||||||
@@ -292,6 +295,7 @@ async fn test_e2e_bucket_policy_existing_object_tag_get_object() -> Result<(), B
|
|||||||
|
|
||||||
/// STS `AssumeRole` with inline `Policy` (session policy): GetObject only when `ExistingObjectTag/security` is `public`.
|
/// STS `AssumeRole` with inline `Policy` (session policy): GetObject only when `ExistingObjectTag/security` is `public`.
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn test_e2e_sts_assume_role_session_policy_existing_object_tag() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
async fn test_e2e_sts_assume_role_session_policy_existing_object_tag() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||||
init_logging();
|
init_logging();
|
||||||
if !awscurl_available() {
|
if !awscurl_available() {
|
||||||
@@ -368,6 +372,7 @@ async fn test_e2e_sts_assume_role_session_policy_existing_object_tag() -> Result
|
|||||||
|
|
||||||
/// STS inline session policy: DeleteObjects must evaluate `s3:DeleteObject` per requested object key.
|
/// STS inline session policy: DeleteObjects must evaluate `s3:DeleteObject` per requested object key.
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn test_e2e_sts_session_policy_delete_objects_object_prefix_only() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
async fn test_e2e_sts_session_policy_delete_objects_object_prefix_only() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||||
init_logging();
|
init_logging();
|
||||||
if !awscurl_available() {
|
if !awscurl_available() {
|
||||||
|
|||||||
@@ -4,8 +4,8 @@ This module is the shared failure-injection boundary for replication end-to-end
|
|||||||
|
|
||||||
`FakeS3Target::start()` creates the listener. Add target buckets with `create_bucket`, point a RustFS remote target at `address()`, use `FAKE_ACCESS_KEY` / `FAKE_SECRET_KEY`, then enqueue per-operation faults with `inject`. Faults for one operation are consumed in FIFO order and do not consume faults queued for another operation. A fault is consumed only after `s3s` verifies the full request signature, so anonymous, other-access-key, and bad-signature traffic cannot disturb a script.
|
`FakeS3Target::start()` creates the listener. Add target buckets with `create_bucket`, point a RustFS remote target at `address()`, use `FAKE_ACCESS_KEY` / `FAKE_SECRET_KEY`, then enqueue per-operation faults with `inject`. Faults for one operation are consumed in FIFO order and do not consume faults queued for another operation. A fault is consumed only after `s3s` verifies the full request signature, so anonymous, other-access-key, and bad-signature traffic cannot disturb a script.
|
||||||
|
|
||||||
Supported data operations are HeadBucket, GetBucketVersioning, PUT/GET/HEAD/DELETE Object, Get/Put/Delete ObjectTagging (tags live per version; Put replaces the whole set, Delete clears it), and create/upload/complete/abort multipart upload. `create_bucket` models general-purpose buckets in S3's shared global namespace; account-regional namespace buckets and their `-an` names are intentionally out of scope. Buckets are versioned: PUT creates a version, DELETE without `versionId` creates a delete marker, and DELETE with `versionId` removes exactly that version. Internal source version IDs must be UUIDs and are stored canonically. Source mtime is honored only for source-replication PUT/DELETE requests; absent or invalid values use receipt time, matching RustFS, while multipart completion always uses receipt time. Replicated versions are ordered newest-first by source mtime so late older versions and delete markers do not become current. Equal mtimes prefer objects over delete markers, then canonical UUID order; RustFS's internal FileMeta signature tie-break is intentionally out of scope because it is not part of the target S3 protocol. Multipart part numbers follow S3's `1..=10000` range, and every completed part except the final part must be at least 5 MiB.
|
Supported data operations are HeadBucket, GetBucketVersioning, PUT/GET/HEAD/DELETE Object, and create/upload/complete/abort multipart upload. `create_bucket` models general-purpose buckets in S3's shared global namespace; account-regional namespace buckets and their `-an` names are intentionally out of scope. Buckets are versioned: PUT creates a version, DELETE without `versionId` creates a delete marker, and DELETE with `versionId` removes exactly that version. Internal source version IDs must be UUIDs and are stored canonically. Source mtime is honored only for source-replication PUT/DELETE requests; absent or invalid values use receipt time, matching RustFS, while multipart completion always uses receipt time. Replicated versions are ordered newest-first by source mtime so late older versions and delete markers do not become current. Equal mtimes prefer objects over delete markers, then canonical UUID order; RustFS's internal FileMeta signature tie-break is intentionally out of scope because it is not part of the target S3 protocol. Multipart part numbers follow S3's `1..=10000` range, and every completed part except the final part must be at least 5 MiB.
|
||||||
|
|
||||||
Fault actions cover HTTP 401/403/503 responses, pre-dispatch delay, connection abort when a logical request-body threshold is reached, streaming slow drain, and a deliberately wrong response ETag (including multipart-complete XML). `requests()` returns the ordered, credential-free request journal for assertions. Each record also journals a `ProxyHeaderSnapshot` — the read-proxy anti-loop marker (`x-{rustfs,minio}-source-proxy-request`), the replication-check exemption header, and the client SSE-C header family (algorithm and key-MD5 values; for the key itself only its presence) — so proxy tests can pin the exact wire contract.
|
Fault actions cover HTTP 401/403/503 responses, pre-dispatch delay, connection abort when a logical request-body threshold is reached, streaming slow drain, and a deliberately wrong response ETag (including multipart-complete XML). `requests()` returns the ordered, credential-free request journal for assertions.
|
||||||
|
|
||||||
The listener is loopback-only. It admits at most 64 active connections and two concurrently buffered request bodies; authenticated multipart-complete XML collection and assembly take both body permits. Keep-alive is disabled, request-header reads are bounded to 30 seconds, a parsed request is bounded to 65 seconds, and the complete connection lifetime is bounded to 100 seconds. It retains at most 256 buckets, 4,096 journal entries, 4,096 scripted faults, 4,096 object versions, 256 multipart uploads, and 10,000 multipart parts. Retained identifiers are capped at 1 KiB, user metadata at 2 KiB, and content type at 1 KiB. A PUT or uploaded part is capped at 64 MiB; a completed multipart object and all stored object/part data are capped at 128 MiB. Body drain, body-permit waits, delay, and slow-drain execution are bounded to 30 seconds; each slow-drain slice delay must be below that bound.
|
The listener is loopback-only. It admits at most 64 active connections and two concurrently buffered request bodies; authenticated multipart-complete XML collection and assembly take both body permits. Keep-alive is disabled, request-header reads are bounded to 30 seconds, a parsed request is bounded to 65 seconds, and the complete connection lifetime is bounded to 100 seconds. It retains at most 256 buckets, 4,096 journal entries, 4,096 scripted faults, 4,096 object versions, 256 multipart uploads, and 10,000 multipart parts. Retained identifiers are capped at 1 KiB, user metadata at 2 KiB, and content type at 1 KiB. A PUT or uploaded part is capped at 64 MiB; a completed multipart object and all stored object/part data are capped at 128 MiB. Body drain, body-permit waits, delay, and slow-drain execution are bounded to 30 seconds; each slow-drain slice delay must be below that bound.
|
||||||
|
|||||||
@@ -30,12 +30,10 @@ use s3s::access::{S3Access, S3AccessContext};
|
|||||||
use s3s::auth::SimpleAuth;
|
use s3s::auth::SimpleAuth;
|
||||||
use s3s::dto::{
|
use s3s::dto::{
|
||||||
AbortMultipartUploadInput, AbortMultipartUploadOutput, CompleteMultipartUploadInput, CompleteMultipartUploadOutput,
|
AbortMultipartUploadInput, AbortMultipartUploadOutput, CompleteMultipartUploadInput, CompleteMultipartUploadOutput,
|
||||||
CreateMultipartUploadInput, CreateMultipartUploadOutput, DeleteMarkerEntry, DeleteObjectInput, DeleteObjectOutput,
|
CreateMultipartUploadInput, CreateMultipartUploadOutput, DeleteMarkerEntry, DeleteObjectInput, DeleteObjectOutput, ETag,
|
||||||
DeleteObjectTaggingInput, DeleteObjectTaggingOutput, ETag, GetBucketVersioningInput, GetBucketVersioningOutput,
|
GetBucketVersioningInput, GetBucketVersioningOutput, GetObjectInput, GetObjectOutput, HeadBucketInput, HeadBucketOutput,
|
||||||
GetObjectInput, GetObjectOutput, GetObjectTaggingInput, GetObjectTaggingOutput, HeadBucketInput, HeadBucketOutput,
|
|
||||||
HeadObjectInput, HeadObjectOutput, ListObjectVersionsInput, ListObjectVersionsOutput, ObjectVersionId, PutObjectInput,
|
HeadObjectInput, HeadObjectOutput, ListObjectVersionsInput, ListObjectVersionsOutput, ObjectVersionId, PutObjectInput,
|
||||||
PutObjectOutput, PutObjectTaggingInput, PutObjectTaggingOutput, StreamingBlob, Tag, TagSet, Timestamp, TimestampFormat,
|
PutObjectOutput, StreamingBlob, Timestamp, TimestampFormat, UploadPartInput, UploadPartOutput,
|
||||||
UploadPartInput, UploadPartOutput,
|
|
||||||
};
|
};
|
||||||
use s3s::service::{S3Service, S3ServiceBuilder};
|
use s3s::service::{S3Service, S3ServiceBuilder};
|
||||||
use s3s::validation::{AwsNameValidation, NameValidation};
|
use s3s::validation::{AwsNameValidation, NameValidation};
|
||||||
@@ -78,25 +76,6 @@ const SOURCE_MTIME_HEADERS: [&str; 2] = ["x-rustfs-source-mtime", "x-minio-sourc
|
|||||||
const SOURCE_REPLICATION_REQUEST_HEADERS: [&str; 2] =
|
const SOURCE_REPLICATION_REQUEST_HEADERS: [&str; 2] =
|
||||||
["x-rustfs-source-replication-request", "x-minio-source-replication-request"];
|
["x-rustfs-source-replication-request", "x-minio-source-replication-request"];
|
||||||
const SOURCE_ETAG_HEADERS: [&str; 2] = ["x-rustfs-source-etag", "x-minio-source-etag"];
|
const SOURCE_ETAG_HEADERS: [&str; 2] = ["x-rustfs-source-etag", "x-minio-source-etag"];
|
||||||
const SOURCE_TAGGING_TIMESTAMP_HEADERS: [&str; 2] = [
|
|
||||||
"x-rustfs-source-replication-tagging-timestamp",
|
|
||||||
"x-minio-source-replication-tagging-timestamp",
|
|
||||||
];
|
|
||||||
const SOURCE_RETENTION_TIMESTAMP_HEADERS: [&str; 2] = [
|
|
||||||
"x-rustfs-source-replication-retention-timestamp",
|
|
||||||
"x-minio-source-replication-retention-timestamp",
|
|
||||||
];
|
|
||||||
const SOURCE_LEGALHOLD_TIMESTAMP_HEADERS: [&str; 2] = [
|
|
||||||
"x-rustfs-source-replication-legalhold-timestamp",
|
|
||||||
"x-minio-source-replication-legalhold-timestamp",
|
|
||||||
];
|
|
||||||
/// Wire prefix of the SSE-C passthrough replication transport headers
|
|
||||||
/// (`X-Rustfs-Replication-*`). In the default mode the fake stores them like a
|
|
||||||
/// RustFS target and echoes SSE-C evidence back on HEAD/GET; with
|
|
||||||
/// [`FakeS3Target::drop_unlisted_replication_headers`] it models MinIO /
|
|
||||||
/// generic S3, which silently discard unknown x-* headers.
|
|
||||||
const REPLICATION_SSE_TRANSPORT_PREFIX: &str = "x-rustfs-replication-";
|
|
||||||
const REPLICATION_SSEC_ALGORITHM_TRANSPORT_HEADER: &str = "x-rustfs-replication-ssec-algorithm";
|
|
||||||
const RESERVED_BUCKET_PREFIXES: [&str; 3] = ["xn--", "sthree-", "amzn-s3-demo-"];
|
const RESERVED_BUCKET_PREFIXES: [&str; 3] = ["xn--", "sthree-", "amzn-s3-demo-"];
|
||||||
const RESERVED_BUCKET_SUFFIXES: [&str; 6] = ["-s3alias", "--ol-s3", ".mrap", "--x-s3", "--table-s3", "-an"];
|
const RESERVED_BUCKET_SUFFIXES: [&str; 6] = ["-s3alias", "--ol-s3", ".mrap", "--x-s3", "--table-s3", "-an"];
|
||||||
|
|
||||||
@@ -112,9 +91,6 @@ pub enum Operation {
|
|||||||
GetObject,
|
GetObject,
|
||||||
HeadObject,
|
HeadObject,
|
||||||
DeleteObject,
|
DeleteObject,
|
||||||
GetObjectTagging,
|
|
||||||
PutObjectTagging,
|
|
||||||
DeleteObjectTagging,
|
|
||||||
ListObjectVersions,
|
ListObjectVersions,
|
||||||
CreateMultipartUpload,
|
CreateMultipartUpload,
|
||||||
UploadPart,
|
UploadPart,
|
||||||
@@ -142,61 +118,6 @@ pub enum FaultAction {
|
|||||||
WrongEtag,
|
WrongEtag,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Replication LWW timestamp headers observed on a request, journaled so
|
|
||||||
/// sender-side tests can assert what a real target would receive.
|
|
||||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
|
||||||
pub struct ReplicationTimestampHeaders {
|
|
||||||
pub tagging: Option<String>,
|
|
||||||
pub retention: Option<String>,
|
|
||||||
pub legalhold: Option<String>,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl ReplicationTimestampHeaders {
|
|
||||||
fn from_headers(headers: &HeaderMap) -> Self {
|
|
||||||
Self {
|
|
||||||
tagging: header_value(headers, &SOURCE_TAGGING_TIMESTAMP_HEADERS).map(bounded_journal_value),
|
|
||||||
retention: header_value(headers, &SOURCE_RETENTION_TIMESTAMP_HEADERS).map(bounded_journal_value),
|
|
||||||
legalhold: header_value(headers, &SOURCE_LEGALHOLD_TIMESTAMP_HEADERS).map(bounded_journal_value),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Read-proxy related headers observed on a request, journaled so proxy
|
|
||||||
/// tests can assert the exact wire contract: the anti-loop marker present,
|
|
||||||
/// the replication-check exemption absent, and the client SSE-C key family
|
|
||||||
/// forwarded verbatim. The SSE-C key value itself is never retained — only
|
|
||||||
/// its presence.
|
|
||||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
|
||||||
pub struct ProxyHeaderSnapshot {
|
|
||||||
pub source_proxy_request: Option<String>,
|
|
||||||
pub replication_check: Option<String>,
|
|
||||||
pub ssec_algorithm: Option<String>,
|
|
||||||
pub ssec_key_present: bool,
|
|
||||||
pub ssec_key_md5: Option<String>,
|
|
||||||
/// Whether the request carried any `X-Rustfs-Replication-*` SSE-C
|
|
||||||
/// passthrough transport header, so fail-closed tests can assert the
|
|
||||||
/// sender really shipped the material a dropping target discarded.
|
|
||||||
pub ssec_transport_present: bool,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl ProxyHeaderSnapshot {
|
|
||||||
fn from_headers(headers: &HeaderMap) -> Self {
|
|
||||||
Self {
|
|
||||||
source_proxy_request: header_value(headers, &["x-rustfs-source-proxy-request", "x-minio-source-proxy-request"])
|
|
||||||
.map(bounded_journal_value),
|
|
||||||
replication_check: header_value(headers, &["x-rustfs-source-replication-check", "x-minio-source-replication-check"])
|
|
||||||
.map(bounded_journal_value),
|
|
||||||
ssec_algorithm: header_value(headers, &["x-amz-server-side-encryption-customer-algorithm"])
|
|
||||||
.map(bounded_journal_value),
|
|
||||||
ssec_key_present: headers.contains_key("x-amz-server-side-encryption-customer-key"),
|
|
||||||
ssec_key_md5: header_value(headers, &["x-amz-server-side-encryption-customer-key-md5"]).map(bounded_journal_value),
|
|
||||||
ssec_transport_present: headers
|
|
||||||
.keys()
|
|
||||||
.any(|name| name.as_str().starts_with(REPLICATION_SSE_TRANSPORT_PREFIX)),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Credential-free request metadata retained for deterministic assertions.
|
/// Credential-free request metadata retained for deterministic assertions.
|
||||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
pub struct RequestRecord {
|
pub struct RequestRecord {
|
||||||
@@ -210,8 +131,6 @@ pub struct RequestRecord {
|
|||||||
pub part_number: Option<i32>,
|
pub part_number: Option<i32>,
|
||||||
pub content_length: Option<u64>,
|
pub content_length: Option<u64>,
|
||||||
pub consumed_bytes: Option<usize>,
|
pub consumed_bytes: Option<usize>,
|
||||||
pub replication_timestamps: ReplicationTimestampHeaders,
|
|
||||||
pub proxy_headers: ProxyHeaderSnapshot,
|
|
||||||
pub fault: Option<FaultAction>,
|
pub fault: Option<FaultAction>,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -227,10 +146,6 @@ struct ControlState {
|
|||||||
struct StoreState {
|
struct StoreState {
|
||||||
assign_own_version_ids: bool,
|
assign_own_version_ids: bool,
|
||||||
assign_own_multipart_version_ids: bool,
|
assign_own_multipart_version_ids: bool,
|
||||||
/// MinIO-like mode: silently discard non-whitelisted replication
|
|
||||||
/// transport headers instead of storing them (see
|
|
||||||
/// [`REPLICATION_SSE_TRANSPORT_PREFIX`]).
|
|
||||||
drop_unlisted_replication_headers: bool,
|
|
||||||
buckets: HashMap<String, BucketState>,
|
buckets: HashMap<String, BucketState>,
|
||||||
uploads: HashMap<String, MultipartState>,
|
uploads: HashMap<String, MultipartState>,
|
||||||
total_bytes: usize,
|
total_bytes: usize,
|
||||||
@@ -252,12 +167,6 @@ struct ObjectVersion {
|
|||||||
delete_marker: bool,
|
delete_marker: bool,
|
||||||
content_type: Option<String>,
|
content_type: Option<String>,
|
||||||
metadata: Option<HashMap<String, String>>,
|
metadata: Option<HashMap<String, String>>,
|
||||||
/// Object tags as ordered key/value pairs (PutObjectTagging replaces the
|
|
||||||
/// whole set, DeleteObjectTagging clears it).
|
|
||||||
tags: Vec<(String, String)>,
|
|
||||||
/// SSE-C passthrough transport headers stored with the version (RustFS
|
|
||||||
/// target behavior); empty when the drop mode discarded them.
|
|
||||||
replication_sse_headers: Vec<(String, String)>,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone)]
|
#[derive(Clone)]
|
||||||
@@ -267,7 +176,6 @@ struct MultipartState {
|
|||||||
version_id: String,
|
version_id: String,
|
||||||
content_type: Option<String>,
|
content_type: Option<String>,
|
||||||
metadata: Option<HashMap<String, String>>,
|
metadata: Option<HashMap<String, String>>,
|
||||||
replication_sse_headers: Vec<(String, String)>,
|
|
||||||
parts: BTreeMap<i32, MultipartPart>,
|
parts: BTreeMap<i32, MultipartPart>,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -488,15 +396,6 @@ impl FakeS3Target {
|
|||||||
|
|
||||||
/// Mint own version ids for the multipart path only — models a target
|
/// Mint own version ids for the multipart path only — models a target
|
||||||
/// that adopts PutObject version ids but not CreateMultipartUpload ones.
|
/// that adopts PutObject version ids but not CreateMultipartUpload ones.
|
||||||
/// MinIO-like mode: silently drop every `X-Rustfs-Replication-*` SSE-C
|
|
||||||
/// passthrough transport header instead of storing it. The default (off)
|
|
||||||
/// models a RustFS target, which preserves the headers and echoes SSE-C
|
|
||||||
/// evidence (`x-amz-server-side-encryption-customer-algorithm`) on
|
|
||||||
/// HEAD/GET of the replica.
|
|
||||||
pub fn drop_unlisted_replication_headers(&self, enabled: bool) {
|
|
||||||
lock(&self.backend.store).drop_unlisted_replication_headers = enabled;
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn assign_own_multipart_version_ids(&self, enabled: bool) {
|
pub fn assign_own_multipart_version_ids(&self, enabled: bool) {
|
||||||
lock(&self.backend.store).assign_own_multipart_version_ids = enabled;
|
lock(&self.backend.store).assign_own_multipart_version_ids = enabled;
|
||||||
}
|
}
|
||||||
@@ -637,17 +536,7 @@ impl S3Access for FaultAccess {
|
|||||||
.get(CONTENT_LENGTH)
|
.get(CONTENT_LENGTH)
|
||||||
.and_then(|value| value.to_str().ok())
|
.and_then(|value| value.to_str().ok())
|
||||||
.and_then(|value| value.parse().ok());
|
.and_then(|value| value.parse().ok());
|
||||||
let replication_timestamps = ReplicationTimestampHeaders::from_headers(context.headers());
|
let fault = record_request(&self.control, operation, context.method().clone(), parsed, content_length);
|
||||||
let proxy_headers = ProxyHeaderSnapshot::from_headers(context.headers());
|
|
||||||
let fault = record_request(
|
|
||||||
&self.control,
|
|
||||||
operation,
|
|
||||||
context.method().clone(),
|
|
||||||
parsed,
|
|
||||||
content_length,
|
|
||||||
replication_timestamps,
|
|
||||||
proxy_headers,
|
|
||||||
);
|
|
||||||
if let Some(RequestFault {
|
if let Some(RequestFault {
|
||||||
action: FaultAction::Status(status),
|
action: FaultAction::Status(status),
|
||||||
..
|
..
|
||||||
@@ -686,9 +575,6 @@ fn operation_from_s3_name(name: &str) -> Operation {
|
|||||||
"GetObject" => Operation::GetObject,
|
"GetObject" => Operation::GetObject,
|
||||||
"HeadObject" => Operation::HeadObject,
|
"HeadObject" => Operation::HeadObject,
|
||||||
"DeleteObject" => Operation::DeleteObject,
|
"DeleteObject" => Operation::DeleteObject,
|
||||||
"GetObjectTagging" => Operation::GetObjectTagging,
|
|
||||||
"PutObjectTagging" => Operation::PutObjectTagging,
|
|
||||||
"DeleteObjectTagging" => Operation::DeleteObjectTagging,
|
|
||||||
"CreateMultipartUpload" => Operation::CreateMultipartUpload,
|
"CreateMultipartUpload" => Operation::CreateMultipartUpload,
|
||||||
"UploadPart" => Operation::UploadPart,
|
"UploadPart" => Operation::UploadPart,
|
||||||
"CompleteMultipartUpload" => Operation::CompleteMultipartUpload,
|
"CompleteMultipartUpload" => Operation::CompleteMultipartUpload,
|
||||||
@@ -703,8 +589,6 @@ fn record_request(
|
|||||||
method: Method,
|
method: Method,
|
||||||
parsed: ParsedRequest,
|
parsed: ParsedRequest,
|
||||||
content_length: Option<u64>,
|
content_length: Option<u64>,
|
||||||
replication_timestamps: ReplicationTimestampHeaders,
|
|
||||||
proxy_headers: ProxyHeaderSnapshot,
|
|
||||||
) -> Option<RequestFault> {
|
) -> Option<RequestFault> {
|
||||||
let mut state = lock(control);
|
let mut state = lock(control);
|
||||||
let action = parsed
|
let action = parsed
|
||||||
@@ -729,8 +613,6 @@ fn record_request(
|
|||||||
part_number: parsed.part_number,
|
part_number: parsed.part_number,
|
||||||
content_length,
|
content_length,
|
||||||
consumed_bytes: None,
|
consumed_bytes: None,
|
||||||
replication_timestamps,
|
|
||||||
proxy_headers,
|
|
||||||
fault: action.clone(),
|
fault: action.clone(),
|
||||||
});
|
});
|
||||||
action.map(|action| RequestFault { sequence, action })
|
action.map(|action| RequestFault { sequence, action })
|
||||||
@@ -797,15 +679,6 @@ fn parse_request(method: &Method, uri: &Uri) -> ParsedRequest {
|
|||||||
(&Method::POST, true) if query.contains_key("uploads") => Operation::CreateMultipartUpload,
|
(&Method::POST, true) if query.contains_key("uploads") => Operation::CreateMultipartUpload,
|
||||||
(&Method::POST, true) if upload_id.is_some() => Operation::CompleteMultipartUpload,
|
(&Method::POST, true) if upload_id.is_some() => Operation::CompleteMultipartUpload,
|
||||||
(&Method::DELETE, true) if upload_id.is_some() => Operation::AbortMultipartUpload,
|
(&Method::DELETE, true) if upload_id.is_some() => Operation::AbortMultipartUpload,
|
||||||
(&Method::GET, true) if query.contains_key("tagging") && only_query_keys(&["tagging", "versionId"]) => {
|
|
||||||
Operation::GetObjectTagging
|
|
||||||
}
|
|
||||||
(&Method::PUT, true) if query.contains_key("tagging") && only_query_keys(&["tagging", "versionId"]) => {
|
|
||||||
Operation::PutObjectTagging
|
|
||||||
}
|
|
||||||
(&Method::DELETE, true) if query.contains_key("tagging") && only_query_keys(&["tagging", "versionId"]) => {
|
|
||||||
Operation::DeleteObjectTagging
|
|
||||||
}
|
|
||||||
// A replication PUT addresses the source version via `?versionId=`.
|
// A replication PUT addresses the source version via `?versionId=`.
|
||||||
(&Method::PUT, true) if only_query_keys(&["versionId"]) => Operation::PutObject,
|
(&Method::PUT, true) if only_query_keys(&["versionId"]) => Operation::PutObject,
|
||||||
(&Method::GET, true) if only_query_keys(&["versionId"]) => Operation::GetObject,
|
(&Method::GET, true) if only_query_keys(&["versionId"]) => Operation::GetObject,
|
||||||
@@ -873,29 +746,6 @@ fn new_version_id(headers: &HeaderMap, assign_own: bool) -> S3Result<String> {
|
|||||||
Ok(version_id.to_string())
|
Ok(version_id.to_string())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Capture the SSE-C passthrough transport headers a replication PUT carried.
|
|
||||||
/// Returns an empty set in the MinIO-like drop mode.
|
|
||||||
fn captured_replication_sse_headers(headers: &HeaderMap, drop_unlisted: bool) -> Vec<(String, String)> {
|
|
||||||
if drop_unlisted {
|
|
||||||
return Vec::new();
|
|
||||||
}
|
|
||||||
headers
|
|
||||||
.iter()
|
|
||||||
.filter(|(name, _)| name.as_str().starts_with(REPLICATION_SSE_TRANSPORT_PREFIX))
|
|
||||||
.filter_map(|(name, value)| Some((name.as_str().to_string(), value.to_str().ok()?.to_string())))
|
|
||||||
.collect()
|
|
||||||
}
|
|
||||||
|
|
||||||
/// SSE-C evidence a RustFS-like target echoes for a stored passthrough
|
|
||||||
/// replica: the customer algorithm restored from the transport headers.
|
|
||||||
fn stored_sse_customer_algorithm(version: &ObjectVersion) -> Option<String> {
|
|
||||||
version
|
|
||||||
.replication_sse_headers
|
|
||||||
.iter()
|
|
||||||
.find(|(name, _)| name == REPLICATION_SSEC_ALGORITHM_TRANSPORT_HEADER)
|
|
||||||
.map(|(_, value)| value.clone())
|
|
||||||
}
|
|
||||||
|
|
||||||
fn source_etag(headers: &HeaderMap) -> S3Result<Option<String>> {
|
fn source_etag(headers: &HeaderMap) -> S3Result<Option<String>> {
|
||||||
header_value(headers, &SOURCE_ETAG_HEADERS)
|
header_value(headers, &SOURCE_ETAG_HEADERS)
|
||||||
.map(|value| validate_retained_identifier(value, "source ETag").map(|value| normalize_etag(&value)))
|
.map(|value| validate_retained_identifier(value, "source ETag").map(|value| normalize_etag(&value)))
|
||||||
@@ -1243,33 +1093,6 @@ fn find_version(state: &StoreState, bucket: &str, key: &str, version_id: Option<
|
|||||||
Ok(version.clone())
|
Ok(version.clone())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Replace (or clear, with an empty vec) the tag set of the addressed
|
|
||||||
/// version, returning its version id. Mirrors `find_version` addressing:
|
|
||||||
/// explicit version id or the latest version, delete markers rejected.
|
|
||||||
fn set_version_tags(
|
|
||||||
state: &mut StoreState,
|
|
||||||
bucket: &str,
|
|
||||||
key: &str,
|
|
||||||
version_id: Option<&str>,
|
|
||||||
tags: Vec<(String, String)>,
|
|
||||||
) -> S3Result<String> {
|
|
||||||
// Resolve first (immutable) so the error paths match find_version.
|
|
||||||
let resolved = find_version(state, bucket, key, version_id)?.version_id;
|
|
||||||
let versions = state
|
|
||||||
.buckets
|
|
||||||
.get_mut(bucket)
|
|
||||||
.expect("bucket existence checked by find_version")
|
|
||||||
.objects
|
|
||||||
.get_mut(key)
|
|
||||||
.expect("key existence checked by find_version");
|
|
||||||
let version = versions
|
|
||||||
.iter_mut()
|
|
||||||
.find(|version| version.version_id == resolved)
|
|
||||||
.expect("version existence checked by find_version");
|
|
||||||
version.tags = tags;
|
|
||||||
Ok(resolved)
|
|
||||||
}
|
|
||||||
|
|
||||||
#[async_trait]
|
#[async_trait]
|
||||||
impl S3 for FakeBackend {
|
impl S3 for FakeBackend {
|
||||||
async fn head_bucket(&self, req: S3Request<HeadBucketInput>) -> S3Result<S3Response<HeadBucketOutput>> {
|
async fn head_bucket(&self, req: S3Request<HeadBucketInput>) -> S3Result<S3Response<HeadBucketOutput>> {
|
||||||
@@ -1366,10 +1189,7 @@ impl S3 for FakeBackend {
|
|||||||
let input = req.input;
|
let input = req.input;
|
||||||
let body = collect_stream(input.body, input.content_length, fault.as_ref(), &self.control).await?;
|
let body = collect_stream(input.body, input.content_length, fault.as_ref(), &self.control).await?;
|
||||||
validate_stored_metadata(&input.content_type, &input.metadata)?;
|
validate_stored_metadata(&input.content_type, &input.metadata)?;
|
||||||
let (assign_own, drop_unlisted) = {
|
let assign_own = lock(&self.store).assign_own_version_ids;
|
||||||
let state = lock(&self.store);
|
|
||||||
(state.assign_own_version_ids, state.drop_unlisted_replication_headers)
|
|
||||||
};
|
|
||||||
let version_id = new_version_id(&headers, assign_own)?;
|
let version_id = new_version_id(&headers, assign_own)?;
|
||||||
let e_tag = match source_etag(&headers)? {
|
let e_tag = match source_etag(&headers)? {
|
||||||
Some(value) => value,
|
Some(value) => value,
|
||||||
@@ -1386,8 +1206,6 @@ impl S3 for FakeBackend {
|
|||||||
delete_marker: false,
|
delete_marker: false,
|
||||||
content_type: input.content_type,
|
content_type: input.content_type,
|
||||||
metadata: input.metadata,
|
metadata: input.metadata,
|
||||||
tags: Vec::new(),
|
|
||||||
replication_sse_headers: captured_replication_sse_headers(&headers, drop_unlisted),
|
|
||||||
};
|
};
|
||||||
upsert_version(&mut lock(&self.store), &input.bucket, input.key, version)?;
|
upsert_version(&mut lock(&self.store), &input.bucket, input.key, version)?;
|
||||||
Ok(apply_response_fault(
|
Ok(apply_response_fault(
|
||||||
@@ -1408,7 +1226,6 @@ impl S3 for FakeBackend {
|
|||||||
let state = lock(&self.store);
|
let state = lock(&self.store);
|
||||||
find_version(&state, &input.bucket, &input.key, input.version_id.as_deref())?
|
find_version(&state, &input.bucket, &input.key, input.version_id.as_deref())?
|
||||||
};
|
};
|
||||||
let sse_customer_algorithm = stored_sse_customer_algorithm(&version);
|
|
||||||
Ok(apply_response_fault(
|
Ok(apply_response_fault(
|
||||||
S3Response::new(GetObjectOutput {
|
S3Response::new(GetObjectOutput {
|
||||||
body: Some(StreamingBlob::new(Body::from(version.body.clone()))),
|
body: Some(StreamingBlob::new(Body::from(version.body.clone()))),
|
||||||
@@ -1418,7 +1235,6 @@ impl S3 for FakeBackend {
|
|||||||
e_tag: Some(ETag::Strong(version.e_tag)),
|
e_tag: Some(ETag::Strong(version.e_tag)),
|
||||||
last_modified: Some(version.last_modified.clone()),
|
last_modified: Some(version.last_modified.clone()),
|
||||||
version_id: Some(version.version_id),
|
version_id: Some(version.version_id),
|
||||||
sse_customer_algorithm,
|
|
||||||
..Default::default()
|
..Default::default()
|
||||||
}),
|
}),
|
||||||
fault.as_ref(),
|
fault.as_ref(),
|
||||||
@@ -1433,7 +1249,6 @@ impl S3 for FakeBackend {
|
|||||||
let state = lock(&self.store);
|
let state = lock(&self.store);
|
||||||
find_version(&state, &input.bucket, &input.key, input.version_id.as_deref())?
|
find_version(&state, &input.bucket, &input.key, input.version_id.as_deref())?
|
||||||
};
|
};
|
||||||
let sse_customer_algorithm = stored_sse_customer_algorithm(&version);
|
|
||||||
Ok(apply_response_fault(
|
Ok(apply_response_fault(
|
||||||
S3Response::new(HeadObjectOutput {
|
S3Response::new(HeadObjectOutput {
|
||||||
content_length: Some(version.body.len() as i64),
|
content_length: Some(version.body.len() as i64),
|
||||||
@@ -1442,79 +1257,12 @@ impl S3 for FakeBackend {
|
|||||||
e_tag: Some(ETag::Strong(version.e_tag)),
|
e_tag: Some(ETag::Strong(version.e_tag)),
|
||||||
last_modified: Some(version.last_modified.clone()),
|
last_modified: Some(version.last_modified.clone()),
|
||||||
version_id: Some(version.version_id),
|
version_id: Some(version.version_id),
|
||||||
sse_customer_algorithm,
|
|
||||||
..Default::default()
|
..Default::default()
|
||||||
}),
|
}),
|
||||||
fault.as_ref(),
|
fault.as_ref(),
|
||||||
))
|
))
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn get_object_tagging(&self, req: S3Request<GetObjectTaggingInput>) -> S3Result<S3Response<GetObjectTaggingOutput>> {
|
|
||||||
let fault = request_fault(&req);
|
|
||||||
apply_non_body_fault(fault.as_ref(), &self.control).await?;
|
|
||||||
let input = req.input;
|
|
||||||
let version = {
|
|
||||||
let state = lock(&self.store);
|
|
||||||
find_version(&state, &input.bucket, &input.key, input.version_id.as_deref())?
|
|
||||||
};
|
|
||||||
let tag_set: TagSet = version
|
|
||||||
.tags
|
|
||||||
.into_iter()
|
|
||||||
.map(|(key, value)| Tag {
|
|
||||||
key: Some(key),
|
|
||||||
value: Some(value),
|
|
||||||
})
|
|
||||||
.collect();
|
|
||||||
Ok(apply_response_fault(
|
|
||||||
S3Response::new(GetObjectTaggingOutput {
|
|
||||||
tag_set,
|
|
||||||
version_id: Some(ObjectVersionId::from(version.version_id)),
|
|
||||||
}),
|
|
||||||
fault.as_ref(),
|
|
||||||
))
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn put_object_tagging(&self, req: S3Request<PutObjectTaggingInput>) -> S3Result<S3Response<PutObjectTaggingOutput>> {
|
|
||||||
let fault = request_fault(&req);
|
|
||||||
apply_non_body_fault(fault.as_ref(), &self.control).await?;
|
|
||||||
let input = req.input;
|
|
||||||
let tags = input
|
|
||||||
.tagging
|
|
||||||
.tag_set
|
|
||||||
.into_iter()
|
|
||||||
.map(|tag| (tag.key.unwrap_or_default(), tag.value.unwrap_or_default()))
|
|
||||||
.collect();
|
|
||||||
let version_id = {
|
|
||||||
let mut state = lock(&self.store);
|
|
||||||
set_version_tags(&mut state, &input.bucket, &input.key, input.version_id.as_deref(), tags)?
|
|
||||||
};
|
|
||||||
Ok(apply_response_fault(
|
|
||||||
S3Response::new(PutObjectTaggingOutput {
|
|
||||||
version_id: Some(ObjectVersionId::from(version_id)),
|
|
||||||
}),
|
|
||||||
fault.as_ref(),
|
|
||||||
))
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn delete_object_tagging(
|
|
||||||
&self,
|
|
||||||
req: S3Request<DeleteObjectTaggingInput>,
|
|
||||||
) -> S3Result<S3Response<DeleteObjectTaggingOutput>> {
|
|
||||||
let fault = request_fault(&req);
|
|
||||||
apply_non_body_fault(fault.as_ref(), &self.control).await?;
|
|
||||||
let input = req.input;
|
|
||||||
let version_id = {
|
|
||||||
let mut state = lock(&self.store);
|
|
||||||
set_version_tags(&mut state, &input.bucket, &input.key, input.version_id.as_deref(), Vec::new())?
|
|
||||||
};
|
|
||||||
Ok(apply_response_fault(
|
|
||||||
S3Response::new(DeleteObjectTaggingOutput {
|
|
||||||
version_id: Some(ObjectVersionId::from(version_id)),
|
|
||||||
}),
|
|
||||||
fault.as_ref(),
|
|
||||||
))
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn delete_object(&self, req: S3Request<DeleteObjectInput>) -> S3Result<S3Response<DeleteObjectOutput>> {
|
async fn delete_object(&self, req: S3Request<DeleteObjectInput>) -> S3Result<S3Response<DeleteObjectOutput>> {
|
||||||
let fault = request_fault(&req);
|
let fault = request_fault(&req);
|
||||||
apply_non_body_fault(fault.as_ref(), &self.control).await?;
|
apply_non_body_fault(fault.as_ref(), &self.control).await?;
|
||||||
@@ -1591,8 +1339,6 @@ impl S3 for FakeBackend {
|
|||||||
delete_marker: true,
|
delete_marker: true,
|
||||||
content_type: None,
|
content_type: None,
|
||||||
metadata: None,
|
metadata: None,
|
||||||
tags: Vec::new(),
|
|
||||||
replication_sse_headers: Vec::new(),
|
|
||||||
},
|
},
|
||||||
)?;
|
)?;
|
||||||
Ok(apply_response_fault(
|
Ok(apply_response_fault(
|
||||||
@@ -1620,10 +1366,9 @@ impl S3 for FakeBackend {
|
|||||||
ensure_upload_budget(&state)?;
|
ensure_upload_budget(&state)?;
|
||||||
validate_stored_metadata(&input.content_type, &input.metadata)?;
|
validate_stored_metadata(&input.content_type, &input.metadata)?;
|
||||||
let upload_id = Uuid::new_v4().to_string();
|
let upload_id = Uuid::new_v4().to_string();
|
||||||
// Read the flags before the mutable borrow of `state.uploads` below
|
// Read the flag before the mutable borrow of `state.uploads` below
|
||||||
// (and never re-lock the store: the mutex is not reentrant).
|
// (and never re-lock the store: the mutex is not reentrant).
|
||||||
let mint_own = state.assign_own_version_ids || state.assign_own_multipart_version_ids;
|
let mint_own = state.assign_own_version_ids || state.assign_own_multipart_version_ids;
|
||||||
let drop_unlisted = state.drop_unlisted_replication_headers;
|
|
||||||
let version_id = new_version_id(&headers, mint_own)?;
|
let version_id = new_version_id(&headers, mint_own)?;
|
||||||
state.uploads.insert(
|
state.uploads.insert(
|
||||||
upload_id.clone(),
|
upload_id.clone(),
|
||||||
@@ -1633,7 +1378,6 @@ impl S3 for FakeBackend {
|
|||||||
version_id,
|
version_id,
|
||||||
content_type: input.content_type,
|
content_type: input.content_type,
|
||||||
metadata: input.metadata,
|
metadata: input.metadata,
|
||||||
replication_sse_headers: captured_replication_sse_headers(&headers, drop_unlisted),
|
|
||||||
parts: BTreeMap::new(),
|
parts: BTreeMap::new(),
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
@@ -1771,7 +1515,6 @@ impl S3 for FakeBackend {
|
|||||||
version_id: upload.version_id.clone(),
|
version_id: upload.version_id.clone(),
|
||||||
content_type: upload.content_type.clone(),
|
content_type: upload.content_type.clone(),
|
||||||
metadata: upload.metadata.clone(),
|
metadata: upload.metadata.clone(),
|
||||||
replication_sse_headers: upload.replication_sse_headers.clone(),
|
|
||||||
parts: BTreeMap::new(),
|
parts: BTreeMap::new(),
|
||||||
},
|
},
|
||||||
selected,
|
selected,
|
||||||
@@ -1798,8 +1541,6 @@ impl S3 for FakeBackend {
|
|||||||
delete_marker: false,
|
delete_marker: false,
|
||||||
content_type: upload.content_type,
|
content_type: upload.content_type,
|
||||||
metadata: upload.metadata,
|
metadata: upload.metadata,
|
||||||
tags: Vec::new(),
|
|
||||||
replication_sse_headers: upload.replication_sse_headers,
|
|
||||||
};
|
};
|
||||||
let mut state = lock(&self.store);
|
let mut state = lock(&self.store);
|
||||||
let current = state
|
let current = state
|
||||||
@@ -1958,111 +1699,6 @@ mod tests {
|
|||||||
.await?)
|
.await?)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn journals_replication_timestamp_headers() -> Result<(), BoxError> {
|
|
||||||
let target = FakeS3Target::start().await?;
|
|
||||||
target.create_bucket("target-bucket");
|
|
||||||
let client = client(&target);
|
|
||||||
|
|
||||||
client
|
|
||||||
.put_object()
|
|
||||||
.bucket("target-bucket")
|
|
||||||
.key("plain")
|
|
||||||
.body(ByteStream::from_static(b"plain"))
|
|
||||||
.send()
|
|
||||||
.await?;
|
|
||||||
client
|
|
||||||
.put_object()
|
|
||||||
.bucket("target-bucket")
|
|
||||||
.key("stamped")
|
|
||||||
.body(ByteStream::from_static(b"stamped"))
|
|
||||||
.customize()
|
|
||||||
.map_request(move |mut request| {
|
|
||||||
let headers = request.headers_mut();
|
|
||||||
headers.insert("x-rustfs-source-replication-tagging-timestamp", "2026-01-02T03:04:05Z");
|
|
||||||
headers.insert("x-minio-source-replication-retention-timestamp", "2026-01-02T03:04:06Z");
|
|
||||||
headers.insert("x-rustfs-source-replication-legalhold-timestamp", "2026-01-02T03:04:07Z");
|
|
||||||
Ok::<_, std::convert::Infallible>(request)
|
|
||||||
})
|
|
||||||
.send()
|
|
||||||
.await?;
|
|
||||||
|
|
||||||
let requests = target.requests();
|
|
||||||
let plain = requests
|
|
||||||
.iter()
|
|
||||||
.find(|record| record.operation == Operation::PutObject && record.key.as_deref() == Some("plain"))
|
|
||||||
.expect("plain PUT must be journaled");
|
|
||||||
assert_eq!(plain.replication_timestamps, ReplicationTimestampHeaders::default());
|
|
||||||
|
|
||||||
let stamped = requests
|
|
||||||
.iter()
|
|
||||||
.find(|record| record.operation == Operation::PutObject && record.key.as_deref() == Some("stamped"))
|
|
||||||
.expect("stamped PUT must be journaled");
|
|
||||||
assert_eq!(stamped.replication_timestamps.tagging.as_deref(), Some("2026-01-02T03:04:05Z"));
|
|
||||||
assert_eq!(stamped.replication_timestamps.retention.as_deref(), Some("2026-01-02T03:04:06Z"));
|
|
||||||
assert_eq!(stamped.replication_timestamps.legalhold.as_deref(), Some("2026-01-02T03:04:07Z"));
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Default mode is RustFS-like: SSE-C passthrough transport headers are
|
|
||||||
/// stored and the customer algorithm is echoed on HEAD/GET. Drop mode is
|
|
||||||
/// MinIO-like: the headers are silently discarded, so no evidence comes
|
|
||||||
/// back — the exact difference the N2 fail-closed audit keys on. Both
|
|
||||||
/// modes journal that the sender shipped the transport headers.
|
|
||||||
#[tokio::test]
|
|
||||||
async fn ssec_passthrough_headers_echo_and_drop_modes() -> Result<(), BoxError> {
|
|
||||||
let target = FakeS3Target::start().await?;
|
|
||||||
target.create_bucket("target-bucket");
|
|
||||||
let client = client(&target);
|
|
||||||
|
|
||||||
let put_with_transport_headers = |key: &'static str| {
|
|
||||||
client
|
|
||||||
.put_object()
|
|
||||||
.bucket("target-bucket")
|
|
||||||
.key(key)
|
|
||||||
.body(ByteStream::from_static(b"ciphertext"))
|
|
||||||
.customize()
|
|
||||||
.map_request(move |mut request| {
|
|
||||||
let headers = request.headers_mut();
|
|
||||||
headers.insert("x-rustfs-replication-ssec-algorithm", "AES256");
|
|
||||||
headers.insert("x-rustfs-replication-ssec-key-md5", "AAAAAAAAAAAAAAAAAAAAAA==");
|
|
||||||
Ok::<_, std::convert::Infallible>(request)
|
|
||||||
})
|
|
||||||
.send()
|
|
||||||
};
|
|
||||||
|
|
||||||
put_with_transport_headers("kept").await?;
|
|
||||||
let head = client.head_object().bucket("target-bucket").key("kept").send().await?;
|
|
||||||
assert_eq!(head.sse_customer_algorithm(), Some("AES256"));
|
|
||||||
let get = client.get_object().bucket("target-bucket").key("kept").send().await?;
|
|
||||||
assert_eq!(get.sse_customer_algorithm(), Some("AES256"));
|
|
||||||
|
|
||||||
target.drop_unlisted_replication_headers(true);
|
|
||||||
put_with_transport_headers("dropped").await?;
|
|
||||||
let head = client.head_object().bucket("target-bucket").key("dropped").send().await?;
|
|
||||||
assert_eq!(head.sse_customer_algorithm(), None, "drop mode must discard SSE-C evidence");
|
|
||||||
|
|
||||||
let requests = target.requests();
|
|
||||||
for key in ["kept", "dropped"] {
|
|
||||||
let record = requests
|
|
||||||
.iter()
|
|
||||||
.find(|record| record.operation == Operation::PutObject && record.key.as_deref() == Some(key))
|
|
||||||
.expect("PUT must be journaled");
|
|
||||||
assert!(
|
|
||||||
record.proxy_headers.ssec_transport_present,
|
|
||||||
"the journal must prove the sender shipped the transport headers for {key}"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
let plain_head = requests
|
|
||||||
.iter()
|
|
||||||
.find(|record| record.operation == Operation::HeadObject)
|
|
||||||
.expect("HEAD must be journaled");
|
|
||||||
assert!(!plain_head.proxy_headers.ssec_transport_present);
|
|
||||||
|
|
||||||
target.shutdown().await;
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
macro_rules! assert_sdk_error {
|
macro_rules! assert_sdk_error {
|
||||||
($error:expr, $status:expr, $code:expr) => {{
|
($error:expr, $status:expr, $code:expr) => {{
|
||||||
let error = &$error;
|
let error = &$error;
|
||||||
@@ -3328,7 +2964,6 @@ mod tests {
|
|||||||
version_id: index.to_string(),
|
version_id: index.to_string(),
|
||||||
content_type: None,
|
content_type: None,
|
||||||
metadata: None,
|
metadata: None,
|
||||||
replication_sse_headers: Vec::new(),
|
|
||||||
parts: BTreeMap::new(),
|
parts: BTreeMap::new(),
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
@@ -3350,8 +2985,6 @@ mod tests {
|
|||||||
part_number: None,
|
part_number: None,
|
||||||
},
|
},
|
||||||
Some(0),
|
Some(0),
|
||||||
ReplicationTimestampHeaders::default(),
|
|
||||||
ProxyHeaderSnapshot::default(),
|
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
let records = lock(&control).requests.clone();
|
let records = lock(&control).requests.clone();
|
||||||
@@ -3373,8 +3006,6 @@ mod tests {
|
|||||||
part_number: None,
|
part_number: None,
|
||||||
},
|
},
|
||||||
None,
|
None,
|
||||||
ReplicationTimestampHeaders::default(),
|
|
||||||
ProxyHeaderSnapshot::default(),
|
|
||||||
);
|
);
|
||||||
{
|
{
|
||||||
let bounded_records = lock(&bounded_control);
|
let bounded_records = lock(&bounded_control);
|
||||||
|
|||||||
@@ -66,6 +66,7 @@ mod tests {
|
|||||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||||
use aws_sdk_s3::primitives::ByteStream;
|
use aws_sdk_s3::primitives::ByteStream;
|
||||||
use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
|
use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
|
||||||
|
use serial_test::serial;
|
||||||
use sha2::{Digest, Sha256};
|
use sha2::{Digest, Sha256};
|
||||||
use std::collections::BTreeMap;
|
use std::collections::BTreeMap;
|
||||||
use std::error::Error;
|
use std::error::Error;
|
||||||
@@ -188,6 +189,8 @@ mod tests {
|
|||||||
("RUSTFS_GET_CODEC_STREAMING_ROLLOUT_PCT", "100"),
|
("RUSTFS_GET_CODEC_STREAMING_ROLLOUT_PCT", "100"),
|
||||||
("RUSTFS_GET_CODEC_STREAMING_BODY_COMPAT_CONFIRMED", "true"),
|
("RUSTFS_GET_CODEC_STREAMING_BODY_COMPAT_CONFIRMED", "true"),
|
||||||
("RUSTFS_GET_CODEC_STREAMING_HEADER_COMPAT_CONFIRMED", "true"),
|
("RUSTFS_GET_CODEC_STREAMING_HEADER_COMPAT_CONFIRMED", "true"),
|
||||||
|
// Lower the min-size floor so every non-inline object below is eligible.
|
||||||
|
("RUSTFS_GET_CODEC_STREAMING_MIN_SIZE", "4096"),
|
||||||
// Route multipart objects through per-part codec streaming too.
|
// Route multipart objects through per-part codec streaming too.
|
||||||
("RUSTFS_GET_CODEC_STREAMING_MULTIPART_ENABLE", "true"),
|
("RUSTFS_GET_CODEC_STREAMING_MULTIPART_ENABLE", "true"),
|
||||||
// Lock optimization is on by default, but pin it so the gate's
|
// Lock optimization is on by default, but pin it so the gate's
|
||||||
@@ -276,6 +279,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn codec_streaming_matches_legacy_duplex_body_and_headers() -> TestResult {
|
async fn codec_streaming_matches_legacy_duplex_body_and_headers() -> TestResult {
|
||||||
init_logging();
|
init_logging();
|
||||||
|
|
||||||
@@ -311,13 +315,6 @@ mod tests {
|
|||||||
},
|
},
|
||||||
payload(64 * 1024, 2),
|
payload(64 * 1024, 2),
|
||||||
),
|
),
|
||||||
(
|
|
||||||
Shape {
|
|
||||||
key: "small-non-inline-256kib-plus",
|
|
||||||
expect_large: true,
|
|
||||||
},
|
|
||||||
payload(256 * 1024 + 1, 6),
|
|
||||||
),
|
|
||||||
(
|
(
|
||||||
Shape {
|
Shape {
|
||||||
key: "mid-1_5mib",
|
key: "mid-1_5mib",
|
||||||
|
|||||||
@@ -1,250 +0,0 @@
|
|||||||
// Copyright 2024 RustFS Team
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
//! E2E proof that a mid-stream GET failure is *reportable* — rustfs#4784.
|
|
||||||
//!
|
|
||||||
//! The functional invariant (a beyond-quorum read must fail rather than return
|
|
||||||
//! a clean short body) is already covered by
|
|
||||||
//! `degraded_read_eof_regression_test`. This suite covers the half that issue
|
|
||||||
//! #4784 got stuck on for a month: whether an operator can tell, from the
|
|
||||||
//! source server's log alone, that a GET failed mid-body and **which object**
|
|
||||||
//! it failed on.
|
|
||||||
//!
|
|
||||||
//! The reporter saw only downstream symptoms — `rclone` reporting
|
|
||||||
//! `unexpected EOF` on its PUT, and the receiving RustFS logging
|
|
||||||
//! `Io error: error reading a body from connection` with a 500. In a cross-remote
|
|
||||||
//! `rclone sync`, the source GET body *is* the destination PUT body, so a source
|
|
||||||
//! read that ends short of its committed `Content-Length` surfaces as a PUT
|
|
||||||
//! failure on the far side. Built-in replication and site replication have the
|
|
||||||
//! same shape (read locally, PUT remotely), which is why every transport in that
|
|
||||||
//! report failed the same way.
|
|
||||||
//!
|
|
||||||
//! The source side, meanwhile, said nothing:
|
|
||||||
//! * `GetObjectReaderStream`'s short-read and read-error arms only incremented
|
|
||||||
//! a metric; their log lines sat behind the `tracing-chunk-debug` cargo
|
|
||||||
//! feature, which is not in the default feature set and therefore is not
|
|
||||||
//! compiled into any released binary.
|
|
||||||
//! * `GetObjectStreamingReader` did log mid-stream failures, but only under a
|
|
||||||
//! `request_id` — with no bucket or object name, a failure could not be
|
|
||||||
//! traced back to the object that caused it.
|
|
||||||
//! * Those lines were `warn!`, while `DEFAULT_LOG_LEVEL` is `error`, so a
|
|
||||||
//! default deployment filtered them out anyway.
|
|
||||||
//!
|
|
||||||
//! This test reproduces the source-side fault against a real server over the S3
|
|
||||||
//! API and asserts the operator-visible evidence, at the **default** log level.
|
|
||||||
|
|
||||||
#[cfg(test)]
|
|
||||||
mod tests {
|
|
||||||
use crate::chaos::DiskFaultHarness;
|
|
||||||
use crate::common::init_logging;
|
|
||||||
use aws_sdk_s3::Client;
|
|
||||||
use aws_sdk_s3::primitives::ByteStream;
|
|
||||||
use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
|
|
||||||
use serial_test::serial;
|
|
||||||
use std::error::Error;
|
|
||||||
use tokio::time::{Duration, timeout};
|
|
||||||
use tracing::info;
|
|
||||||
|
|
||||||
type TestResult = Result<(), Box<dyn Error + Send + Sync>>;
|
|
||||||
|
|
||||||
const MIB: usize = 1024 * 1024;
|
|
||||||
const OP_TIMEOUT: Duration = Duration::from_secs(90);
|
|
||||||
|
|
||||||
/// The structured event name every GET body failure is tagged with.
|
|
||||||
const STREAM_BODY_EVENT: &str = "get_object_stream_body";
|
|
||||||
|
|
||||||
fn payload(len: usize, seed: u8) -> Vec<u8> {
|
|
||||||
(0..len)
|
|
||||||
.map(|i| (i as u64).wrapping_mul(2654435761).wrapping_add(seed as u64) as u8)
|
|
||||||
.collect()
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Upload a multipart object so the data lands in real `part.*` shard files
|
|
||||||
/// rather than being inlined into `xl.meta` (inlined objects cannot be
|
|
||||||
/// corrupted shard-wise, and never exercise the streaming read path).
|
|
||||||
async fn put_multipart(
|
|
||||||
client: &Client,
|
|
||||||
bucket: &str,
|
|
||||||
key: &str,
|
|
||||||
parts: Vec<Vec<u8>>,
|
|
||||||
) -> Result<usize, Box<dyn Error + Send + Sync>> {
|
|
||||||
let total_len = parts.iter().map(Vec::len).sum();
|
|
||||||
|
|
||||||
let create = client.create_multipart_upload().bucket(bucket).key(key).send().await?;
|
|
||||||
let upload_id = create.upload_id().ok_or("missing upload id")?.to_string();
|
|
||||||
|
|
||||||
let mut completed = Vec::with_capacity(parts.len());
|
|
||||||
for (index, part_body) in parts.into_iter().enumerate() {
|
|
||||||
let part_number = (index + 1) as i32;
|
|
||||||
let uploaded = timeout(
|
|
||||||
OP_TIMEOUT,
|
|
||||||
client
|
|
||||||
.upload_part()
|
|
||||||
.bucket(bucket)
|
|
||||||
.key(key)
|
|
||||||
.upload_id(&upload_id)
|
|
||||||
.part_number(part_number)
|
|
||||||
.body(ByteStream::from(part_body))
|
|
||||||
.send(),
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
.map_err(|_| format!("upload_part {part_number} timed out"))??;
|
|
||||||
completed.push(
|
|
||||||
CompletedPart::builder()
|
|
||||||
.part_number(part_number)
|
|
||||||
.e_tag(uploaded.e_tag().ok_or("missing part etag")?)
|
|
||||||
.build(),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
timeout(
|
|
||||||
OP_TIMEOUT,
|
|
||||||
client
|
|
||||||
.complete_multipart_upload()
|
|
||||||
.bucket(bucket)
|
|
||||||
.key(key)
|
|
||||||
.upload_id(&upload_id)
|
|
||||||
.multipart_upload(CompletedMultipartUpload::builder().set_parts(Some(completed)).build())
|
|
||||||
.send(),
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
.map_err(|_| "complete_multipart_upload timed out")??;
|
|
||||||
|
|
||||||
Ok(total_len)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// rustfs#4784: reproduce the source-side fault the reporter kept hitting —
|
|
||||||
/// a GET that commits `200` + a full `Content-Length` and then cannot finish
|
|
||||||
/// the body — and assert the server log names the object, at the log level a
|
|
||||||
/// default deployment actually runs with.
|
|
||||||
#[tokio::test]
|
|
||||||
#[serial]
|
|
||||||
async fn midstream_get_failure_is_logged_with_the_object_at_default_log_level() -> TestResult {
|
|
||||||
init_logging();
|
|
||||||
info!("rustfs#4784: a mid-stream GET failure must name its object in the source log");
|
|
||||||
|
|
||||||
let mut harness = DiskFaultHarness::new(4).await?;
|
|
||||||
|
|
||||||
// Capture the child's stdout so the test can read what an operator would.
|
|
||||||
let log_path = format!("{}/server.log", harness.env.temp_dir);
|
|
||||||
harness.env.capture_log_path = Some(log_path.clone());
|
|
||||||
|
|
||||||
// Reproduce a DEFAULT deployment's logging, not the e2e harness's
|
|
||||||
// permissive `rustfs=info`: `DEFAULT_LOG_LEVEL` is `error`. Before the
|
|
||||||
// #4784 fix these failures were `warn!`, so a default deployment
|
|
||||||
// filtered them out entirely — which is why the reporter's source logs
|
|
||||||
// were empty. extra_env is applied after the harness's own RUST_LOG, so
|
|
||||||
// this wins.
|
|
||||||
harness.set_env("RUST_LOG", "error");
|
|
||||||
harness.set_env("RUSTFS_OBS_LOGGER_LEVEL", "error");
|
|
||||||
|
|
||||||
harness.start_server().await?;
|
|
||||||
let client = harness.env.create_s3_client();
|
|
||||||
|
|
||||||
let bucket = "issue4784-source-read";
|
|
||||||
client.create_bucket().bucket(bucket).send().await?;
|
|
||||||
|
|
||||||
// Named after the reporter's restic index objects, which is where they
|
|
||||||
// saw the failures.
|
|
||||||
let key = "index/3b18542ab3af4c3d03f804c7a24173e7836ef7fa447b5d1e9d634f975cc51611";
|
|
||||||
let expected_len = put_multipart(
|
|
||||||
&client,
|
|
||||||
bucket,
|
|
||||||
key,
|
|
||||||
vec![payload(5 * MIB, 71), payload(5 * MIB, 72), payload(5 * MIB, 73)],
|
|
||||||
)
|
|
||||||
.await?;
|
|
||||||
|
|
||||||
// Baseline: the object reads back completely before any corruption.
|
|
||||||
let baseline = timeout(OP_TIMEOUT, client.get_object().bucket(bucket).key(key).send())
|
|
||||||
.await
|
|
||||||
.map_err(|_| "baseline GET timed out")??
|
|
||||||
.body
|
|
||||||
.collect()
|
|
||||||
.await?;
|
|
||||||
assert_eq!(baseline.into_bytes().len(), expected_len, "baseline GET must return the whole object");
|
|
||||||
|
|
||||||
// Corrupt three of four shards in a 2+2 set: below the 2-shard read
|
|
||||||
// quorum. The corruption sits mid-file, so block 0 still reads clean —
|
|
||||||
// the server commits 200 + the full Content-Length and only then cannot
|
|
||||||
// reconstruct. That is the mid-stream window the reporter's downstream
|
|
||||||
// saw as `unexpected EOF`.
|
|
||||||
harness.corrupt_object_shard(0, bucket, key)?;
|
|
||||||
harness.corrupt_object_shard(1, bucket, key)?;
|
|
||||||
harness.corrupt_object_shard(2, bucket, key)?;
|
|
||||||
|
|
||||||
let response = timeout(OP_TIMEOUT, client.get_object().bucket(bucket).key(key).send())
|
|
||||||
.await
|
|
||||||
.map_err(|_| "degraded GET timed out")?;
|
|
||||||
|
|
||||||
// Either outcome is functionally correct (that invariant belongs to
|
|
||||||
// degraded_read_eof_regression_test); this suite only needs the read to
|
|
||||||
// have failed so there is something to report.
|
|
||||||
let delivered = match response {
|
|
||||||
Err(err) => {
|
|
||||||
info!("degraded GET failed before the body: {err}");
|
|
||||||
None
|
|
||||||
}
|
|
||||||
Ok(response) => match response.body.collect().await {
|
|
||||||
Ok(aggregated) => Some(aggregated.into_bytes().len()),
|
|
||||||
Err(err) => {
|
|
||||||
info!("degraded GET failed mid-body as expected: {err}");
|
|
||||||
None
|
|
||||||
}
|
|
||||||
},
|
|
||||||
};
|
|
||||||
assert_ne!(
|
|
||||||
delivered,
|
|
||||||
Some(expected_len),
|
|
||||||
"the beyond-quorum read unexpectedly succeeded; this suite needs a failed read to have something to report"
|
|
||||||
);
|
|
||||||
|
|
||||||
// Give the child a moment to flush its stdout.
|
|
||||||
tokio::time::sleep(Duration::from_millis(500)).await;
|
|
||||||
let logged = std::fs::read_to_string(&log_path)?;
|
|
||||||
|
|
||||||
let failure_lines: Vec<&str> = logged.lines().filter(|line| line.contains(STREAM_BODY_EVENT)).collect();
|
|
||||||
|
|
||||||
assert!(
|
|
||||||
!failure_lines.is_empty(),
|
|
||||||
"a mid-stream GET failure produced no `{STREAM_BODY_EVENT}` line at the default log level. \
|
|
||||||
This is the #4784 blind spot: the failure was only counted in a metric, or logged below \
|
|
||||||
`error` and filtered out. Captured log:\n{logged}"
|
|
||||||
);
|
|
||||||
|
|
||||||
// The identity is the whole point: a request_id alone cannot be resolved
|
|
||||||
// back to an object once the request is over.
|
|
||||||
assert!(
|
|
||||||
failure_lines.iter().any(|line| line.contains(key)),
|
|
||||||
"no `{STREAM_BODY_EVENT}` line named the failing object `{key}`, so the report is still \
|
|
||||||
unactionable. Lines seen:\n{}",
|
|
||||||
failure_lines.join("\n")
|
|
||||||
);
|
|
||||||
assert!(
|
|
||||||
failure_lines.iter().any(|line| line.contains(bucket)),
|
|
||||||
"no `{STREAM_BODY_EVENT}` line named the failing bucket `{bucket}`. Lines seen:\n{}",
|
|
||||||
failure_lines.join("\n")
|
|
||||||
);
|
|
||||||
|
|
||||||
info!(
|
|
||||||
"source-side evidence now present: {} stream-body failure line(s) naming the object",
|
|
||||||
failure_lines.len()
|
|
||||||
);
|
|
||||||
for line in &failure_lines {
|
|
||||||
info!("operator-visible evidence: {line}");
|
|
||||||
}
|
|
||||||
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -14,9 +14,10 @@
|
|||||||
|
|
||||||
//! E2E tests for group management (fixes #2028).
|
//! E2E tests for group management (fixes #2028).
|
||||||
|
|
||||||
use crate::common::{RustFSTestEnvironment, admin_request, awscurl_delete, awscurl_get, awscurl_put, init_logging};
|
use crate::common::{RustFSTestEnvironment, awscurl_delete, awscurl_get, awscurl_put, init_logging};
|
||||||
use aws_sdk_s3::config::{Credentials, Region};
|
use aws_sdk_s3::config::{Credentials, Region};
|
||||||
use aws_sdk_s3::{Client, Config};
|
use aws_sdk_s3::{Client, Config};
|
||||||
|
use serial_test::serial;
|
||||||
use tracing::info;
|
use tracing::info;
|
||||||
|
|
||||||
fn create_user_s3_client(env: &RustFSTestEnvironment, access_key: &str, secret_key: &str) -> Client {
|
fn create_user_s3_client(env: &RustFSTestEnvironment, access_key: &str, secret_key: &str) -> Client {
|
||||||
@@ -31,58 +32,9 @@ fn create_user_s3_client(env: &RustFSTestEnvironment, access_key: &str, secret_k
|
|||||||
Client::from_conf(config)
|
Client::from_conf(config)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test(flavor = "multi_thread")]
|
|
||||||
async fn update_group_members_rejects_invalid_new_group_names() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
|
||||||
init_logging();
|
|
||||||
|
|
||||||
let mut env = RustFSTestEnvironment::new().await?;
|
|
||||||
env.start_rustfs_server(vec![]).await?;
|
|
||||||
|
|
||||||
let invalid_groups = [
|
|
||||||
("test group", "group name contains whitespace"),
|
|
||||||
("test=group", "group name contains reserved characters =,"),
|
|
||||||
("test,group", "group name contains reserved characters =,"),
|
|
||||||
];
|
|
||||||
|
|
||||||
for (group, expected_message) in invalid_groups {
|
|
||||||
let body = serde_json::json!({
|
|
||||||
"group": group,
|
|
||||||
"members": [],
|
|
||||||
"isRemove": false,
|
|
||||||
"groupStatus": "enabled"
|
|
||||||
})
|
|
||||||
.to_string();
|
|
||||||
let (status, response_body) = admin_request(
|
|
||||||
&env.url,
|
|
||||||
http::Method::PUT,
|
|
||||||
"/rustfs/admin/v3/update-group-members",
|
|
||||||
Some(body),
|
|
||||||
&env.access_key,
|
|
||||||
&env.secret_key,
|
|
||||||
)
|
|
||||||
.await?;
|
|
||||||
|
|
||||||
assert_eq!(
|
|
||||||
status,
|
|
||||||
reqwest::StatusCode::BAD_REQUEST,
|
|
||||||
"invalid group {group:?} must return HTTP 400, body: {response_body}"
|
|
||||||
);
|
|
||||||
assert!(
|
|
||||||
response_body.contains("<Code>InvalidArgument</Code>"),
|
|
||||||
"invalid group {group:?} must return InvalidArgument, body: {response_body}"
|
|
||||||
);
|
|
||||||
assert!(
|
|
||||||
response_body.contains(&format!("<Message>{expected_message}</Message>")),
|
|
||||||
"invalid group {group:?} returned an unexpected message: {response_body}"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
env.stop_server();
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Test that deleting a group with members fails, and deleting an empty group succeeds.
|
/// Test that deleting a group with members fails, and deleting an empty group succeeds.
|
||||||
#[tokio::test(flavor = "multi_thread")]
|
#[tokio::test(flavor = "multi_thread")]
|
||||||
|
#[serial]
|
||||||
#[ignore = "requires awscurl and spawns a real RustFS server"]
|
#[ignore = "requires awscurl and spawns a real RustFS server"]
|
||||||
async fn test_delete_group_requires_empty_membership() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
async fn test_delete_group_requires_empty_membership() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||||
init_logging();
|
init_logging();
|
||||||
@@ -142,6 +94,7 @@ async fn test_delete_group_requires_empty_membership() -> Result<(), Box<dyn std
|
|||||||
/// Test that a user with only group membership (no explicit user policy) gets group policies
|
/// Test that a user with only group membership (no explicit user policy) gets group policies
|
||||||
/// and can perform actions allowed by the group (regression test for #2028.1).
|
/// and can perform actions allowed by the group (regression test for #2028.1).
|
||||||
#[tokio::test(flavor = "multi_thread")]
|
#[tokio::test(flavor = "multi_thread")]
|
||||||
|
#[serial]
|
||||||
#[ignore = "requires awscurl and spawns a real RustFS server"]
|
#[ignore = "requires awscurl and spawns a real RustFS server"]
|
||||||
async fn test_user_with_only_group_gets_group_policies() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
async fn test_user_with_only_group_gets_group_policies() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||||
init_logging();
|
init_logging();
|
||||||
@@ -209,6 +162,7 @@ async fn test_user_with_only_group_gets_group_policies() -> Result<(), Box<dyn s
|
|||||||
/// Test that after deleting a user who was the only member of a group, the group can be deleted
|
/// Test that after deleting a user who was the only member of a group, the group can be deleted
|
||||||
/// (regression test for #2028.2: delete group uses backend membership, not stale cache).
|
/// (regression test for #2028.2: delete group uses backend membership, not stale cache).
|
||||||
#[tokio::test(flavor = "multi_thread")]
|
#[tokio::test(flavor = "multi_thread")]
|
||||||
|
#[serial]
|
||||||
#[ignore = "requires awscurl and spawns a real RustFS server"]
|
#[ignore = "requires awscurl and spawns a real RustFS server"]
|
||||||
async fn test_delete_group_after_deleting_user() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
async fn test_delete_group_after_deleting_user() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||||
init_logging();
|
init_logging();
|
||||||
|
|||||||
@@ -16,6 +16,7 @@ use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
|
|||||||
use aws_sdk_s3::presigning::PresigningConfig;
|
use aws_sdk_s3::presigning::PresigningConfig;
|
||||||
use aws_sdk_s3::primitives::ByteStream;
|
use aws_sdk_s3::primitives::ByteStream;
|
||||||
use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
|
use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
|
||||||
|
use serial_test::serial;
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
use tracing::info;
|
use tracing::info;
|
||||||
|
|
||||||
@@ -28,6 +29,7 @@ fn list_contains_key(output: &aws_sdk_s3::operation::list_objects_v2::ListObject
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn head_object_consistency_after_write_and_multipart_and_presigned_head()
|
async fn head_object_consistency_after_write_and_multipart_and_presigned_head()
|
||||||
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||||
init_logging();
|
init_logging();
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
use crate::common::{RustFSTestEnvironment, init_logging};
|
use crate::common::{RustFSTestEnvironment, init_logging};
|
||||||
use aws_sdk_s3::primitives::ByteStream;
|
use aws_sdk_s3::primitives::ByteStream;
|
||||||
|
use serial_test::serial;
|
||||||
use tracing::info;
|
use tracing::info;
|
||||||
|
|
||||||
const RANGE_HEAD_BUCKET: &str = "range-head-test-bucket";
|
const RANGE_HEAD_BUCKET: &str = "range-head-test-bucket";
|
||||||
@@ -7,6 +8,7 @@ const RANGE_HEAD_KEY: &str = "range-head-object.bin";
|
|||||||
const ACCEPT_RANGES_BYTES: &str = "bytes";
|
const ACCEPT_RANGES_BYTES: &str = "bytes";
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn head_object_advertises_accept_ranges() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
async fn head_object_advertises_accept_ranges() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||||
init_logging();
|
init_logging();
|
||||||
info!("Starting HeadObject Accept-Ranges regression test");
|
info!("Starting HeadObject Accept-Ranges regression test");
|
||||||
|
|||||||
@@ -19,6 +19,7 @@ mod tests {
|
|||||||
use crate::chaos::signed_admin_post;
|
use crate::chaos::signed_admin_post;
|
||||||
use crate::common::{RustFSTestClusterEnvironment, RustFSTestEnvironment, init_logging};
|
use crate::common::{RustFSTestClusterEnvironment, RustFSTestEnvironment, init_logging};
|
||||||
use aws_sdk_s3::primitives::ByteStream;
|
use aws_sdk_s3::primitives::ByteStream;
|
||||||
|
use serial_test::serial;
|
||||||
use std::collections::HashSet;
|
use std::collections::HashSet;
|
||||||
use std::error::Error;
|
use std::error::Error;
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
@@ -62,6 +63,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn test_auto_heal_rebuilds_runtime_wiped_disk_without_restart() {
|
async fn test_auto_heal_rebuilds_runtime_wiped_disk_without_restart() {
|
||||||
init_logging();
|
init_logging();
|
||||||
info!("Issue #1533: auto heal should rebuild a runtime-wiped disk in a 4-disk single-node erasure set without restart");
|
info!("Issue #1533: auto heal should rebuild a runtime-wiped disk in a 4-disk single-node erasure set without restart");
|
||||||
@@ -180,6 +182,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn test_admin_deep_heal_rebuilds_cleared_disk_in_single_node_erasure_set() {
|
async fn test_admin_deep_heal_rebuilds_cleared_disk_in_single_node_erasure_set() {
|
||||||
init_logging();
|
init_logging();
|
||||||
info!("Discussion #2964: admin deep heal should rebuild a wiped disk in a 4-disk single-node erasure set");
|
info!("Discussion #2964: admin deep heal should rebuild a wiped disk in a 4-disk single-node erasure set");
|
||||||
@@ -329,6 +332,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test(flavor = "multi_thread")]
|
#[tokio::test(flavor = "multi_thread")]
|
||||||
|
#[serial]
|
||||||
async fn test_cluster_root_heal_rebuilds_replaced_remote_disk() -> Result<(), Box<dyn Error + Send + Sync>> {
|
async fn test_cluster_root_heal_rebuilds_replaced_remote_disk() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||||
init_logging();
|
init_logging();
|
||||||
info!("Root recursive heal should rebuild data on a remote node after its disk is replaced and the node rejoins");
|
info!("Root recursive heal should rebuild data on a remote node after its disk is replaced and the node rejoins");
|
||||||
@@ -440,6 +444,7 @@ mod tests {
|
|||||||
/// topology early-return or the merge hard-fail) turns the down-window
|
/// topology early-return or the merge hard-fail) turns the down-window
|
||||||
/// response into a 500 and fails this test.
|
/// response into a 500 and fails this test.
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn test_background_heal_status_degrades_while_peer_down_and_recovers_after_rejoin()
|
async fn test_background_heal_status_degrades_while_peer_down_and_recovers_after_rejoin()
|
||||||
-> Result<(), Box<dyn Error + Send + Sync>> {
|
-> Result<(), Box<dyn Error + Send + Sync>> {
|
||||||
init_logging();
|
init_logging();
|
||||||
|
|||||||
@@ -67,9 +67,6 @@ type MetricValues = Arc<Mutex<BTreeMap<String, MetricPointVersions>>>;
|
|||||||
|
|
||||||
const KIB: usize = 1024;
|
const KIB: usize = 1024;
|
||||||
const READER_PATH_COUNTER: &str = "rustfs_io_get_object_reader_path_by_size_total";
|
const READER_PATH_COUNTER: &str = "rustfs_io_get_object_reader_path_by_size_total";
|
||||||
/// Physical bytes the erasure layer pulled from disk, emitted per shard read by
|
|
||||||
/// `crates/ecstore/src/erasure/coding/decode.rs`.
|
|
||||||
const SHARD_READ_BYTES_COUNTER: &str = "rustfs_io_get_object_shard_read_observed_bytes_total";
|
|
||||||
const MSGPACK_JSON_DECODE_COUNTER: &str = "rustfs_system_network_internode_msgpack_json_decode_total";
|
const MSGPACK_JSON_DECODE_COUNTER: &str = "rustfs_system_network_internode_msgpack_json_decode_total";
|
||||||
const MSGPACK_JSON_FALLBACK_COUNTER: &str = "rustfs_system_network_internode_msgpack_json_fallback_total";
|
const MSGPACK_JSON_FALLBACK_COUNTER: &str = "rustfs_system_network_internode_msgpack_json_fallback_total";
|
||||||
const MSGPACK_JSON_DECODE_ERROR_COUNTER: &str = "rustfs_system_network_internode_msgpack_json_decode_error_total";
|
const MSGPACK_JSON_DECODE_ERROR_COUNTER: &str = "rustfs_system_network_internode_msgpack_json_decode_error_total";
|
||||||
@@ -149,7 +146,6 @@ struct OtlpMetricCollector {
|
|||||||
decode_values: MetricValues,
|
decode_values: MetricValues,
|
||||||
fallback_values: MetricValues,
|
fallback_values: MetricValues,
|
||||||
decode_error_values: MetricValues,
|
decode_error_values: MetricValues,
|
||||||
shard_read_values: MetricValues,
|
|
||||||
task: JoinHandle<()>,
|
task: JoinHandle<()>,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -161,12 +157,10 @@ impl OtlpMetricCollector {
|
|||||||
let decode_values = Arc::new(Mutex::new(BTreeMap::new()));
|
let decode_values = Arc::new(Mutex::new(BTreeMap::new()));
|
||||||
let fallback_values = Arc::new(Mutex::new(BTreeMap::new()));
|
let fallback_values = Arc::new(Mutex::new(BTreeMap::new()));
|
||||||
let decode_error_values = Arc::new(Mutex::new(BTreeMap::new()));
|
let decode_error_values = Arc::new(Mutex::new(BTreeMap::new()));
|
||||||
let shard_read_values = Arc::new(Mutex::new(BTreeMap::new()));
|
|
||||||
let task_values = values.clone();
|
let task_values = values.clone();
|
||||||
let task_decode_values = decode_values.clone();
|
let task_decode_values = decode_values.clone();
|
||||||
let task_fallback_values = fallback_values.clone();
|
let task_fallback_values = fallback_values.clone();
|
||||||
let task_decode_error_values = decode_error_values.clone();
|
let task_decode_error_values = decode_error_values.clone();
|
||||||
let task_shard_read_values = shard_read_values.clone();
|
|
||||||
let task = tokio::spawn(async move {
|
let task = tokio::spawn(async move {
|
||||||
loop {
|
loop {
|
||||||
let Ok((stream, _)) = listener.accept().await else {
|
let Ok((stream, _)) = listener.accept().await else {
|
||||||
@@ -176,7 +170,6 @@ impl OtlpMetricCollector {
|
|||||||
let decode_values = task_decode_values.clone();
|
let decode_values = task_decode_values.clone();
|
||||||
let fallback_values = task_fallback_values.clone();
|
let fallback_values = task_fallback_values.clone();
|
||||||
let decode_error_values = task_decode_error_values.clone();
|
let decode_error_values = task_decode_error_values.clone();
|
||||||
let shard_read_values = task_shard_read_values.clone();
|
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
let _ = hyper::server::conn::http1::Builder::new()
|
let _ = hyper::server::conn::http1::Builder::new()
|
||||||
.serve_connection(
|
.serve_connection(
|
||||||
@@ -188,7 +181,6 @@ impl OtlpMetricCollector {
|
|||||||
decode_values.clone(),
|
decode_values.clone(),
|
||||||
fallback_values.clone(),
|
fallback_values.clone(),
|
||||||
decode_error_values.clone(),
|
decode_error_values.clone(),
|
||||||
shard_read_values.clone(),
|
|
||||||
)
|
)
|
||||||
}),
|
}),
|
||||||
)
|
)
|
||||||
@@ -202,48 +194,10 @@ impl OtlpMetricCollector {
|
|||||||
decode_values,
|
decode_values,
|
||||||
fallback_values,
|
fallback_values,
|
||||||
decode_error_values,
|
decode_error_values,
|
||||||
shard_read_values,
|
|
||||||
task,
|
task,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Total physical bytes read from disk across every shard-read label set.
|
|
||||||
async fn shard_read_bytes_total(&self) -> u64 {
|
|
||||||
self.shard_read_values
|
|
||||||
.lock()
|
|
||||||
.await
|
|
||||||
.values()
|
|
||||||
.map(|versions| versions.values().map(|(_, value)| *value).sum::<u64>())
|
|
||||||
.sum()
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Waits until the shard-read counter stops advancing so a measurement window
|
|
||||||
/// is not polluted by exports still in flight.
|
|
||||||
///
|
|
||||||
/// Requires several consecutive equal samples spanning more than one export
|
|
||||||
/// interval (`RUSTFS_OBS_METER_INTERVAL=1`): a single unchanged sample only
|
|
||||||
/// proves the latest export has not landed yet, which silently reads as "no
|
|
||||||
/// disk reads happened" and makes any upper-bound assertion vacuous.
|
|
||||||
async fn wait_for_shard_read_bytes_to_settle(&self) -> TestResult<u64> {
|
|
||||||
const REQUIRED_STABLE_SAMPLES: usize = 5;
|
|
||||||
let mut last = self.shard_read_bytes_total().await;
|
|
||||||
let mut stable = 0;
|
|
||||||
for _ in 0..60 {
|
|
||||||
sleep(Duration::from_millis(500)).await;
|
|
||||||
let current = self.shard_read_bytes_total().await;
|
|
||||||
if current == last {
|
|
||||||
stable += 1;
|
|
||||||
if stable >= REQUIRED_STABLE_SAMPLES {
|
|
||||||
return Ok(current);
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
stable = 0;
|
|
||||||
last = current;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Err("timed out waiting for shard-read byte counter to settle".into())
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn reader_path_total(&self, path: &str, object_class: &str, size_bucket: &str) -> u64 {
|
async fn reader_path_total(&self, path: &str, object_class: &str, size_bucket: &str) -> u64 {
|
||||||
self.reader_path_values(path, object_class, size_bucket).await.values().sum()
|
self.reader_path_values(path, object_class, size_bucket).await.values().sum()
|
||||||
}
|
}
|
||||||
@@ -367,7 +321,6 @@ async fn handle_metric_export(
|
|||||||
decode_values: MetricValues,
|
decode_values: MetricValues,
|
||||||
fallback_values: MetricValues,
|
fallback_values: MetricValues,
|
||||||
decode_error_values: MetricValues,
|
decode_error_values: MetricValues,
|
||||||
shard_read_values: MetricValues,
|
|
||||||
) -> Result<Response<Full<Bytes>>, Infallible> {
|
) -> Result<Response<Full<Bytes>>, Infallible> {
|
||||||
if request.uri().path() != "/v1/metrics" {
|
if request.uri().path() != "/v1/metrics" {
|
||||||
return Ok(response(StatusCode::NOT_FOUND));
|
return Ok(response(StatusCode::NOT_FOUND));
|
||||||
@@ -401,9 +354,7 @@ async fn handle_metric_export(
|
|||||||
let mut decode_values = decode_values.lock().await;
|
let mut decode_values = decode_values.lock().await;
|
||||||
let mut fallback_values = fallback_values.lock().await;
|
let mut fallback_values = fallback_values.lock().await;
|
||||||
let mut decode_error_values = decode_error_values.lock().await;
|
let mut decode_error_values = decode_error_values.lock().await;
|
||||||
let mut shard_read_values = shard_read_values.lock().await;
|
|
||||||
record_reader_path_metrics(&export, &mut values);
|
record_reader_path_metrics(&export, &mut values);
|
||||||
record_shard_read_bytes_metrics(&export, &mut shard_read_values);
|
|
||||||
record_msgpack_decode_metrics(&export, &mut decode_values);
|
record_msgpack_decode_metrics(&export, &mut decode_values);
|
||||||
record_msgpack_fallback_metrics(&export, &mut fallback_values);
|
record_msgpack_fallback_metrics(&export, &mut fallback_values);
|
||||||
record_msgpack_decode_error_metrics(&export, &mut decode_error_values);
|
record_msgpack_decode_error_metrics(&export, &mut decode_error_values);
|
||||||
@@ -424,50 +375,6 @@ fn reader_path_metric_key(path: &str, object_class: &str, size_bucket: &str) ->
|
|||||||
format!("{path}\u{1f}{object_class}\u{1f}{size_bucket}")
|
format!("{path}\u{1f}{object_class}\u{1f}{size_bucket}")
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Accumulates `SHARD_READ_BYTES_COUNTER` across all label sets. Only the total
|
|
||||||
/// matters: it is the number of physical bytes the erasure layer actually pulled
|
|
||||||
/// from disk, which is what separates a bounded per-part read from a decode of
|
|
||||||
/// the whole object.
|
|
||||||
fn record_shard_read_bytes_metrics(export: &ExportMetricsServiceRequest, values: &mut BTreeMap<String, MetricPointVersions>) {
|
|
||||||
for resource_metrics in &export.resource_metrics {
|
|
||||||
for scope_metrics in &resource_metrics.scope_metrics {
|
|
||||||
for metric in &scope_metrics.metrics {
|
|
||||||
if metric.name != SHARD_READ_BYTES_COUNTER {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
let Some(metric::Data::Sum(sum)) = &metric.data else {
|
|
||||||
continue;
|
|
||||||
};
|
|
||||||
for point in &sum.data_points {
|
|
||||||
let Some(number_data_point::Value::AsInt(value)) = point.value.as_ref() else {
|
|
||||||
continue;
|
|
||||||
};
|
|
||||||
let value = u64::try_from(*value).unwrap_or_default();
|
|
||||||
// Keyed by labels, not by position: point order within an export
|
|
||||||
// is not guaranteed stable, so an index key would alias distinct
|
|
||||||
// series across batches.
|
|
||||||
let key = format!(
|
|
||||||
"{}\u{1f}{}\u{1f}{}",
|
|
||||||
attribute_string(&point.attributes, "path").unwrap_or_default(),
|
|
||||||
attribute_string(&point.attributes, "role").unwrap_or_default(),
|
|
||||||
attribute_string(&point.attributes, "outcome").unwrap_or_default(),
|
|
||||||
);
|
|
||||||
values
|
|
||||||
.entry(key)
|
|
||||||
.or_default()
|
|
||||||
.entry(point.start_time_unix_nano)
|
|
||||||
.and_modify(|current| {
|
|
||||||
if point.time_unix_nano >= current.0 {
|
|
||||||
*current = (point.time_unix_nano, value);
|
|
||||||
}
|
|
||||||
})
|
|
||||||
.or_insert((point.time_unix_nano, value));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn record_reader_path_metrics(export: &ExportMetricsServiceRequest, values: &mut BTreeMap<String, MetricPointVersions>) {
|
fn record_reader_path_metrics(export: &ExportMetricsServiceRequest, values: &mut BTreeMap<String, MetricPointVersions>) {
|
||||||
for resource_metrics in &export.resource_metrics {
|
for resource_metrics in &export.resource_metrics {
|
||||||
for scope_metrics in &resource_metrics.scope_metrics {
|
for scope_metrics in &resource_metrics.scope_metrics {
|
||||||
@@ -1028,6 +935,20 @@ impl<'a> ReaderPathExpectation<'a> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn with_size_bucket(
|
||||||
|
object: ReaderObject<'a>,
|
||||||
|
expected_path: &'a str,
|
||||||
|
object_class: &'a str,
|
||||||
|
expected_size_bucket: &'a str,
|
||||||
|
) -> Self {
|
||||||
|
Self {
|
||||||
|
object,
|
||||||
|
expected_path,
|
||||||
|
object_class,
|
||||||
|
expected_size_bucket: Some(expected_size_bucket),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn with_any_size_bucket(object: ReaderObject<'a>, expected_path: &'a str, object_class: &'a str) -> Self {
|
fn with_any_size_bucket(object: ReaderObject<'a>, expected_path: &'a str, object_class: &'a str) -> Self {
|
||||||
Self {
|
Self {
|
||||||
object,
|
object,
|
||||||
@@ -1895,129 +1816,51 @@ async fn four_node_compressed_inline_fallback() -> TestResult {
|
|||||||
assert_reader_path(
|
assert_reader_path(
|
||||||
&collector,
|
&collector,
|
||||||
&client,
|
&client,
|
||||||
ReaderPathExpectation::for_class(ReaderObject::new(bucket, key, &body, put.e_tag(), None), LEGACY_DUPLEX, COMPRESSED),
|
ReaderPathExpectation::with_size_bucket(
|
||||||
|
ReaderObject::new(bucket, key, &body, put.e_tag(), None),
|
||||||
|
LEGACY_DUPLEX,
|
||||||
|
COMPRESSED,
|
||||||
|
size_bucket(4 * KIB),
|
||||||
|
),
|
||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Multipart disk compression is live again, so a compression-enabled cluster classifies multipart objects as compressed and the roundtrip (full GET plus partNumber GET) must still return the original bytes.
|
|
||||||
/// Reverting the multipart compression fix must fail this test.
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
#[serial]
|
||||||
async fn four_node_multipart_disk_compression_roundtrip() -> TestResult {
|
async fn four_node_multipart_ignores_disk_compression_fallback() -> TestResult {
|
||||||
init_logging();
|
init_logging();
|
||||||
|
|
||||||
let collector = OtlpMetricCollector::start().await?;
|
let collector = OtlpMetricCollector::start().await?;
|
||||||
let mut cluster = RustFSTestClusterEnvironment::new(4).await?;
|
let mut cluster = RustFSTestClusterEnvironment::new(4).await?;
|
||||||
configure_reader_metric_cluster(&mut cluster, &collector);
|
configure_reader_metric_cluster(&mut cluster, &collector);
|
||||||
cluster.set_env("RUSTFS_COMPRESSION_ENABLED", "true");
|
cluster.set_env("RUSTFS_COMPRESSION_ENABLED", "true");
|
||||||
cluster.set_env("RUSTFS_COMPRESSION_MULTIPART_ENABLED", "true");
|
|
||||||
cluster.start().await?;
|
cluster.start().await?;
|
||||||
|
|
||||||
let bucket = "inline-multipart-compression-roundtrip";
|
let bucket = "inline-multipart-compression-fallback";
|
||||||
cluster.create_test_bucket(bucket).await?;
|
cluster.create_test_bucket(bucket).await?;
|
||||||
let client = cluster.create_s3_client(0)?;
|
let client = cluster.create_s3_client(0)?;
|
||||||
let key = "multipart/compressed.txt";
|
let key = "multipart/compression-disabled.txt";
|
||||||
let (body, second_part, etag) = put_two_part_multipart(&client, bucket, key).await?;
|
let (body, second_part, etag) = put_two_part_multipart(&client, bucket, key).await?;
|
||||||
|
|
||||||
assert_reader_path(
|
assert_reader_path(
|
||||||
&collector,
|
&collector,
|
||||||
&client,
|
&client,
|
||||||
ReaderPathExpectation::for_class(ReaderObject::new(bucket, key, &body, etag.as_deref(), None), LEGACY_DUPLEX, COMPRESSED),
|
ReaderPathExpectation::for_class(ReaderObject::new(bucket, key, &body, etag.as_deref(), None), LEGACY_DUPLEX, MULTIPART),
|
||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
assert_part_number_reader_path(
|
assert_part_number_reader_path(
|
||||||
&collector,
|
&collector,
|
||||||
&client,
|
&client,
|
||||||
PartNumberReaderPathExpectation::new(bucket, key, &second_part, body.len(), COMPRESSED, LEGACY_DUPLEX),
|
PartNumberReaderPathExpectation::new(bucket, key, &second_part, body.len(), MULTIPART, LEGACY_DUPLEX),
|
||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// A tail range over a compressed multipart object must read only the physical
|
|
||||||
/// data it needs, not decode the object from byte zero.
|
|
||||||
///
|
|
||||||
/// The byte-exactness tests around this one stay green even if the seek path
|
|
||||||
/// regresses into decoding from the start of the object: the bytes returned are
|
|
||||||
/// still correct, only the read amplification explodes. This asserts the cost
|
|
||||||
/// side, using `SHARD_READ_BYTES_COUNTER` — already emitted per shard read by the
|
|
||||||
/// erasure layer, so no production code is instrumented for the test.
|
|
||||||
///
|
|
||||||
/// `get_compressed_offsets` skips whole preceding parts by their stored size and
|
|
||||||
/// then seeks inside the covering part via its compression index, so a bounded
|
|
||||||
/// read costs on the order of the covering part's block size against a ~5 MiB
|
|
||||||
/// object.
|
|
||||||
#[tokio::test]
|
|
||||||
#[serial]
|
|
||||||
async fn four_node_compressed_multipart_tail_range_reads_are_bounded() -> TestResult {
|
|
||||||
init_logging();
|
|
||||||
|
|
||||||
let collector = OtlpMetricCollector::start().await?;
|
|
||||||
let mut cluster = RustFSTestClusterEnvironment::new(4).await?;
|
|
||||||
configure_reader_metric_cluster(&mut cluster, &collector);
|
|
||||||
cluster.set_env("RUSTFS_COMPRESSION_ENABLED", "true");
|
|
||||||
cluster.set_env("RUSTFS_COMPRESSION_MULTIPART_ENABLED", "true");
|
|
||||||
cluster.start().await?;
|
|
||||||
|
|
||||||
let bucket = "inline-multipart-compression-tail-range";
|
|
||||||
cluster.create_test_bucket(bucket).await?;
|
|
||||||
let client = cluster.create_s3_client(0)?;
|
|
||||||
let key = "multipart/tail-range.txt";
|
|
||||||
let (body, _second_part, etag) = put_two_part_multipart(&client, bucket, key).await?;
|
|
||||||
|
|
||||||
// Establish that the object really took the compressed read path; otherwise a
|
|
||||||
// small delta below would only prove compression never happened.
|
|
||||||
assert_reader_path(
|
|
||||||
&collector,
|
|
||||||
&client,
|
|
||||||
ReaderPathExpectation::for_class(ReaderObject::new(bucket, key, &body, etag.as_deref(), None), LEGACY_DUPLEX, COMPRESSED),
|
|
||||||
)
|
|
||||||
.await?;
|
|
||||||
|
|
||||||
let baseline = collector.wait_for_shard_read_bytes_to_settle().await?;
|
|
||||||
|
|
||||||
let tail_len = 4 * KIB;
|
|
||||||
let start = body.len() - tail_len;
|
|
||||||
let end = body.len() - 1;
|
|
||||||
let range = client
|
|
||||||
.get_object()
|
|
||||||
.bucket(bucket)
|
|
||||||
.key(key)
|
|
||||||
.range(format!("bytes={start}-{end}"))
|
|
||||||
.send()
|
|
||||||
.await?;
|
|
||||||
let tail = range.body.collect().await?.into_bytes();
|
|
||||||
assert_eq!(tail.as_ref(), &body[start..], "tail range returned wrong bytes");
|
|
||||||
|
|
||||||
let after = collector.wait_for_shard_read_bytes_to_settle().await?;
|
|
||||||
let read_bytes = after.saturating_sub(baseline);
|
|
||||||
|
|
||||||
// A zero delta means the window caught nothing — an unexported counter, or a
|
|
||||||
// read served without touching the erasure layer — which would make the upper
|
|
||||||
// bound vacuously true. Fail instead of passing blind.
|
|
||||||
assert!(
|
|
||||||
read_bytes > 0,
|
|
||||||
"no shard reads observed for the tail range; the budget assertion below would be vacuous"
|
|
||||||
);
|
|
||||||
|
|
||||||
// Part 1 alone is MPU_PART_1_SIZE, so a whole-object decode cannot come in
|
|
||||||
// under it. Half the logical size leaves generous headroom for erasure padding
|
|
||||||
// and unrelated background reads while still failing loudly on a full decode.
|
|
||||||
let budget = (body.len() / 2) as u64;
|
|
||||||
assert!(
|
|
||||||
read_bytes < budget,
|
|
||||||
"tail range read {read_bytes} physical bytes for a {tail_len}-byte range (budget {budget}, object {} bytes): \
|
|
||||||
the read is not bounded to the covering part",
|
|
||||||
body.len()
|
|
||||||
);
|
|
||||||
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
#[serial]
|
||||||
async fn four_node_mixed_msgpack_compat_mode_preserves_fallback_controls() -> TestResult {
|
async fn four_node_mixed_msgpack_compat_mode_preserves_fallback_controls() -> TestResult {
|
||||||
@@ -2028,7 +1871,6 @@ async fn four_node_mixed_msgpack_compat_mode_preserves_fallback_controls() -> Te
|
|||||||
let sse_master_key = base64::engine::general_purpose::STANDARD.encode([0x42u8; 32]);
|
let sse_master_key = base64::engine::general_purpose::STANDARD.encode([0x42u8; 32]);
|
||||||
cluster.set_env("RUSTFS_SSE_S3_MASTER_KEY", sse_master_key);
|
cluster.set_env("RUSTFS_SSE_S3_MASTER_KEY", sse_master_key);
|
||||||
cluster.set_env("RUSTFS_COMPRESSION_ENABLED", "true");
|
cluster.set_env("RUSTFS_COMPRESSION_ENABLED", "true");
|
||||||
cluster.set_env("RUSTFS_COMPRESSION_MULTIPART_ENABLED", "true");
|
|
||||||
configure_mixed_msgpack_cluster(&mut cluster, &collector)?;
|
configure_mixed_msgpack_cluster(&mut cluster, &collector)?;
|
||||||
cluster.start().await?;
|
cluster.start().await?;
|
||||||
|
|
||||||
@@ -2048,21 +1890,14 @@ async fn four_node_mixed_msgpack_compat_mode_preserves_fallback_controls() -> Te
|
|||||||
ReaderPathExpectation::for_class(
|
ReaderPathExpectation::for_class(
|
||||||
ReaderObject::new(bucket, multipart_key, &multipart_body, multipart_etag.as_deref(), None),
|
ReaderObject::new(bucket, multipart_key, &multipart_body, multipart_etag.as_deref(), None),
|
||||||
LEGACY_DUPLEX,
|
LEGACY_DUPLEX,
|
||||||
COMPRESSED,
|
MULTIPART,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
assert_part_number_reader_path(
|
assert_part_number_reader_path(
|
||||||
&collector,
|
&collector,
|
||||||
&client,
|
&client,
|
||||||
PartNumberReaderPathExpectation::new(
|
PartNumberReaderPathExpectation::new(bucket, multipart_key, &second_part, multipart_body.len(), MULTIPART, LEGACY_DUPLEX),
|
||||||
bucket,
|
|
||||||
multipart_key,
|
|
||||||
&second_part,
|
|
||||||
multipart_body.len(),
|
|
||||||
COMPRESSED,
|
|
||||||
LEGACY_DUPLEX,
|
|
||||||
),
|
|
||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
assert_msgpack_decode_observed(&collector, &decode_before).await?;
|
assert_msgpack_decode_observed(&collector, &decode_before).await?;
|
||||||
@@ -2255,7 +2090,6 @@ async fn four_node_manual_transition_distributed_admission_conflict_reports_stat
|
|||||||
hot.set_env("RUSTFS_SCANNER_CYCLE", "3600");
|
hot.set_env("RUSTFS_SCANNER_CYCLE", "3600");
|
||||||
hot.set_env("RUSTFS_MAX_TRANSITION_WORKERS", "1");
|
hot.set_env("RUSTFS_MAX_TRANSITION_WORKERS", "1");
|
||||||
hot.set_env("RUSTFS_TRANSITION_QUEUE_CAPACITY", "1");
|
hot.set_env("RUSTFS_TRANSITION_QUEUE_CAPACITY", "1");
|
||||||
hot.set_env("RUSTFS_TRANSITION_QUEUE_SEND_TIMEOUT_MS", "1");
|
|
||||||
hot.start().await?;
|
hot.start().await?;
|
||||||
|
|
||||||
let hot_client = hot.create_s3_client(0)?;
|
let hot_client = hot.create_s3_client(0)?;
|
||||||
@@ -2272,7 +2106,7 @@ async fn four_node_manual_transition_distributed_admission_conflict_reports_stat
|
|||||||
.put_object()
|
.put_object()
|
||||||
.bucket(&bucket)
|
.bucket(&bucket)
|
||||||
.key(key)
|
.key(key)
|
||||||
.body(ByteStream::from(payload(1024 * KIB, index)))
|
.body(ByteStream::from(payload(64 * KIB, index)))
|
||||||
.send()
|
.send()
|
||||||
.await?;
|
.await?;
|
||||||
}
|
}
|
||||||
@@ -2519,11 +2353,7 @@ async fn four_node_mixed_msgpack_compat_mode_preserves_fallback_controls_during_
|
|||||||
hot_client.create_bucket().bucket(bucket).send().await?;
|
hot_client.create_bucket().bucket(bucket).send().await?;
|
||||||
put_lifecycle_with_transition_retry(&hot_client, bucket, &tier_name).await?;
|
put_lifecycle_with_transition_retry(&hot_client, bucket, &tier_name).await?;
|
||||||
|
|
||||||
// `.zip` sits on the disk-compression exclusion list: this test pins
|
let key = "transition/mixed-multipart.bin";
|
||||||
// msgpack compat controls across ILM transition, and a compressed object
|
|
||||||
// would classify as `compressed` instead of `remote` (and the warm-tier
|
|
||||||
// read path does not decode compression — tracked separately).
|
|
||||||
let key = "transition/mixed-multipart.zip";
|
|
||||||
let (body, second_part, etag) = put_two_part_multipart(&hot_client, bucket, key).await?;
|
let (body, second_part, etag) = put_two_part_multipart(&hot_client, bucket, key).await?;
|
||||||
wait_for_transition(&hot_client, bucket, key, &tier_name).await?;
|
wait_for_transition(&hot_client, bucket, key, &tier_name).await?;
|
||||||
assert!(
|
assert!(
|
||||||
|
|||||||
@@ -101,6 +101,7 @@ use rustfs_config::{
|
|||||||
};
|
};
|
||||||
use rustfs_protos::canonical_make_volume_request_body;
|
use rustfs_protos::canonical_make_volume_request_body;
|
||||||
use rustfs_protos::proto_gen::node_service::{MakeVolumeRequest, MakeVolumeResponse, PingRequest, PingResponse};
|
use rustfs_protos::proto_gen::node_service::{MakeVolumeRequest, MakeVolumeResponse, PingRequest, PingResponse};
|
||||||
|
use serial_test::serial;
|
||||||
use sha2::{Digest, Sha256};
|
use sha2::{Digest, Sha256};
|
||||||
use std::error::Error;
|
use std::error::Error;
|
||||||
use tonic::{Code, Request, Response, Status};
|
use tonic::{Code, Request, Response, Status};
|
||||||
@@ -396,6 +397,7 @@ fn assert_rejected(result: Result<MakeVolumeResponse, Status>, expected: Code, e
|
|||||||
/// Grouped into one server start because each case is independent and spawning
|
/// Grouped into one server start because each case is independent and spawning
|
||||||
/// a `rustfs` process per assertion would dominate the runtime.
|
/// a `rustfs` process per assertion would dominate the runtime.
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn internode_rpc_signature_default_posture_e2e() -> TestResult {
|
async fn internode_rpc_signature_default_posture_e2e() -> TestResult {
|
||||||
init_logging();
|
init_logging();
|
||||||
align_rpc_secret_with_server();
|
align_rpc_secret_with_server();
|
||||||
@@ -422,6 +424,7 @@ async fn internode_rpc_signature_default_posture_e2e() -> TestResult {
|
|||||||
/// epoch is learned from a real response, then the same server is restarted in place to prove its
|
/// epoch is learned from a real response, then the same server is restarted in place to prove its
|
||||||
/// replacement epoch rejects the captured request even though the nonce cache is necessarily new.
|
/// replacement epoch rejects the captured request even though the nonce cache is necessarily new.
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn replay_scope_rejects_replay_path_transplant_and_stale_epoch_e2e() -> TestResult {
|
async fn replay_scope_rejects_replay_path_transplant_and_stale_epoch_e2e() -> TestResult {
|
||||||
init_logging();
|
init_logging();
|
||||||
align_rpc_secret_with_server();
|
align_rpc_secret_with_server();
|
||||||
@@ -494,6 +497,7 @@ async fn replay_scope_rejects_replay_path_transplant_and_stale_epoch_e2e() -> Te
|
|||||||
/// A mutating v2 request cannot use that lane; once the epoch proof is returned, the first v3
|
/// A mutating v2 request cannot use that lane; once the epoch proof is returned, the first v3
|
||||||
/// mutation succeeds. This protects a server restart without reopening a general downgrade path.
|
/// mutation succeeds. This protects a server restart without reopening a general downgrade path.
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn replay_scope_strict_requires_v3_after_ping_bootstrap_e2e() -> TestResult {
|
async fn replay_scope_strict_requires_v3_after_ping_bootstrap_e2e() -> TestResult {
|
||||||
init_logging();
|
init_logging();
|
||||||
align_rpc_secret_with_server();
|
align_rpc_secret_with_server();
|
||||||
@@ -700,6 +704,7 @@ async fn legacy_only_signature_is_accepted_in_default_posture(url: &str) {
|
|||||||
///
|
///
|
||||||
/// The paired v2 positive control rules out "strict simply breaks everything".
|
/// The paired v2 positive control rules out "strict simply breaks everything".
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn signature_strict_rejects_legacy_only_downgrade() -> TestResult {
|
async fn signature_strict_rejects_legacy_only_downgrade() -> TestResult {
|
||||||
init_logging();
|
init_logging();
|
||||||
align_rpc_secret_with_server();
|
align_rpc_secret_with_server();
|
||||||
@@ -736,6 +741,7 @@ async fn signature_strict_rejects_legacy_only_downgrade() -> TestResult {
|
|||||||
/// takes the still-open legacy lane), which is what pins the rejection to the
|
/// takes the still-open legacy lane), which is what pins the rejection to the
|
||||||
/// handler's digest gate; the cited message confirms which check spoke.
|
/// handler's digest gate; the cited message confirms which check spoke.
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn body_digest_strict_rejects_digestless_mutation() -> TestResult {
|
async fn body_digest_strict_rejects_digestless_mutation() -> TestResult {
|
||||||
init_logging();
|
init_logging();
|
||||||
align_rpc_secret_with_server();
|
align_rpc_secret_with_server();
|
||||||
|
|||||||
@@ -27,10 +27,12 @@ use aws_sdk_s3::types::{
|
|||||||
ServerSideEncryptionByDefault, ServerSideEncryptionConfiguration, ServerSideEncryptionRule,
|
ServerSideEncryptionByDefault, ServerSideEncryptionConfiguration, ServerSideEncryptionRule,
|
||||||
};
|
};
|
||||||
use rustfs_rio::{Checksum, ChecksumType};
|
use rustfs_rio::{Checksum, ChecksumType};
|
||||||
|
use serial_test::serial;
|
||||||
use tracing::{debug, info, warn};
|
use tracing::{debug, info, warn};
|
||||||
|
|
||||||
/// Test 1: When bucket is configured with default SSE-S3 encryption, put_object should automatically apply encryption
|
/// Test 1: When bucket is configured with default SSE-S3 encryption, put_object should automatically apply encryption
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn test_bucket_default_sse_s3_put_object() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
async fn test_bucket_default_sse_s3_put_object() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||||
init_logging();
|
init_logging();
|
||||||
info!("Testing bucket default SSE-S3 encryption impact on put_object");
|
info!("Testing bucket default SSE-S3 encryption impact on put_object");
|
||||||
@@ -153,6 +155,7 @@ async fn test_bucket_default_sse_s3_put_object() -> Result<(), Box<dyn std::erro
|
|||||||
|
|
||||||
/// Test 2: When bucket is configured with default SSE-KMS encryption, put_object should automatically apply encryption and use the specified KMS key
|
/// Test 2: When bucket is configured with default SSE-KMS encryption, put_object should automatically apply encryption and use the specified KMS key
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn test_bucket_default_sse_kms_put_object() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
async fn test_bucket_default_sse_kms_put_object() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||||
init_logging();
|
init_logging();
|
||||||
info!("Testing bucket default SSE-KMS encryption impact on put_object");
|
info!("Testing bucket default SSE-KMS encryption impact on put_object");
|
||||||
@@ -272,6 +275,7 @@ async fn test_bucket_default_sse_kms_put_object() -> Result<(), Box<dyn std::err
|
|||||||
|
|
||||||
/// Test 3: When bucket is configured with default encryption, create_multipart_upload should inherit the configuration
|
/// Test 3: When bucket is configured with default encryption, create_multipart_upload should inherit the configuration
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn test_bucket_default_sse_kms_multipart_crc32() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
async fn test_bucket_default_sse_kms_multipart_crc32() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||||
init_logging();
|
init_logging();
|
||||||
info!("Testing bucket default encryption impact on create_multipart_upload");
|
info!("Testing bucket default encryption impact on create_multipart_upload");
|
||||||
@@ -469,6 +473,7 @@ async fn test_bucket_default_sse_kms_multipart_crc32() -> Result<(), Box<dyn std
|
|||||||
|
|
||||||
/// Test 4: Explicitly specified encryption parameters in requests should override bucket default configuration
|
/// Test 4: Explicitly specified encryption parameters in requests should override bucket default configuration
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn test_explicit_encryption_overrides_bucket_default() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
async fn test_explicit_encryption_overrides_bucket_default() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||||
init_logging();
|
init_logging();
|
||||||
info!("Testing explicitly specified encryption parameters override bucket default configuration");
|
info!("Testing explicitly specified encryption parameters override bucket default configuration");
|
||||||
@@ -564,6 +569,7 @@ async fn test_explicit_encryption_overrides_bucket_default() -> Result<(), Box<d
|
|||||||
/// Test 5: Setting SSE-KMS without a specific key ID should auto-populate the
|
/// Test 5: Setting SSE-KMS without a specific key ID should auto-populate the
|
||||||
/// default KMS key ID so that GetBucketEncryption returns it (issue #3039).
|
/// default KMS key ID so that GetBucketEncryption returns it (issue #3039).
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn test_sse_kms_without_key_id_populates_default() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
async fn test_sse_kms_without_key_id_populates_default() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||||
init_logging();
|
init_logging();
|
||||||
info!("Testing SSE-KMS without explicit key ID populates default key");
|
info!("Testing SSE-KMS without explicit key ID populates default key");
|
||||||
|
|||||||
@@ -20,6 +20,7 @@ use super::common::{
|
|||||||
};
|
};
|
||||||
use aws_sdk_s3::primitives::ByteStream;
|
use aws_sdk_s3::primitives::ByteStream;
|
||||||
use aws_sdk_s3::types::{BucketVersioningStatus, ServerSideEncryption, VersioningConfiguration};
|
use aws_sdk_s3::types::{BucketVersioningStatus, ServerSideEncryption, VersioningConfiguration};
|
||||||
|
use serial_test::serial;
|
||||||
use std::error::Error;
|
use std::error::Error;
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
|
|
||||||
@@ -385,6 +386,7 @@ async fn assert_versioned_sse_kms_roundtrip_and_cleanup(
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn test_configured_local_kms_admin_and_versioned_cleanup() -> TestResult {
|
async fn test_configured_local_kms_admin_and_versioned_cleanup() -> TestResult {
|
||||||
let mut env = LocalKMSTestEnvironment::new().await?;
|
let mut env = LocalKMSTestEnvironment::new().await?;
|
||||||
env.base_env.start_rustfs_server(Vec::new()).await?;
|
env.base_env.start_rustfs_server(Vec::new()).await?;
|
||||||
@@ -432,6 +434,7 @@ async fn test_configured_local_kms_admin_and_versioned_cleanup() -> TestResult {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
#[ignore = "requires a Vault binary"]
|
#[ignore = "requires a Vault binary"]
|
||||||
async fn test_configured_vault_kms_admin_and_versioned_cleanup() -> TestResult {
|
async fn test_configured_vault_kms_admin_and_versioned_cleanup() -> TestResult {
|
||||||
let mut env = VaultTestEnvironment::new().await?;
|
let mut env = VaultTestEnvironment::new().await?;
|
||||||
|
|||||||
@@ -32,9 +32,11 @@ use aws_sdk_s3::types::{
|
|||||||
MetadataDirective, ServerSideEncryption, ServerSideEncryptionByDefault, ServerSideEncryptionConfiguration,
|
MetadataDirective, ServerSideEncryption, ServerSideEncryptionByDefault, ServerSideEncryptionConfiguration,
|
||||||
ServerSideEncryptionRule,
|
ServerSideEncryptionRule,
|
||||||
};
|
};
|
||||||
|
use serial_test::serial;
|
||||||
use tracing::info;
|
use tracing::info;
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn test_metadata_replace_self_copy_of_sse_object_stays_decryptable() {
|
async fn test_metadata_replace_self_copy_of_sse_object_stays_decryptable() {
|
||||||
init_logging();
|
init_logging();
|
||||||
info!("same-key CopyObject with REPLACE metadata must not re-key an SSE-S3 object");
|
info!("same-key CopyObject with REPLACE metadata must not re-key an SSE-S3 object");
|
||||||
@@ -134,6 +136,7 @@ async fn test_metadata_replace_self_copy_of_sse_object_stays_decryptable() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn test_metadata_replace_self_copy_dropping_sse_rewrites_plaintext() {
|
async fn test_metadata_replace_self_copy_dropping_sse_rewrites_plaintext() {
|
||||||
init_logging();
|
init_logging();
|
||||||
info!("same-key CopyObject that drops SSE must rewrite the data, not orphan the ciphertext");
|
info!("same-key CopyObject that drops SSE must rewrite the data, not orphan the ciphertext");
|
||||||
@@ -230,6 +233,7 @@ async fn test_metadata_replace_self_copy_dropping_sse_rewrites_plaintext() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn test_metadata_replace_self_copy_under_bucket_default_sse_stays_decryptable() {
|
async fn test_metadata_replace_self_copy_under_bucket_default_sse_stays_decryptable() {
|
||||||
init_logging();
|
init_logging();
|
||||||
info!("bucket default encryption must also keep a same-key copy off the metadata-only path");
|
info!("bucket default encryption must also keep a same-key copy off the metadata-only path");
|
||||||
|
|||||||
@@ -25,9 +25,11 @@ use super::common::{LocalKMSTestEnvironment, create_key_with_specific_id};
|
|||||||
use crate::common::init_logging;
|
use crate::common::init_logging;
|
||||||
use aws_sdk_s3::primitives::ByteStream;
|
use aws_sdk_s3::primitives::ByteStream;
|
||||||
use aws_sdk_s3::types::{BucketVersioningStatus, ServerSideEncryption, VersioningConfiguration};
|
use aws_sdk_s3::types::{BucketVersioningStatus, ServerSideEncryption, VersioningConfiguration};
|
||||||
|
use serial_test::serial;
|
||||||
use tracing::info;
|
use tracing::info;
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
async fn test_self_copy_of_historical_sse_s3_version_is_readable() {
|
async fn test_self_copy_of_historical_sse_s3_version_is_readable() {
|
||||||
init_logging();
|
init_logging();
|
||||||
info!("Issue #4238 (SSE): restoring an encrypted historical version must stay decryptable");
|
info!("Issue #4238 (SSE): restoring an encrypted historical version must stay decryptable");
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user