mirror of
https://github.com/rustfs/rustfs.git
synced 2026-11-18 14:05:27 +00:00
Compare commits
160 Commits
gw/T1.9-admin-a
...
1.0.6
| Author | SHA1 | Date | |
|---|---|---|---|
| de9009d882 | |||
| 94d54f8f55 | |||
| 01edeb0eab | |||
| dd2713687c | |||
| 4304e1b3ed | |||
| 5b331fad73 | |||
| 130a19de8f | |||
| 3e882ea0cf | |||
| 517ea37eb2 | |||
| 6d0832b992 | |||
| 34e6449ed9 | |||
| 0f9258594c | |||
| 33fc735ee2 | |||
| 509869cfc5 | |||
| 395876ccd0 | |||
| 6083fbbeea | |||
| 42d3267d88 | |||
| 1d35e381b7 | |||
| 0b109b4885 | |||
| 1dea00ffa7 | |||
| addf7a6219 | |||
| 420cf32b2a | |||
| 907410b2f5 | |||
| f10079f1a3 | |||
| a6c3016493 | |||
| 68fb52b752 | |||
| 16f618d41b | |||
| 565d262ede | |||
| 7b010ddd70 | |||
| 272b6587e0 | |||
| e3ed2cf2f4 | |||
| bf03b1117a | |||
| c3fe0f9649 | |||
| 6e31f6a183 | |||
| d66d144309 | |||
| 150836d707 | |||
| e4057b6746 | |||
| cfdd705b62 | |||
| a1d0fa41d0 | |||
| 7d4982f206 | |||
| 1a7fb2461d | |||
| e1b9d1cb94 | |||
| 85313fc06f | |||
| b5f3207378 | |||
| 5490e493ab | |||
| 134820c1e0 | |||
| 2b9217b6f9 | |||
| 70a46d0e8c | |||
| a910ef0dd0 | |||
| 430ade191f | |||
| c5b15b7279 | |||
| 2a6da8208b | |||
| 93b704420f | |||
| dfd4d3e213 | |||
| 1c1baa6b08 | |||
| 0147b8671d | |||
| 8cd751f9d1 | |||
| 5ea2991850 | |||
| 1556e05b8a | |||
| 32238a179c | |||
| 6b70584dde | |||
| d6210ac17b | |||
| 158aa276e9 | |||
| 3362421240 | |||
| 57df9181d0 | |||
| 371f5e4ffc | |||
| 8a5a2c9c93 | |||
| babfeaed00 | |||
| 736bf777b4 | |||
| 7babc7d621 | |||
| 4d46bd25f8 | |||
| ff5f491c5a | |||
| a8be876fd9 | |||
| fe3ada0689 | |||
| 926355ae1f | |||
| 7f5fa4b8f6 | |||
| a97c65895c | |||
| 73e7afa01a | |||
| e0f26a540d | |||
| 3b3cc018aa | |||
| f4f5635aab | |||
| 36ad5a6b87 | |||
| 6c14128072 | |||
| addc258a24 | |||
| 73ea527f55 | |||
| 55da14ad82 | |||
| 116a5299b2 | |||
| fbc54f7212 | |||
| 3aa492f5ee | |||
| 00a81b4048 | |||
| 20ef9ebf15 | |||
| cc08e0d11b | |||
| 69294523d5 | |||
| cd4d86f28a | |||
| 2f55cf69e1 | |||
| fff6e94378 | |||
| 41b13aeb8b | |||
| 88587a8f94 | |||
| 8c8b5da63f | |||
| 00a238f838 | |||
| 965d805a4f | |||
| 9d66e04ae3 | |||
| 12c58d25b5 | |||
| 1ac1d883f4 | |||
| 45ba33291a | |||
| 636eded277 | |||
| ed4099ebe1 | |||
| 3a1fac16e3 | |||
| b00e85f1a4 | |||
| ecbddc188a | |||
| 2d63931bba | |||
| d429c6f792 | |||
| e494dc2b4b | |||
| 2b7c927ee8 | |||
| 647846956f | |||
| 04690f7da5 | |||
| 3c0829049d | |||
| 890a463218 | |||
| 25f45f6309 | |||
| e3700614a1 | |||
| d4ab7704b3 | |||
| 422e724051 | |||
| e0bf54197d | |||
| b4af43c884 | |||
| 1cb4c0ebe2 | |||
| e5563e6004 | |||
| 0a72b29a34 | |||
| 51275cb7d8 | |||
| 3bf270b49f | |||
| d8d064bd18 | |||
| e5653ca055 | |||
| 6413a11889 | |||
| 9e4ea9fdd1 | |||
| 1a841fb7c7 | |||
| b8ee0301e9 | |||
| 0e5fe8c658 | |||
| e46877dd1c | |||
| 3fc610c8a2 | |||
| 4992034cc2 | |||
| b4c9bf9abb | |||
| 7d45685d70 | |||
| 76bab81aa6 | |||
| 72ff5dbbad | |||
| 8ece50ff8c | |||
| 7f13ebe2a5 | |||
| 696dc1496e | |||
| 68ee16bbbe | |||
| bc0706b2e3 | |||
| c6152954f6 | |||
| 0d2bf69e2d | |||
| 7e536843ea | |||
| 7672867476 | |||
| c8f7f02817 | |||
| dcbec68d45 | |||
| 589b499e3b | |||
| 69f8e364b0 | |||
| 191c95a469 | |||
| dacb003856 | |||
| c67f9fc377 | |||
| 4fcbf60f10 |
@@ -18,6 +18,7 @@ These implementation, style, and quality rules apply to all contributions, regar
|
||||
- Do not include agent, tool, contributor, account, or organization names in branch names.
|
||||
- Push to the user-requested remote or the repository's configured push remote. Do not hard-code or infer a remote from an account name.
|
||||
- Check free space before artifact-heavy builds, tests, coverage, or downloads. Re-check before a broad gate when space is tight.
|
||||
- Before handoff, remove the Cargo target directories and temporary worktrees the task created unless they stay intentionally or hold unmerged work. Identify build output by Cargo's markers (`.rustc_info.json`, `CACHEDIR.TAG`), not by directory name: `target` is also a source directory here.
|
||||
- Remove only task-owned temporary/build artifacts. Never delete another task's worktree or uncommitted data.
|
||||
- At handoff, mention disk or cleanup details only when they affected execution or artifacts/worktrees remain intentionally.
|
||||
- For Docker verification, ensure the tested image contains the intended binary. `buildx build` without `--load` leaves the image in the build cache; use the Makefile's loaded image and rebuild without cache only when stale artifacts are evidenced.
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"rollout": "enforced",
|
||||
"core_packages": ["rustfs", "rustfs-ecstore", "rustfs-filemeta", "rustfs-iam", "rustfs-policy"],
|
||||
"connect_packages": [
|
||||
"rustfs-credentials", "rustfs-config", "rustfs-common", "rustfs-crypto",
|
||||
"rustfs-iam", "rustfs-policy", "rustfs-kms", "rustfs-license", "rustfs-security-governance",
|
||||
"rustfs-utils", "rustfs-tls-runtime", "rustfs-trusted-proxies", "rustfs-filemeta",
|
||||
"rustfs-lock", "rustfs-obs", "rustfs-signer", "rustfs-io-metrics", "rustfs-s3-ops"
|
||||
],
|
||||
"full_paths": [
|
||||
"Cargo.toml", "Cargo.lock", "*/Cargo.toml", "rust-toolchain*", "rustfmt.toml",
|
||||
"clippy.toml", ".cargo/*", ".config/*", ".github/*", "Makefile",
|
||||
"scripts/*", "build.rs", "*.proto"
|
||||
]
|
||||
}
|
||||
@@ -1,2 +1,2 @@
|
||||
sha256-linux=eb33067d90ee66c0a97e8f3658e46c152e47f897d2fbfdc6cb5ad93cfa55414b
|
||||
sha256-darwin=eb33067d90ee66c0a97e8f3658e46c152e47f897d2fbfdc6cb5ad93cfa55414b
|
||||
sha256-linux=348fa5eed9ae58962c06b33283ff4f15122cff97bce9b4eb1e3ffcb227264b0b
|
||||
sha256-darwin=348fa5eed9ae58962c06b33283ff4f15122cff97bce9b4eb1e3ffcb227264b0b
|
||||
|
||||
@@ -1,2 +1,2 @@
|
||||
sha256-darwin=67251cb7d9978d38e899268cac4d7898297ff7e5b5c48c9c1546dc760b93568d
|
||||
sha256-linux=673da76c2e5710c4622fe2fa32d2b297730895769487fcf1152ee1c2366f32f4
|
||||
sha256-darwin=9901a4950099b94d3d2618524d540f1ed4bf6023044903886f98b6c4bf5f30fb
|
||||
sha256-linux=09fd5f48f5556c9fc576357222dccd7ebb0b90b790798f1834db2ec5d67b4952
|
||||
|
||||
@@ -1,2 +1,2 @@
|
||||
sha256-darwin=3814cdde8cacf4187e18188d342ed70931cc32e268e42e5bb438d565bb4d0f81
|
||||
sha256-linux=663025a80273579f95ae3f49af02c0afcbfce94df09701c360a13e607b7637db
|
||||
sha256-darwin=e84e04208f19dc0596b10d4d1b799ce43d6cc431415902f9c5458aa8232bc2d8
|
||||
sha256-linux=1f9c7add72c44d1c12ef4ac050bb7917431be657c0c90b29f70c3df56f8bbae3
|
||||
|
||||
@@ -1 +1 @@
|
||||
sha256=e9dc896774122a304e2b8b90f28c82d4d4e0af8ded36a82b147b30945ec994e3
|
||||
sha256=5bdd4c2eadbc41625ae3184fff34062e000d328108436259b2b7c4fe2531ecaf
|
||||
|
||||
@@ -90,6 +90,7 @@ test-wiring-check: ## Check tests stay registered and selected by their intended
|
||||
@echo "🧪 Checking test wiring..."
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/check_test_wiring.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/ci_gate.py --check-workflow
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/validation_registry.py check
|
||||
|
||||
.PHONY: log-analyzer-rules-check
|
||||
log-analyzer-rules-check: core-deps ## Check log-analyzer rule anchors still exist verbatim in source
|
||||
|
||||
@@ -28,6 +28,7 @@ script-tests: ## Run shell script tests
|
||||
./scripts/test_build_rustfs_options.sh
|
||||
./scripts/test_docker_runtime_timezone.sh
|
||||
./scripts/test_entrypoint_credentials.sh
|
||||
./scripts/test_entrypoint_directories.sh
|
||||
./scripts/test_internode_grpc_ab_bench.sh
|
||||
./scripts/test_object_batch_bench_enhanced.sh
|
||||
./scripts/test_package_service_scripts.sh
|
||||
@@ -53,18 +54,26 @@ script-tests: ## Run shell script tests
|
||||
./scripts/check_embedded_secrets.sh --self-test
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/check_test_wiring.py --self-test
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_e2e_binary.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_download_release_asset.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_inspect_btf_prototypes.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_release_build_workflow.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_release_glibc.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_upgrade_workflow.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_migration_gate_evidence.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/ci_gate.py --self-test
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/check_connect_release_boundary.py --self-test
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_ci_test_artifact.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_ci_test_shards.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_ci_merge_queue.py
|
||||
$(RUSTFS_PYTHON_BIN) -m unittest discover -s scripts/ci -p 'test_*.py'
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/check_security_coverage.py --self-test
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/check_scheduled_validation_freshness.py --self-test
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_security_workflow.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_nightly_candidate.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_functional_chain.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_functional_chain_health.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_repro_rejoin_readiness_docker.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_validation_registry.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_inspect_validation_runner.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_retain_validation_evidence.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_ci_timing_report.py
|
||||
|
||||
+32
-6
@@ -110,7 +110,7 @@ threads-required = "num-test-threads"
|
||||
# Reserve capacity so sibling disk-heavy tests cannot consume its 2s
|
||||
# registration budget; retain the queued backup and deadlock assertions.
|
||||
[[profile.default.overrides]]
|
||||
filter = 'package(rustfs) & test(=admin::handlers::table_catalog::tests::cold_strong_registration_completes_while_other_bucket_backup_waits)'
|
||||
filter = 'package(rustfs) & test(=admin::handlers::table_catalog::tests::cold_object_registration_completes_while_other_bucket_backup_waits)'
|
||||
threads-required = "num-test-threads"
|
||||
|
||||
[[profile.default.overrides]]
|
||||
@@ -234,7 +234,7 @@ test-group = 'embedded-test-ports'
|
||||
# processes cannot exhaust those budgets. Keep deadlines and assertions intact.
|
||||
# The short HTTP fixtures are tracked in rustfs/backlog#2714.
|
||||
[[profile.default.overrides]]
|
||||
filter = 'package(rustfs) & (binary(/^connect_registration$/) | binary(connect_inventory) | binary(connect_perf_drive) | (binary(connect_perf_client) & test(=configured_proxy_and_direct_endpoint_failures_are_distinct)) | (binary(connect_perf_object) & test(/^(real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put|cleanup_uses_a_budget_independent_of_the_measurement_window|oversized_response_is_rejected_without_buffering_it)$/)))'
|
||||
filter = 'package(rustfs) & (binary(/^connect_registration$/) | binary(connect_inventory) | binary(connect_perf_drive) | (binary(connect_perf_client) & test(=configured_proxy_and_direct_endpoint_failures_are_distinct)) | (binary(connect_perf_object) & test(/^(real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put|cleanup_uses_a_budget_independent_of_the_measurement_window|cleanup_accepts_missing_object_and_still_deletes_the_bucket|cleanup_diagnostics_identify_each_disconnected_transport_stage|cleanup_diagnostics_identify_each_failed_http_stage_without_private_context|cleanup_diagnostics_identify_timeouts_without_resetting_the_shared_deadline|oversized_response_is_rejected_without_buffering_it)$/)))'
|
||||
threads-required = "num-test-threads"
|
||||
|
||||
# This cancellation probe hashes the serving executable before acquiring its
|
||||
@@ -410,7 +410,7 @@ threads-required = "num-test-threads"
|
||||
# Reserve capacity so sibling disk-heavy tests cannot consume its 2s
|
||||
# registration budget; retain the queued backup and deadlock assertions.
|
||||
[[profile.ci.overrides]]
|
||||
filter = 'package(rustfs) & test(=admin::handlers::table_catalog::tests::cold_strong_registration_completes_while_other_bucket_backup_waits)'
|
||||
filter = 'package(rustfs) & test(=admin::handlers::table_catalog::tests::cold_object_registration_completes_while_other_bucket_backup_waits)'
|
||||
threads-required = "num-test-threads"
|
||||
|
||||
[[profile.ci.overrides]]
|
||||
@@ -548,7 +548,7 @@ filter = 'package(rustfs) & (binary(/^embedded.*_test$/) | binary(admin_diagnost
|
||||
test-group = 'embedded-test-ports'
|
||||
|
||||
[[profile.ci.overrides]]
|
||||
filter = 'package(rustfs) & (binary(/^connect_registration$/) | binary(connect_inventory) | binary(connect_perf_drive) | (binary(connect_perf_client) & test(=configured_proxy_and_direct_endpoint_failures_are_distinct)) | (binary(connect_perf_object) & test(/^(real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put|cleanup_uses_a_budget_independent_of_the_measurement_window|oversized_response_is_rejected_without_buffering_it)$/)))'
|
||||
filter = 'package(rustfs) & (binary(/^connect_registration$/) | binary(connect_inventory) | binary(connect_perf_drive) | (binary(connect_perf_client) & test(=configured_proxy_and_direct_endpoint_failures_are_distinct)) | (binary(connect_perf_object) & test(/^(real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put|cleanup_uses_a_budget_independent_of_the_measurement_window|cleanup_accepts_missing_object_and_still_deletes_the_bucket|cleanup_diagnostics_identify_each_disconnected_transport_stage|cleanup_diagnostics_identify_each_failed_http_stage_without_private_context|cleanup_diagnostics_identify_timeouts_without_resetting_the_shared_deadline|oversized_response_is_rejected_without_buffering_it)$/)))'
|
||||
threads-required = "num-test-threads"
|
||||
|
||||
# Preserve the cancellation probe's startup budget and assertions in CI too.
|
||||
@@ -701,7 +701,7 @@ test-group = 'ecstore-serial-flaky'
|
||||
[profile.e2e-smoke]
|
||||
default-filter = """
|
||||
package(e2e_test) & (
|
||||
test(/^(delete_marker_migration_semantics|version_id_regression|list_objects_v2_pagination|list_object_versions_regression|list_objects_duplicates|list_buckets_double_slash|list_buckets_auth|list_buckets_iam_filter|leading_slash_key|special_chars|create_bucket_region|delete_objects_versioning|head_object_consistency|head_object_range|copy_object_metadata|copy_object_tagging|copy_source_invalid_date|content_encoding|compression|multipart_storage_class|storage_class_capability|ssec_copy|anonymous_access|bucket_policy_check|presigned_negative|negative_sigv4|admin_auth|notification_webhook|tls_hot_reload|console_smoke|admin_iam_crud|admin_pools|sts_query_compat|data_usage)_test::|^fake_s3_target::/)
|
||||
test(/^(delete_marker_migration_semantics|version_id_regression|list_objects_v2_pagination|list_object_versions_regression|list_objects_duplicates|list_buckets_double_slash|list_buckets_auth|list_buckets_iam_filter|leading_slash_key|special_chars|create_bucket_region|delete_objects_versioning|head_object_consistency|head_object_range|copy_object_metadata|copy_object_tagging|copy_source_invalid_date|content_encoding|compression|multipart_storage_class|multipart_upload_id|storage_class_capability|ssec_copy|anonymous_access|bucket_policy_check|presigned_negative|negative_sigv4|admin_auth|notification_webhook|tls_hot_reload|console_smoke|admin_iam_crud|admin_pools|sts_query_compat|data_usage)_test::|^fake_s3_target::/)
|
||||
| test(/^replication_extension_test::(test_replication_check_succeeds_with_remote_target|test_replication_check_rejects_target_without_object_lock|test_set_remote_target_rejects_unversioned_source_bucket|test_replication_check_rejects_unversioned_source_bucket|test_replication_check_rejects_missing_replication_config|test_replication_check_rejects_invalid_bucket|test_set_remote_target_rejects_same_bucket_on_same_deployment|test_set_remote_target_rejects_unversioned_target_bucket|test_set_remote_target_update_requires_arn|test_set_remote_target_update_rejects_missing_target|test_set_remote_target_rejects_invalid_target_url|test_set_remote_target_rejects_self_signed_https_target_without_skip_tls_verify|test_set_remote_target_rejects_private_ca_https_target_without_ca_cert_pem|test_list_remote_targets_rejects_empty_bucket|test_list_remote_targets_rejects_invalid_bucket|test_remove_remote_target_rejects_missing_target|test_remove_remote_target_rejects_missing_arn|test_remove_remote_target_rejects_invalid_bucket|test_remove_remote_target_rejects_target_used_by_replication|test_delete_bucket_replication_removes_remote_target)$/)
|
||||
| test(/^reliant::lifecycle::/)
|
||||
| test(/^conditional_delete_objects_test::/)
|
||||
@@ -827,7 +827,7 @@ slow-timeout = { period = "120s", terminate-after = 12, grace-period = "10s" }
|
||||
# ---------------------------------------------------------------------------
|
||||
# e2e-distributed profile — 4-node 4-disk Actions suite
|
||||
# ---------------------------------------------------------------------------
|
||||
# Storage-sensitive PR / nightly / dispatch lane owned by
|
||||
# Nightly and manual-dispatch lane owned by
|
||||
# .github/workflows/e2e-distributed.yml.
|
||||
# Each case starts four rustfs processes (and for site replication, two
|
||||
# clusters). Upgrade cases also require RUSTFS_UPGRADE_SOURCE_BINARY.
|
||||
@@ -960,6 +960,12 @@ test-group = 'e2e-inline-boundaries'
|
||||
filter = 'package(e2e_test) & test(/^heal_erasure_disk_rebuild_test::/)'
|
||||
test-group = 'e2e-cluster-nightly'
|
||||
|
||||
# The multi-pool crash fixture seeds three nodes and 36 drives. Reserve this
|
||||
# nextest run's workers so other multi-node tests do not starve its deadline.
|
||||
[[profile.e2e-full.overrides]]
|
||||
filter = 'package(e2e_test) & test(=heal_erasure_disk_rebuild_test::tests::test_cluster_root_heal_recovers_ec84_shards_across_multi_pool_after_background_target_crash)'
|
||||
threads-required = "num-test-threads"
|
||||
|
||||
[[profile.e2e-full.overrides]]
|
||||
filter = 'package(e2e_test) & (test(/^kms::kms_vault_test::/) | test(/^kms::kms_rekey_sweep_test::/) | test(/^kms::configured_roundtrip_test::test_configured_vault_kms_admin_and_versioned_cleanup$/))'
|
||||
test-group = 'e2e-vault'
|
||||
@@ -969,3 +975,23 @@ test-group = 'e2e-vault'
|
||||
[[profile.e2e-full.overrides]]
|
||||
filter = 'package(e2e_test) & (test(/^list_objects_v2_pagination_test::tests::(test_list_objects_v2_delimiter_small_page_traverses_all|test_list_objects_v2_max_keys_above_limit_returns_token|test_list_objects_v2_maxkeys_above_limit_with_delimiter)$/) | test(=reliant::tiering::test_hermetic_transition_restore_failure_expiry_and_retry))'
|
||||
threads-required = "num-test-threads"
|
||||
|
||||
# The mc fixture uploads hundreds of objects concurrently before measuring listing.
|
||||
# Reserve the runner so other server fixtures do not starve its disk operations.
|
||||
[[profile.e2e-full.overrides]]
|
||||
filter = 'package(e2e_test) & test(=mc_mirror_small_bucket_test::test_mc_mirror_small_bucket_completes_without_list_timeout)'
|
||||
threads-required = "num-test-threads"
|
||||
|
||||
# Fixed, small contract for every executable server change. These existing
|
||||
# scenarios exercise write/read, anonymous denial and retained bytes on restart.
|
||||
[profile.core-contract]
|
||||
default-filter = '''
|
||||
package(e2e_test) & test(/^(anonymous_access_test::test_anonymous_access_denied_when_restrict_public_buckets_enabled|head_object_consistency_test::head_object_consistency_after_write_and_multipart_and_presigned_head|storage_class_capability_test::tests::historical_label_only_metadata_is_standard_without_hiding_a_real_transition_tier)$/)
|
||||
'''
|
||||
retries = 0
|
||||
fail-fast = false
|
||||
test-threads = 1
|
||||
slow-timeout = { period = "60s", terminate-after = 3 }
|
||||
|
||||
[profile.core-contract.junit]
|
||||
path = "junit.xml"
|
||||
|
||||
+33
-3
@@ -22,9 +22,9 @@ The following files in the project root are essential for Docker operations:
|
||||
| File | Description | Usage |
|
||||
| :--- | :--- | :--- |
|
||||
| **`docker-buildx.sh`** | **Multi-Arch Build Script**<br>Automates building and pushing Docker images for `amd64` and `arm64`. Supports release and dev channels. | `./docker-buildx.sh --push` |
|
||||
| **`Dockerfile`** | **Production Image (Alpine)**<br>Lightweight image using musl libc. Downloads pre-built binaries from GitHub Releases. | `docker build -t rustfs:latest .` |
|
||||
| **`Dockerfile.glibc`** | **Production Image (Ubuntu)**<br>Standard image using glibc. Useful if you need specific dynamic libraries. | `docker build -f Dockerfile.glibc .` |
|
||||
| **`Dockerfile.source`** | **Development Image**<br>Builds RustFS from source code. Includes build tools. Ideal for local development and CI. | `docker build -f Dockerfile.source .` |
|
||||
| **`Dockerfile`** | **Production Image (Alpine)**<br>Lightweight image using musl libc. Downloads verified RustFS and `rc` binaries from GitHub Releases and bundles `/etc/rustfs/bootstrap`. | `docker build -t rustfs:latest .` |
|
||||
| **`Dockerfile.glibc`** | **Production Image (Ubuntu)**<br>Standard image using glibc. Includes the bundled bootstrap configuration. | `docker build -f Dockerfile.glibc .` |
|
||||
| **`Dockerfile.source`** | **Development Image**<br>Builds RustFS from source code and bundles `rc` and the default bootstrap configuration. | `docker build -f Dockerfile.source .` |
|
||||
|
||||
### Docker Compose Configurations
|
||||
|
||||
@@ -110,6 +110,36 @@ docker compose up -d
|
||||
|
||||
---
|
||||
|
||||
## Docker Compose environment values
|
||||
|
||||
Use map syntax when quoting environment values in YAML:
|
||||
|
||||
```yaml
|
||||
services:
|
||||
rustfs:
|
||||
image: rustfs/rustfs:latest
|
||||
ports:
|
||||
- "9000:9000"
|
||||
- "9001:9001"
|
||||
volumes:
|
||||
- ./data:/data
|
||||
- ./logs:/var/log/rustfs
|
||||
environment:
|
||||
RUSTFS_ADDRESS: ":9000"
|
||||
RUSTFS_CONSOLE_ADDRESS: ":9001"
|
||||
RUSTFS_CONSOLE_ENABLE: "true"
|
||||
RUSTFS_OBS_LOGGER_LEVEL: error
|
||||
RUSTFS_OBS_LOG_DIRECTORY: "/var/log/rustfs/"
|
||||
RUSTFS_ACCESS_KEY: "${RUSTFS_ACCESS_KEY:?Set RUSTFS_ACCESS_KEY}"
|
||||
RUSTFS_SECRET_KEY: "${RUSTFS_SECRET_KEY:?Set RUSTFS_SECRET_KEY}"
|
||||
```
|
||||
|
||||
Create the host directories and grant the container user write access before starting this example; official images run as UID/GID `10001:10001`. Set both credential variables to your own values, then run `docker compose config` to inspect the resolved configuration locally before `docker compose up -d`. The resolved output includes credentials, so do not share it publicly.
|
||||
|
||||
With list syntax, use `- RUSTFS_OBS_LOG_DIRECTORY=/var/log/rustfs/` or quote the whole item as `- "RUSTFS_OBS_LOG_DIRECTORY=/var/log/rustfs/"`. Writing `- RUSTFS_OBS_LOG_DIRECTORY="/var/log/rustfs/"` passes the inner quotes literally, making this a relative path beginning with `"`, which can fail with `mkdir: cannot create directory '"': Permission denied`. This also applies to addresses and credentials: fix every affected list item, rather than only the log directory. The entrypoint preserves configured values, including literal quotes in valid paths and secrets.
|
||||
|
||||
---
|
||||
|
||||
## 🔧 Common Operations
|
||||
|
||||
### Cleaning Up
|
||||
|
||||
@@ -28,6 +28,15 @@ processors:
|
||||
check_interval: 1s
|
||||
limit_mib: 1024
|
||||
spike_limit_mib: 256
|
||||
transform/metrics:
|
||||
error_mode: ignore
|
||||
metric_statements:
|
||||
# RustFS process metrics already expose process_pid as a metric label.
|
||||
# Remove only the duplicate datapoint label and keep the shared resource
|
||||
# process.pid available to sibling metrics and lock histograms.
|
||||
- context: datapoint
|
||||
statements:
|
||||
- delete_key(attributes, "process_pid") where IsMatch(metric.name, "^rustfs_system_process_")
|
||||
transform/logs:
|
||||
error_mode: ignore
|
||||
log_statements:
|
||||
@@ -112,7 +121,7 @@ service:
|
||||
exporters: [ otlp/tempo, otlp/jaeger ]
|
||||
metrics:
|
||||
receivers: [ otlp ]
|
||||
processors: [ batch ]
|
||||
processors: [ transform/metrics, memory_limiter, batch ]
|
||||
exporters: [ prometheus ]
|
||||
logs:
|
||||
receivers: [ otlp ]
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
# syntax=docker/dockerfile:1.6
|
||||
# Copyright 2026 RustFS Team
|
||||
# Licensed under the Apache License, Version 2.0.
|
||||
|
||||
# A compatible native runtime with Python 3.11+, getent and stat is required by
|
||||
# the isolated fixture's staging/provenance checks. This is a test image.
|
||||
ARG RUNTIME_BASE=python:3.13-slim-trixie
|
||||
FROM ${RUNTIME_BASE}
|
||||
|
||||
ARG SOURCE_REVISION
|
||||
ARG BINARY_SHA256
|
||||
ARG BUILD_PROFILE=dev
|
||||
COPY rustfs /usr/bin/rustfs
|
||||
RUN python3 - "$SOURCE_REVISION" "$BINARY_SHA256" "$BUILD_PROFILE" <<'PY'
|
||||
import hashlib
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import sys
|
||||
|
||||
source, digest, profile = sys.argv[1:]
|
||||
if not re.fullmatch(r"[0-9a-f]{40}", source):
|
||||
raise SystemExit("invalid source revision")
|
||||
if not re.fullmatch(r"[0-9a-f]{64}", digest):
|
||||
raise SystemExit("invalid binary digest")
|
||||
if profile not in ("dev", "release"):
|
||||
raise SystemExit("invalid build profile")
|
||||
with open("/usr/bin/rustfs", "rb") as binary:
|
||||
if hashlib.file_digest(binary, "sha256").hexdigest() != digest:
|
||||
raise SystemExit("binary digest mismatch")
|
||||
for tool in ("getent", "stat"):
|
||||
if shutil.which(tool) is None:
|
||||
raise SystemExit(f"missing runtime tool: {tool}")
|
||||
os.chmod("/usr/bin/rustfs", 0o755)
|
||||
PY
|
||||
LABEL io.rustfs.repro.source_revision="${SOURCE_REVISION}" \
|
||||
io.rustfs.repro.binary_sha256="${BINARY_SHA256}" \
|
||||
io.rustfs.repro.build_profile="${BUILD_PROFILE}"
|
||||
USER 10001:10001
|
||||
ENTRYPOINT ["/usr/bin/rustfs"]
|
||||
@@ -13,7 +13,7 @@
|
||||
# limitations under the License.
|
||||
|
||||
name: Quick Checks
|
||||
description: Run the shared compile-free RustFS quality checks.
|
||||
description: Run the shared RustFS quality checks.
|
||||
|
||||
runs:
|
||||
using: composite
|
||||
@@ -24,19 +24,30 @@ runs:
|
||||
tool: |
|
||||
ripgrep@15.2.0
|
||||
shellcheck@0.11.0
|
||||
cargo-nextest@0.9.138
|
||||
|
||||
- name: Install actionlint
|
||||
shell: bash
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
ASSET_DOWNLOADER: ${{ github.action_path }}/../setup/download_release_asset.py
|
||||
run: |
|
||||
actionlint_dir="$(mktemp -d "${RUNNER_TEMP}/actionlint.XXXXXX")"
|
||||
curl --fail --location --silent --show-error \
|
||||
--output "$actionlint_dir/actionlint.tar.gz" \
|
||||
https://github.com/rhysd/actionlint/releases/download/v1.7.12/actionlint_1.7.12_linux_amd64.tar.gz
|
||||
python3 "$ASSET_DOWNLOADER" --repository rhysd/actionlint --tag v1.7.12 --asset actionlint_1.7.12_linux_amd64.tar.gz --output "$actionlint_dir/actionlint.tar.gz" --sha256 8aca8db96f1b94770f1b0d72b6dddcb1ebb8123cb3712530b08cc387b349a3d8
|
||||
echo "8aca8db96f1b94770f1b0d72b6dddcb1ebb8123cb3712530b08cc387b349a3d8 $actionlint_dir/actionlint.tar.gz" | sha256sum --check --status
|
||||
tar -xzf "$actionlint_dir/actionlint.tar.gz" -C "$actionlint_dir" actionlint
|
||||
rm "$actionlint_dir/actionlint.tar.gz"
|
||||
echo "$actionlint_dir" >> "$GITHUB_PATH"
|
||||
|
||||
- name: Install Python
|
||||
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- name: Install workflow parser
|
||||
shell: bash
|
||||
run: python3 -m pip install --disable-pip-version-check PyYAML==6.0.3
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
|
||||
with:
|
||||
@@ -104,7 +115,10 @@ runs:
|
||||
|
||||
- name: Check test wiring
|
||||
shell: bash
|
||||
run: python3 ./scripts/check_test_wiring.py
|
||||
run: |
|
||||
python3 ./scripts/check_test_wiring.py
|
||||
python3 ./scripts/ci_gate.py --check-workflow
|
||||
python3 ./scripts/validation_registry.py check
|
||||
|
||||
- name: Check no planning docs committed
|
||||
shell: bash
|
||||
|
||||
@@ -0,0 +1,107 @@
|
||||
name: Runner capability and build pilot
|
||||
description: Inspect an existing runner and measure isolated fixed-source cold/warm builds
|
||||
inputs:
|
||||
workload:
|
||||
description: probe, server or workspace-tests
|
||||
required: true
|
||||
source-sha:
|
||||
description: Exact merged main commit for builds
|
||||
default: ""
|
||||
toolchain:
|
||||
description: Exact Rust version for builds
|
||||
default: ""
|
||||
scratch-root:
|
||||
description: Existing absolute scratch root
|
||||
default: ""
|
||||
build-jobs:
|
||||
description: Cargo concurrency
|
||||
default: "2"
|
||||
local-ssd:
|
||||
description: Require operator storage attestation
|
||||
default: "false"
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Validate pilot input and inspect existing capabilities
|
||||
shell: bash
|
||||
env:
|
||||
PILOT_WORKLOAD: ${{ inputs.workload }}
|
||||
PILOT_SHA: ${{ inputs.source-sha }}
|
||||
PILOT_TOOLCHAIN: ${{ inputs.toolchain }}
|
||||
PILOT_SCRATCH: ${{ inputs.scratch-root }}
|
||||
PILOT_JOBS: ${{ inputs.build-jobs }}
|
||||
PILOT_SSD: ${{ inputs.local-ssd }}
|
||||
run: |
|
||||
[[ "$PILOT_WORKLOAD" =~ ^(probe|server|workspace-tests)$ ]]
|
||||
[[ "$PILOT_JOBS" =~ ^([1-9]|1[0-6])$ ]]
|
||||
if [[ "$PILOT_WORKLOAD" != probe ]]; then
|
||||
[[ "$PILOT_SHA" =~ ^[a-f0-9]{40}$ ]]
|
||||
[[ "$PILOT_TOOLCHAIN" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]
|
||||
fi
|
||||
scratch="${PILOT_SCRATCH:-$RUNNER_TEMP}"
|
||||
[[ "$scratch" == /* && -d "$scratch" ]]
|
||||
minimum_disk=1
|
||||
minimum_memory=0
|
||||
if [[ "$PILOT_WORKLOAD" == server ]]; then minimum_disk=24; minimum_memory=8; fi
|
||||
if [[ "$PILOT_WORKLOAD" == workspace-tests ]]; then minimum_disk=90; minimum_memory=16; fi
|
||||
extra=()
|
||||
if [[ "$PILOT_SSD" == true ]]; then extra+=(--local-ssd-attestation /etc/rustfs-runner/storage.json); fi
|
||||
python3 scripts/ci/runner_pilot.py --root "$scratch" \
|
||||
--output "$RUNNER_TEMP/runner-pilot-evidence" \
|
||||
--min-free-gib "$minimum_disk" --min-memory-gib "$minimum_memory" --architecture x86_64 "${extra[@]}"
|
||||
- name: Checkout the trusted baseline history
|
||||
if: inputs.workload != 'probe'
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
ref: main
|
||||
path: .runner-pilot-source
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
- name: Bind the build to a merged main commit
|
||||
if: inputs.workload != 'probe'
|
||||
shell: bash
|
||||
env:
|
||||
PILOT_SHA: ${{ inputs.source-sha }}
|
||||
run: |
|
||||
git -C .runner-pilot-source merge-base --is-ancestor "$PILOT_SHA" HEAD
|
||||
git -C .runner-pilot-source checkout --detach "$PILOT_SHA"
|
||||
- name: Install build dependencies
|
||||
if: inputs.workload != 'probe'
|
||||
shell: bash
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y build-essential pkg-config libssl-dev cmake clang time
|
||||
- name: Install the fixed protobuf compiler
|
||||
if: inputs.workload != 'probe'
|
||||
uses: rustfs/setup-protoc@a3705324d8f9bf5b6c3573fb6cf8ae421db55dd6 # v3.0.1
|
||||
with:
|
||||
version: "35.1"
|
||||
repo-token: ${{ github.token }}
|
||||
- name: Install the fixed flatbuffer compiler
|
||||
if: inputs.workload != 'probe'
|
||||
uses: Nugine/setup-flatc@698800de72a96bfb22cf60431dc21a2ff9a7e07b # v1
|
||||
with:
|
||||
version: "25.12.19"
|
||||
- name: Install the requested exact Rust toolchain
|
||||
if: inputs.workload != 'probe'
|
||||
uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
|
||||
with:
|
||||
toolchain: ${{ inputs.toolchain }}
|
||||
- name: Measure cold and warm builds without shared target caches
|
||||
if: inputs.workload != 'probe'
|
||||
shell: bash
|
||||
env:
|
||||
PILOT_WORKLOAD: ${{ inputs.workload }}
|
||||
PILOT_SHA: ${{ inputs.source-sha }}
|
||||
PILOT_TOOLCHAIN: ${{ inputs.toolchain }}
|
||||
PILOT_SCRATCH: ${{ inputs.scratch-root }}
|
||||
PILOT_JOBS: ${{ inputs.build-jobs }}
|
||||
PILOT_SSD: ${{ inputs.local-ssd }}
|
||||
run: |
|
||||
extra=()
|
||||
if [[ "$PILOT_SSD" == true ]]; then extra+=(--local-ssd-attestation /etc/rustfs-runner/storage.json); fi
|
||||
python3 scripts/ci/runner_pilot.py --root "${PILOT_SCRATCH:-$RUNNER_TEMP}" \
|
||||
--output "$RUNNER_TEMP/runner-pilot-evidence" \
|
||||
--source .runner-pilot-source --source-sha "$PILOT_SHA" \
|
||||
--toolchain "$PILOT_TOOLCHAIN" --benchmark "$PILOT_WORKLOAD" \
|
||||
--jobs "$PILOT_JOBS" --timeout-seconds 3600 "${extra[@]}"
|
||||
@@ -66,10 +66,7 @@ inputs:
|
||||
runs:
|
||||
using: "composite"
|
||||
steps:
|
||||
# protobuf-compiler is deliberately absent: the setup-protoc step below
|
||||
# installs 35.1 into the tool cache and prepends it to PATH, so the apt
|
||||
# build (older, and never version-matched) was shadowed on every run and
|
||||
# simply never used.
|
||||
# Use the pinned code generators rather than Ubuntu's older packages.
|
||||
- name: Install system dependencies (Ubuntu)
|
||||
if: runner.os == 'Linux'
|
||||
shell: bash
|
||||
@@ -79,7 +76,9 @@ runs:
|
||||
build-essential \
|
||||
pkg-config \
|
||||
libssl-dev \
|
||||
ripgrep
|
||||
ripgrep \
|
||||
python3 \
|
||||
gh
|
||||
|
||||
# musl-gcc is needed by the native musl release leg, and zip/unzip by the
|
||||
# release packaging steps. No CI test lane touches any of them.
|
||||
@@ -88,13 +87,32 @@ runs:
|
||||
shell: bash
|
||||
run: sudo apt-get install -y musl-tools zip unzip
|
||||
|
||||
- name: Install pinned code generators through GitHub API (Linux x64)
|
||||
if: runner.os == 'Linux' && runner.arch == 'X64'
|
||||
shell: bash
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
ASSET_DOWNLOADER: ${{ github.action_path }}/download_release_asset.py
|
||||
run: |
|
||||
python3 "$ASSET_DOWNLOADER" --repository protocolbuffers/protobuf --tag v35.1 --asset protoc-35.1-linux-x86_64.zip --output "$RUNNER_TEMP/protoc.zip"
|
||||
python3 "$ASSET_DOWNLOADER" --repository google/flatbuffers --tag v25.12.19 --asset 'Linux.flatc.binary.g++-13.zip' --output "$RUNNER_TEMP/flatc.zip"
|
||||
python3 -m zipfile -e "$RUNNER_TEMP/protoc.zip" "$RUNNER_TEMP/protoc"
|
||||
python3 -m zipfile -e "$RUNNER_TEMP/flatc.zip" "$RUNNER_TEMP/flatc"
|
||||
chmod +x "$RUNNER_TEMP/protoc/bin/protoc" "$RUNNER_TEMP/flatc/flatc"
|
||||
"$RUNNER_TEMP/protoc/bin/protoc" --version
|
||||
"$RUNNER_TEMP/flatc/flatc" --version
|
||||
echo "$RUNNER_TEMP/protoc/bin" >> "$GITHUB_PATH"
|
||||
echo "$RUNNER_TEMP/flatc" >> "$GITHUB_PATH"
|
||||
|
||||
- name: Install protoc
|
||||
if: runner.os != 'Linux' || runner.arch != 'X64'
|
||||
uses: rustfs/setup-protoc@a3705324d8f9bf5b6c3573fb6cf8ae421db55dd6 # v3.0.1
|
||||
with:
|
||||
version: "35.1"
|
||||
repo-token: ${{ github.token }}
|
||||
|
||||
- name: Install flatc
|
||||
if: runner.os != 'Linux' || runner.arch != 'X64'
|
||||
uses: Nugine/setup-flatc@698800de72a96bfb22cf60431dc21a2ff9a7e07b # v1
|
||||
with:
|
||||
version: "25.12.19"
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Download a named GitHub release asset through its API and verify its digest."""
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from urllib.parse import quote
|
||||
|
||||
|
||||
def download(repository, tag, name, output, expected_sha256=None):
|
||||
if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*/[A-Za-z0-9][A-Za-z0-9_.-]*", repository):
|
||||
raise ValueError("invalid release repository")
|
||||
if not tag or not name or "/" in name or "\\" in name:
|
||||
raise ValueError("invalid release tag or asset name")
|
||||
if expected_sha256 is not None and not re.fullmatch(r"[0-9a-f]{64}", expected_sha256):
|
||||
raise ValueError("invalid pinned asset digest")
|
||||
result = subprocess.run(
|
||||
["gh", "api", "--hostname", "github.com", f"repos/{repository}/releases/tags/{quote(tag, safe='')}"],
|
||||
check=True, capture_output=True, text=True, timeout=60,
|
||||
)
|
||||
release = json.loads(result.stdout)
|
||||
if not isinstance(release, dict) or release.get("tag_name") != tag:
|
||||
raise ValueError("release metadata does not match the requested tag")
|
||||
assets = release.get("assets")
|
||||
if not isinstance(assets, list) or any(not isinstance(asset, dict) for asset in assets):
|
||||
raise ValueError("invalid release asset inventory")
|
||||
matches = [asset for asset in assets if asset.get("name") == name]
|
||||
if len(matches) != 1:
|
||||
raise ValueError("release must contain exactly one named asset")
|
||||
asset = matches[0]
|
||||
asset_id, size, digest = asset.get("id"), asset.get("size"), asset.get("digest")
|
||||
if type(asset_id) is not int or asset_id <= 0 or type(size) is not int or not 0 < size <= 512 * 1024 * 1024:
|
||||
raise ValueError("invalid release asset identity or size")
|
||||
if not isinstance(digest, str) or not re.fullmatch(r"sha256:[0-9a-f]{64}", digest):
|
||||
raise ValueError("release asset has no valid SHA-256 digest")
|
||||
sha256 = digest.removeprefix("sha256:")
|
||||
if expected_sha256 is not None and expected_sha256 != sha256:
|
||||
raise ValueError("release digest differs from the pinned digest")
|
||||
output = Path(output)
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
temporary = None
|
||||
try:
|
||||
with tempfile.NamedTemporaryFile(dir=output.parent, prefix=".release-asset-", delete=False) as stream:
|
||||
temporary = Path(stream.name)
|
||||
# gh authenticates to the API; its HTTP client strips credentials on
|
||||
# cross-host redirects. No browser URL or token is supplied in argv.
|
||||
subprocess.run(
|
||||
["gh", "api", "--hostname", "github.com", "--header", "Accept: application/octet-stream", f"repos/{repository}/releases/assets/{asset_id}"],
|
||||
check=True, stdout=stream, timeout=300,
|
||||
)
|
||||
with temporary.open("rb") as stream:
|
||||
actual = hashlib.file_digest(stream, "sha256").hexdigest()
|
||||
if temporary.stat().st_size != size or actual != sha256:
|
||||
raise ValueError("downloaded release asset failed size or SHA-256 verification")
|
||||
temporary.replace(output)
|
||||
print(f"Verified release asset: {repository} {tag} {name} sha256={actual}")
|
||||
finally:
|
||||
if temporary is not None:
|
||||
temporary.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--repository", required=True)
|
||||
parser.add_argument("--tag", required=True)
|
||||
parser.add_argument("--asset", required=True)
|
||||
parser.add_argument("--output", required=True, type=Path)
|
||||
parser.add_argument("--sha256")
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
download(args.repository, args.tag, args.asset, args.output, args.sha256)
|
||||
except (ValueError, KeyError, TypeError, json.JSONDecodeError, subprocess.SubprocessError, OSError) as error:
|
||||
print(f"Release asset download failed: {error}", file=sys.stderr)
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,16 @@
|
||||
name: Verification runtime
|
||||
description: Install the runtime for prebuilt test archives without a target cache
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Install runtime tools
|
||||
shell: bash
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y libssl-dev ripgrep
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
|
||||
- name: Install cargo-nextest
|
||||
uses: taiki-e/install-action@96c7780c1d8a2b8723e12031def873a434d39d8d # nextest
|
||||
with:
|
||||
tool: cargo-nextest@0.9.138
|
||||
@@ -32,6 +32,8 @@ updates:
|
||||
- "overtrue"
|
||||
- "majinghe"
|
||||
ignore:
|
||||
- dependency-name: "pulsar"
|
||||
versions: [ "6.9.x" ]
|
||||
- dependency-name: "object_store"
|
||||
versions: [ "0.13.x" ]
|
||||
- dependency-name: "libunftp"
|
||||
|
||||
+1731
-25
File diff suppressed because it is too large
Load Diff
@@ -32,7 +32,7 @@ A job runs on `ubuntu-latest` unless it has one of these reasons, and the reason
|
||||
|
||||
Jobs that do only the following have no reason to be self-hosted: shell or Python scripts, `cargo fmt`, `cargo deny`, typos, actionlint, API calls (`gh`, dependency review, issue and stale bots, CLA checks), downloading an artifact and running a prebuilt binary, or a `cargo build -p rustfs` with its own cache key. `e2e-distributed.yml` and `e2e-upgrade.yml` build the server and the `e2e_test` harness on `ubuntu-latest` inside their budgets, so "it compiles something" alone is not a reason.
|
||||
|
||||
The functional chain's `prepare` and `complete-chain` jobs, the performance runner availability check, and runner hygiene use `ubuntu-latest`: they inspect APIs or artifacts, not the lab hosts. The actual functional and performance suites remain on their lab runners. Functional-chain health collection stays on the operator-managed `smoke-testing` runner as part of the automation environment. Docker image packaging also uses `ubuntu-latest` for both musl and glibc variants; it downloads release binaries rather than compiling the workspace, and still sets up QEMU and Buildx explicitly.
|
||||
The functional chain's `prepare` and `complete-chain` jobs, the performance runner availability check, and runner hygiene use `ubuntu-latest`: they inspect APIs or artifacts, not the lab hosts. The actual functional and performance suites remain on their lab runners. Functional-chain health collection uses `ubuntu-latest` so a lab outage cannot also stop its independent monitor. Docker image packaging also uses `ubuntu-latest` for both musl and glibc variants; it downloads release binaries rather than compiling the workspace, and still sets up QEMU and Buildx explicitly.
|
||||
|
||||
Moving these jobs off AKS reduces capacity demand, not necessarily the Azure bill immediately. VM costs fall only when nodes scale down or smaller capacity replaces them.
|
||||
|
||||
|
||||
@@ -193,7 +193,7 @@ jobs:
|
||||
{"target_id":"linux-aarch64-musl","os":"ubicloud-standard-16-arm","target":"aarch64-unknown-linux-musl","cross":false,"platform":"linux","rustflags":"","features":"","glibc":""},
|
||||
{"target_id":"linux-x86_64-gnu","os":"sm-standard-4","target":"x86_64-unknown-linux-gnu","cross":true,"platform":"linux","rustflags":"","features":"pyroscope","glibc":"2.31"},
|
||||
{"target_id":"linux-aarch64-gnu","os":"ubicloud-standard-16-arm","target":"aarch64-unknown-linux-gnu","cross":true,"platform":"linux","rustflags":"","features":"pyroscope","glibc":"2.31"},
|
||||
{"target_id":"macos-aarch64","os":"macos-latest","target":"aarch64-apple-darwin","cross":false,"platform":"macos","rustflags":"","features":"pyroscope","glibc":""},
|
||||
{"target_id":"macos-aarch64","os":"macos-latest-xlarge","target":"aarch64-apple-darwin","cross":false,"platform":"macos","rustflags":"","features":"pyroscope","glibc":""},
|
||||
{"target_id":"windows-x86_64","os":"windows-latest","target":"x86_64-pc-windows-msvc","cross":false,"platform":"windows","rustflags":"","features":"","glibc":""}
|
||||
]}'
|
||||
|
||||
|
||||
@@ -96,7 +96,7 @@ env:
|
||||
RUST_BACKTRACE: 1
|
||||
|
||||
jobs:
|
||||
# Readers: test-and-lint (workspace and ILM), build-rustfs-debug-binary,
|
||||
# Readers: workspace-lint-and-build, workspace-tests (workspace and ILM), build-rustfs-debug-binary,
|
||||
# e2e-tests, e2e-full.
|
||||
warm-ci-dev:
|
||||
name: Warm ci-dev
|
||||
@@ -146,7 +146,7 @@ jobs:
|
||||
if-no-files-found: error
|
||||
|
||||
# --all-targets covers the test binaries nextest builds, including
|
||||
# e2e_test, which test-and-lint's own run excludes. The second build adds
|
||||
# e2e_test, which the workspace run excludes. The second build adds
|
||||
# the e2e-test-hooks feature resolution that build-rustfs-debug-binary uses
|
||||
# and that no lint lane enables.
|
||||
- name: Build ci-dev superset
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
name: CI Resource Accounting
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
since:
|
||||
description: "Inclusive run creation timestamp, for example 2026-10-01T00:00:00Z"
|
||||
required: true
|
||||
type: string
|
||||
until:
|
||||
description: "Inclusive end timestamp; keep the window below 1000 runs"
|
||||
required: true
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
pull-requests: read
|
||||
|
||||
concurrency:
|
||||
group: ci-resource-accounting
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
collect:
|
||||
if: github.repository == 'rustfs/rustfs'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Collect all workflow attempts in the window
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
ACCOUNTING_SINCE: ${{ inputs.since }}
|
||||
ACCOUNTING_UNTIL: ${{ inputs.until }}
|
||||
ACCOUNTING_REPOSITORY: ${{ github.repository }}
|
||||
run: |
|
||||
python3 scripts/ci/actions_cost_report.py \
|
||||
--repository "$ACCOUNTING_REPOSITORY" \
|
||||
--since "$ACCOUNTING_SINCE" --until "$ACCOUNTING_UNTIL" \
|
||||
--output "$RUNNER_TEMP/ci-accounting.json" \
|
||||
--summary "$RUNNER_TEMP/ci-accounting.md"
|
||||
- name: Show available accounting summary
|
||||
if: always()
|
||||
run: |
|
||||
if [[ -f "$RUNNER_TEMP/ci-accounting.md" ]]; then
|
||||
cat "$RUNNER_TEMP/ci-accounting.md" >> "$GITHUB_STEP_SUMMARY"
|
||||
fi
|
||||
- name: Retain recomputable accounting evidence
|
||||
if: always()
|
||||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
|
||||
with:
|
||||
name: ci-accounting-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: |
|
||||
${{ runner.temp }}/ci-accounting.json
|
||||
${{ runner.temp }}/ci-accounting.md
|
||||
if-no-files-found: error
|
||||
retention-days: 30
|
||||
@@ -0,0 +1,92 @@
|
||||
name: CI Runner Pilot
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
pool:
|
||||
description: "Existing runner pool; no infrastructure is created"
|
||||
required: true
|
||||
type: choice
|
||||
options: [hosted, arc]
|
||||
default: hosted
|
||||
workload:
|
||||
description: "Probe only, or perform cold/warm fixed-source builds"
|
||||
required: true
|
||||
type: choice
|
||||
options: [probe, server, workspace-tests]
|
||||
default: probe
|
||||
source_sha:
|
||||
description: "Exact 40-character merged main SHA; required for builds"
|
||||
type: string
|
||||
toolchain:
|
||||
description: "Exact Rust version such as 1.92.0; required for builds"
|
||||
type: string
|
||||
scratch_root:
|
||||
description: "Existing absolute scratch directory; empty uses runner.temp"
|
||||
type: string
|
||||
build_jobs:
|
||||
description: "Fixed Cargo concurrency for both comparisons (1-16)"
|
||||
type: string
|
||||
default: "2"
|
||||
local_ssd:
|
||||
description: "Require operator /etc/rustfs-runner/storage.json attestation"
|
||||
type: boolean
|
||||
default: false
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ci-runner-pilot
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
hosted:
|
||||
if: github.repository == 'rustfs/rustfs' && github.ref == 'refs/heads/main' && inputs.pool == 'hosted'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 150
|
||||
steps:
|
||||
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: ./.github/actions/runner-pilot
|
||||
with:
|
||||
workload: ${{ inputs.workload }}
|
||||
source-sha: ${{ inputs.source_sha }}
|
||||
toolchain: ${{ inputs.toolchain }}
|
||||
scratch-root: ${{ inputs.scratch_root }}
|
||||
build-jobs: ${{ inputs.build_jobs }}
|
||||
local-ssd: ${{ inputs.local_ssd }}
|
||||
- name: Retain hosted pilot evidence
|
||||
if: always()
|
||||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
|
||||
with:
|
||||
name: runner-pilot-hosted-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/runner-pilot-evidence
|
||||
if-no-files-found: error
|
||||
retention-days: 30
|
||||
arc:
|
||||
if: github.repository == 'rustfs/rustfs' && github.ref == 'refs/heads/main' && inputs.pool == 'arc'
|
||||
# Disk and hardware: measure the existing workspace build pool's storage.
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 150
|
||||
steps:
|
||||
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: ./.github/actions/runner-pilot
|
||||
with:
|
||||
workload: ${{ inputs.workload }}
|
||||
source-sha: ${{ inputs.source_sha }}
|
||||
toolchain: ${{ inputs.toolchain }}
|
||||
scratch-root: ${{ inputs.scratch_root }}
|
||||
build-jobs: ${{ inputs.build_jobs }}
|
||||
local-ssd: ${{ inputs.local_ssd }}
|
||||
- name: Retain ARC pilot evidence
|
||||
if: always()
|
||||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
|
||||
with:
|
||||
name: runner-pilot-arc-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/runner-pilot-evidence
|
||||
if-no-files-found: error
|
||||
retention-days: 30
|
||||
+432
-181
@@ -44,6 +44,11 @@ on:
|
||||
schedule:
|
||||
- cron: "11 0 * * *" # Nightly 00:11 UTC, after the 22:41 UTC cache-warm run
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
rehearse_workspace_shards:
|
||||
description: Compare one full execution with at most two resource-safe shards of the same archive
|
||||
type: boolean
|
||||
default: false
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -83,6 +88,11 @@ jobs:
|
||||
timeout-minutes: 10
|
||||
outputs:
|
||||
mode: ${{ steps.scope.outputs.mode }}
|
||||
plan: ${{ steps.scope.outputs.plan }}
|
||||
connect: ${{ steps.scope.outputs.connect || 'true' }}
|
||||
packages: ${{ steps.scope.outputs.packages }}
|
||||
full_validation: ${{ steps.scope.outputs.full_validation || 'true' }}
|
||||
rehearse_shards: ${{ steps.scope.outputs.rehearse_shards || 'false' }}
|
||||
steps:
|
||||
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
@@ -91,16 +101,37 @@ jobs:
|
||||
- name: Select scope using the base revision's policy
|
||||
id: scope
|
||||
env:
|
||||
CI_BASE_SHA: ${{ github.event.pull_request.base.sha }}
|
||||
CI_BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha || github.event.before }}
|
||||
CI_EVENT_ACTION: ${{ github.event.action }}
|
||||
CI_REHEARSE_SHARDS: ${{ inputs.rehearse_workspace_shards && 'true' || 'false' }}
|
||||
run: |
|
||||
if [[ "$GITHUB_EVENT_NAME" != "pull_request" ]]; then
|
||||
printf '%s\n' 'mode=full' >> "$GITHUB_OUTPUT"
|
||||
python3 -I scripts/ci_gate.py select
|
||||
elif [[ "$CI_BASE_SHA" =~ ^[0-9a-f]{40}$ ]] && git show "$CI_BASE_SHA:scripts/ci_gate.py" > "$RUNNER_TEMP/ci-gate-base.py"; then
|
||||
python3 -I "$RUNNER_TEMP/ci-gate-base.py" select
|
||||
else
|
||||
printf '%s\n' 'mode=full' >> "$GITHUB_OUTPUT"
|
||||
echo "Base CI policy unavailable; running the full matrix."
|
||||
fi
|
||||
if ! grep -q '^plan=' "$GITHUB_OUTPUT" || ! grep -q '^plan_contract=3$' "$GITHUB_OUTPUT"; then
|
||||
printf '%s\n' 'mode=full' 'connect=true' 'full_validation=true' 'rehearse_shards=false' 'packages=' 'plan=' >> "$GITHUB_OUTPUT"
|
||||
echo "Base policy does not cover this workflow contract; retaining the full matrix."
|
||||
fi
|
||||
- name: Retain verification plan
|
||||
if: steps.scope.outputs.plan != ''
|
||||
env:
|
||||
CI_PLAN: ${{ steps.scope.outputs.plan }}
|
||||
run: |
|
||||
mkdir -p artifacts/ci-plan
|
||||
printf '%s\n' "$CI_PLAN" > artifacts/ci-plan/plan.json
|
||||
- name: Upload verification plan
|
||||
if: steps.scope.outputs.plan != ''
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: verification-plan-${{ github.run_attempt }}
|
||||
path: artifacts/ci-plan/plan.json
|
||||
if-no-files-found: error
|
||||
retention-days: 7
|
||||
|
||||
typos:
|
||||
name: Typos
|
||||
@@ -111,15 +142,23 @@ jobs:
|
||||
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Install pinned typos through GitHub API
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
TYPOS_VERSION: 1.50.2
|
||||
run: |
|
||||
python3 .github/actions/setup/download_release_asset.py --repository crate-ci/typos --tag "v${TYPOS_VERSION}" --asset "typos-v${TYPOS_VERSION}-x86_64-unknown-linux-musl.tar.gz" --output "$RUNNER_TEMP/typos.tar.gz"
|
||||
mkdir -p "$RUNNER_TEMP/typos"
|
||||
tar -xzf "$RUNNER_TEMP/typos.tar.gz" -C "$RUNNER_TEMP/typos"
|
||||
- name: Typos check with custom config file
|
||||
uses: crate-ci/typos@512fc24f32f44ab01972217aaaf3dc86ec234d53 # v1.50.2
|
||||
run: '"$RUNNER_TEMP/typos/typos" .'
|
||||
|
||||
# Fail early with compile-free checks for every pull request.
|
||||
quick-checks:
|
||||
name: Quick Checks
|
||||
if: github.event_name != 'pull_request' || github.event.action != 'closed'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
@@ -129,14 +168,15 @@ jobs:
|
||||
- name: Run shared quick checks
|
||||
uses: ./.github/actions/quick-checks
|
||||
|
||||
test-and-lint:
|
||||
name: Workspace Test and Lint
|
||||
workspace-lint-and-build:
|
||||
name: Workspace Lint and Build
|
||||
if: contains(fromJSON('["full", "e2e"]'), needs.classify-changes.outputs.mode) && (github.event_name != 'pull_request' || github.event.action != 'closed')
|
||||
needs: [ quick-checks, classify-changes ]
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 90
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
CI_PACKAGES: ${{ needs.classify-changes.outputs.packages }}
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
@@ -161,13 +201,13 @@ jobs:
|
||||
|
||||
- name: Prepare test evidence
|
||||
run: |
|
||||
mkdir -p artifacts/test-and-lint
|
||||
mkdir -p artifacts/workspace
|
||||
{
|
||||
echo "run_id=${GITHUB_RUN_ID}"
|
||||
echo "job=${GITHUB_JOB}"
|
||||
echo "runner=${RUNNER_NAME}"
|
||||
echo "started_at=$(date --utc --iso-8601=seconds)"
|
||||
} > artifacts/test-and-lint/run-metadata.txt
|
||||
} > artifacts/workspace/run-metadata.txt
|
||||
|
||||
# Clippy gates every event, pull requests included. When it ran only after
|
||||
# merge, a lint regression merged freely and then failed every main run
|
||||
@@ -179,73 +219,34 @@ jobs:
|
||||
run: |
|
||||
./scripts/ci/resource_sampler.sh start clippy
|
||||
trap './scripts/ci/resource_sampler.sh stop' EXIT
|
||||
cargo clippy --all-targets -- -D warnings
|
||||
python3 scripts/ci_test_artifact.py cargo lint --configuration workspace
|
||||
|
||||
- name: Run nextest tests
|
||||
- name: Archive workspace test binaries
|
||||
env:
|
||||
# #5394 mitigation, settled by the backlog#1601 experiment.
|
||||
#
|
||||
# 2 was chosen when three concurrent workspace test links were believed
|
||||
# to saturate the runner's overlay I/O and wedge Cargo until the 75m
|
||||
# timeout. cgroup v2 readings from the sampler show the pod actually
|
||||
# has 14 CPUs and 28GB (peak use 2.1GB): the label name "sm-standard-4"
|
||||
# had led everyone, including the original mitigation, to assume 4
|
||||
# cores, and memory was never the constraint.
|
||||
#
|
||||
# Measured on the same ci-dev cache, the test-profile compile took
|
||||
# 28m21s at 2 on a pull request and 16-23 min at 3 across three
|
||||
# successful main runs. The first pull-request run at 3 that met a
|
||||
# busy node (run 39193827281) then hit the 75m timeout before any
|
||||
# test started: the sampler shows cargo spinning for 70 minutes with
|
||||
# the pod's io.pressure "full" at 86-88% and CPU pressure at zero,
|
||||
# while node-level IO pressure was already 36% from co-tenants when
|
||||
# the step began. The constraint is the pod's shared storage, not
|
||||
# CPU or memory, and the limit that storage sustains on a contended
|
||||
# node is 2. Main pushes and dispatches keep 3 (rarer, and three
|
||||
# successes); pull requests stay at 2 until the pods get local disk
|
||||
# for target/, which is the lever that would unlock the 14 cores.
|
||||
#
|
||||
# Must stay step-level: rust-cache hashes CARGO/CC/CFLAGS/CXX/CMAKE/RUST
|
||||
# prefixed variables from process.env into the cache key, so promoting
|
||||
# this to job level would rotate every key on this lane.
|
||||
CARGO_BUILD_JOBS: ${{ (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && '3' || '2' }}
|
||||
# Workspace test compilation has high per-process memory use.
|
||||
CARGO_BUILD_JOBS: "2"
|
||||
run: |
|
||||
mkdir -p artifacts/test-and-lint
|
||||
rm -f target/nextest/ci/junit.xml
|
||||
./scripts/ci/resource_sampler.sh start nextest
|
||||
./scripts/ci/resource_sampler.sh start workspace-archive
|
||||
trap './scripts/ci/resource_sampler.sh stop' EXIT
|
||||
set +e
|
||||
NEXTEST_HIDE_PROGRESS_BAR=1 timeout --verbose --signal=TERM --kill-after=30s 75m \
|
||||
cargo nextest run --profile ci --all --exclude e2e_test \
|
||||
--status-level all --final-status-level all \
|
||||
2>&1 | tee artifacts/test-and-lint/nextest.log
|
||||
status=${PIPESTATUS[0]}
|
||||
if [[ "${status}" -eq 0 ]]; then
|
||||
cargo nextest list --profile ci --all --exclude e2e_test --message-format json \
|
||||
> artifacts/test-and-lint/core-test-listing.json \
|
||||
&& python3 scripts/check_test_wiring.py --check-core artifacts/test-and-lint/core-test-listing.json \
|
||||
&& test -s target/nextest/ci/junit.xml || status=$?
|
||||
fi
|
||||
{
|
||||
echo "command=cargo nextest run --profile ci --all --exclude e2e_test"
|
||||
echo "exit_status=${status}"
|
||||
echo "finished_at=$(date --utc --iso-8601=seconds)"
|
||||
echo
|
||||
echo "Remaining test-related processes:"
|
||||
pgrep -af 'cargo|nextest|target/.*/deps/' || true
|
||||
echo
|
||||
echo "Kernel OOM / kill events:"
|
||||
dmesg -T 2>/dev/null | grep -iE 'oom|out of memory|killed process' | tail -20 || true
|
||||
} > artifacts/test-and-lint/nextest-diagnostics.txt
|
||||
exit "${status}"
|
||||
python3 scripts/ci_test_artifact.py build --configuration workspace --archive "$RUNNER_TEMP/workspace-tests.tar.zst"
|
||||
|
||||
- name: Upload workspace test binaries
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: workspace-test-binaries
|
||||
overwrite: true
|
||||
path: ${{ runner.temp }}/workspace-tests.tar.zst*
|
||||
compression-level: 0
|
||||
if-no-files-found: error
|
||||
retention-days: 1
|
||||
|
||||
- name: Run documentation tests
|
||||
run: |
|
||||
mkdir -p artifacts/test-and-lint
|
||||
mkdir -p artifacts/workspace
|
||||
set +e
|
||||
timeout --verbose --signal=TERM --kill-after=30s 15m \
|
||||
cargo test --all --exclude e2e_test --doc \
|
||||
2>&1 | tee artifacts/test-and-lint/doctest.log
|
||||
python3 scripts/ci_test_artifact.py cargo doc --configuration workspace \
|
||||
2>&1 | tee artifacts/workspace/doctest.log
|
||||
status=${PIPESTATUS[0]}
|
||||
{
|
||||
echo "command=cargo test --all --exclude e2e_test --doc"
|
||||
@@ -254,17 +255,16 @@ jobs:
|
||||
echo
|
||||
echo "Remaining test-related processes:"
|
||||
pgrep -af 'cargo|rustdoc|target/.*/deps/' || true
|
||||
} > artifacts/test-and-lint/doctest-diagnostics.txt
|
||||
} > artifacts/workspace/doctest-diagnostics.txt
|
||||
exit "${status}"
|
||||
|
||||
- name: Upload test reports and diagnostics
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: junit-test-and-lint-${{ github.run_number }}
|
||||
name: junit-test-and-lint-${{ github.run_number }}-${{ github.run_attempt }}
|
||||
path: |
|
||||
target/nextest/ci/junit.xml
|
||||
artifacts/test-and-lint
|
||||
artifacts/workspace
|
||||
retention-days: 3
|
||||
if-no-files-found: error
|
||||
|
||||
@@ -275,33 +275,78 @@ jobs:
|
||||
- name: Check log-analyzer rule anchors
|
||||
run: ./scripts/check_log_analyzer_rules.sh
|
||||
|
||||
# Preserve the migration floor and require successful execution evidence
|
||||
# from the workspace run, without compiling or executing its subset again.
|
||||
- name: Verify rebalance/decommission migration proofs
|
||||
run: ./scripts/check_migration_gate_count.sh evidence artifacts/test-and-lint/core-test-listing.json target/nextest/ci/junit.xml
|
||||
workspace-tests:
|
||||
name: Workspace tests
|
||||
needs: [ workspace-lint-and-build, classify-changes ]
|
||||
# The archive retains compile-time fixture paths and the large test set
|
||||
# requires the same runner image until the hosted-runtime pilot passes.
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 90
|
||||
outputs:
|
||||
workspace_shards: ${{ steps.shards.outputs.workspace_shards || '[]' }}
|
||||
env:
|
||||
CI_PACKAGES: ${{ needs.classify-changes.outputs.packages }}
|
||||
steps:
|
||||
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: ./.github/actions/test-runtime
|
||||
- name: Protect Connect test home
|
||||
run: chmod go-w "$(realpath "$HOME")"
|
||||
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: workspace-test-binaries
|
||||
path: ${{ runner.temp }}
|
||||
- name: Verify workspace archive identity
|
||||
run: python3 scripts/ci_test_artifact.py verify --configuration workspace --archive "$RUNNER_TEMP/workspace-tests.tar.zst"
|
||||
|
||||
- name: Extract and list workspace tests
|
||||
run: |
|
||||
mkdir -p artifacts/workspace
|
||||
cargo nextest list --profile ci --archive-file "$RUNNER_TEMP/workspace-tests.tar.zst" --workspace-remap "$GITHUB_WORKSPACE" --extract-to "$GITHUB_WORKSPACE" --extract-overwrite --message-format json \
|
||||
> artifacts/workspace/core-test-listing.json
|
||||
python3 scripts/check_test_wiring.py --check-core artifacts/workspace/core-test-listing.json
|
||||
|
||||
- name: Run nextest tests
|
||||
run: |
|
||||
mkdir -p artifacts/workspace
|
||||
rm -f target/nextest/ci/junit.xml
|
||||
./scripts/ci/resource_sampler.sh start nextest
|
||||
trap './scripts/ci/resource_sampler.sh stop' EXIT
|
||||
set +e
|
||||
NEXTEST_HIDE_PROGRESS_BAR=1 timeout --verbose --signal=TERM --kill-after=30s 75m \
|
||||
cargo nextest run --profile ci --cargo-metadata target/nextest/cargo-metadata.json --binaries-metadata target/nextest/binaries-metadata.json \
|
||||
--workspace-remap "$GITHUB_WORKSPACE" --target-dir-remap "$GITHUB_WORKSPACE/target" \
|
||||
--status-level all --final-status-level all \
|
||||
2>&1 | tee artifacts/workspace/nextest.log
|
||||
status=${PIPESTATUS[0]}
|
||||
{
|
||||
echo "command=workspace archive execution"
|
||||
echo "exit_status=${status}"
|
||||
echo "finished_at=$(date --utc --iso-8601=seconds)"
|
||||
echo
|
||||
echo "Remaining test-related processes:"
|
||||
pgrep -af 'cargo|nextest|target/.*/deps/' || true
|
||||
echo
|
||||
echo "Kernel OOM / kill events:"
|
||||
dmesg -T 2>/dev/null | grep -iE 'oom|out of memory|killed process' | tail -20 || true
|
||||
} > artifacts/workspace/nextest-diagnostics.txt
|
||||
exit "${status}"
|
||||
|
||||
- name: Verify workspace execution completeness
|
||||
run: |
|
||||
python3 scripts/ci_test_artifact.py evidence artifacts/workspace/core-test-listing.json target/nextest/ci/junit.xml
|
||||
./scripts/check_migration_gate_count.sh evidence artifacts/workspace/core-test-listing.json target/nextest/ci/junit.xml
|
||||
|
||||
# The workspace already compiled these ignored tests. Keep the same Cargo
|
||||
# package selection to reuse their binaries, then run them serially in
|
||||
# separate processes because they share fixed ports and global state.
|
||||
# test_transition_and_restore_flows was re-enabled by rustfs/backlog#1303:
|
||||
# its "missing xl.meta on disk2" was a test-util bug (open_disk hardcoded
|
||||
# disk_index 0), not an EC metadata-distribution issue.
|
||||
# restore_object_usecase_reports_ongoing_conflict_and_completion was
|
||||
# re-enabled by backlog#1304 (restore accepts serialize on a short CAS
|
||||
# guard; the copy-back no longer holds the #4877 whole-copy-back lock,
|
||||
# so the mid-restore ongoing read and fast 409 rejection it asserts are
|
||||
# the implemented contract).
|
||||
- name: Run ignored ILM integration tests serially
|
||||
if: needs.classify-changes.outputs.mode == 'full'
|
||||
env:
|
||||
# Match the preceding workspace build and reuse its feature graph.
|
||||
CARGO_BUILD_JOBS: ${{ (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && '3' || '2' }}
|
||||
run: |
|
||||
mkdir -p artifacts/ilm-integration
|
||||
set +e
|
||||
NEXTEST_HIDE_PROGRESS_BAR=1 timeout --verbose --signal=TERM --kill-after=30s 80m \
|
||||
cargo nextest run --profile default -j1 --run-ignored ignored-only \
|
||||
--all --exclude e2e_test \
|
||||
--cargo-metadata target/nextest/cargo-metadata.json --binaries-metadata target/nextest/binaries-metadata.json \
|
||||
--workspace-remap "$GITHUB_WORKSPACE" --target-dir-remap "$GITHUB_WORKSPACE/target" \
|
||||
-E 'binary(lifecycle_integration_test) or (package(rustfs) and test(lifecycle_transition_api_test))' \
|
||||
--status-level all --final-status-level all \
|
||||
2>&1 | tee artifacts/ilm-integration/nextest.log
|
||||
@@ -315,21 +360,105 @@ jobs:
|
||||
} > artifacts/ilm-integration/diagnostics.txt
|
||||
exit "${status}"
|
||||
|
||||
- name: Upload ILM test diagnostics
|
||||
if: always() && needs.classify-changes.outputs.mode == 'full'
|
||||
- name: Upload workspace test evidence
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: ilm-integration-${{ github.run_number }}-${{ github.run_attempt }}
|
||||
path: artifacts/ilm-integration
|
||||
name: workspace-test-evidence-${{ github.run_attempt }}
|
||||
path: |
|
||||
target/nextest/ci/junit.xml
|
||||
artifacts/workspace
|
||||
artifacts/ilm-integration
|
||||
retention-days: 3
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Prepare bounded shards from this candidate's observed durations
|
||||
id: shards
|
||||
if: needs.classify-changes.outputs.rehearse_shards == 'true'
|
||||
run: |
|
||||
python3 scripts/ci_test_shards.py prepare \
|
||||
--listing artifacts/workspace/core-test-listing.json \
|
||||
--junit target/nextest/ci/junit.xml --output artifacts/workspace-shard-plan
|
||||
- name: Upload same-run shard plan
|
||||
if: needs.classify-changes.outputs.rehearse_shards == 'true'
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: workspace-shard-plan-${{ github.run_attempt }}
|
||||
path: artifacts/workspace-shard-plan
|
||||
retention-days: 3
|
||||
if-no-files-found: error
|
||||
|
||||
workspace-shards:
|
||||
name: Workspace shards (${{ matrix.shard }})
|
||||
if: needs.classify-changes.outputs.rehearse_shards == 'true'
|
||||
needs: [classify-changes, workspace-tests]
|
||||
# Reuse the producer image and checkout path for the full RustFS archive.
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 90
|
||||
strategy:
|
||||
fail-fast: false
|
||||
max-parallel: 2
|
||||
matrix:
|
||||
shard: ${{ fromJSON(needs.workspace-tests.outputs.workspace_shards || '[]') }}
|
||||
env:
|
||||
CI_PACKAGES: ${{ needs.classify-changes.outputs.packages }}
|
||||
steps:
|
||||
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: ./.github/actions/test-runtime
|
||||
- name: Protect Connect test home
|
||||
run: chmod go-w "$(realpath "$HOME")"
|
||||
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: workspace-test-binaries-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}
|
||||
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: workspace-shard-plan-${{ github.run_attempt }}
|
||||
path: artifacts/workspace-shard-plan
|
||||
- name: Verify and execute this shard without compilation
|
||||
env:
|
||||
SHARD_INDEX: ${{ matrix.shard }}
|
||||
run: |
|
||||
python3 scripts/ci_test_artifact.py verify --configuration workspace --archive "$RUNNER_TEMP/workspace-tests.tar.zst"
|
||||
python3 scripts/ci_test_shards.py execute --plan artifacts/workspace-shard-plan \
|
||||
--archive "$RUNNER_TEMP/workspace-tests.tar.zst" --index "$SHARD_INDEX" --output artifacts/workspace-shard-result
|
||||
- name: Retain shard evidence including failures
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: workspace-shard-result-${{ matrix.shard }}-${{ github.run_attempt }}
|
||||
path: artifacts/workspace-shard-result
|
||||
retention-days: 3
|
||||
if-no-files-found: error
|
||||
|
||||
workspace-shard-evidence:
|
||||
name: Workspace shard completeness
|
||||
if: always() && needs.classify-changes.outputs.rehearse_shards == 'true'
|
||||
needs: [classify-changes, workspace-tests, workspace-shards]
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: workspace-shard-plan-${{ github.run_attempt }}
|
||||
path: artifacts/workspace-shard-plan
|
||||
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
pattern: workspace-shard-result-*-${{ github.run_attempt }}
|
||||
path: artifacts/workspace-shards
|
||||
- name: Require the exact union of successful shard executions
|
||||
run: |
|
||||
python3 scripts/ci_test_shards.py verify --plan artifacts/workspace-shard-plan --evidence artifacts/workspace-shards
|
||||
./scripts/check_migration_gate_count.sh evidence artifacts/workspace-shard-plan/listing.json artifacts/workspace-shards/combined.xml
|
||||
|
||||
# Both Connect boundaries rebuild only the rustfs crate on top of the restored
|
||||
# ci-dev cache: the two e2e features are empty, so every dependency keeps its
|
||||
# cached dev-profile artifact and each feature set recompiles rustfs once.
|
||||
# The budget covers a ci-dev miss, where the first step builds the whole graph.
|
||||
connect-feature-boundaries:
|
||||
name: Connect Feature Boundaries
|
||||
if: needs.classify-changes.outputs.mode == 'full' && (github.event_name != 'pull_request' || github.event.action != 'closed')
|
||||
if: needs.classify-changes.outputs.mode == 'full' && needs.classify-changes.outputs.connect != 'false' && contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name)
|
||||
needs: [ quick-checks, classify-changes ]
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 90
|
||||
@@ -357,12 +486,10 @@ jobs:
|
||||
env:
|
||||
CARGO_BUILD_JOBS: "2"
|
||||
run: |
|
||||
cargo test -p rustfs --test connect_registration \
|
||||
--features connect-e2e-short-credentials \
|
||||
registration_enforces_build_profile_credential_lifetime -- --exact
|
||||
cargo test -p rustfs --test connect_registration \
|
||||
--features connect-e2e-short-credentials \
|
||||
rotation_waits_for_threshold_and_stops_on_revocation -- --exact
|
||||
cargo test --locked -p rustfs --test connect_registration \
|
||||
--features connect-e2e-short-credentials -- --exact --test-threads=1 \
|
||||
registration_enforces_build_profile_credential_lifetime \
|
||||
rotation_waits_for_threshold_and_stops_on_revocation
|
||||
|
||||
# The guard in rustfs/src/connect/registration.rs is cfg(not(debug_assertions)),
|
||||
# and it must be tripped by a profile that turns debug assertions off for
|
||||
@@ -374,37 +501,77 @@ jobs:
|
||||
# link-concurrency limit; it is cold because ci-dev holds no release
|
||||
# artifacts.
|
||||
- name: Reject short credentials in release builds
|
||||
run: |
|
||||
log="$(mktemp)"
|
||||
set +e
|
||||
CARGO_TERM_COLOR=never cargo check -p rustfs --release \
|
||||
--features connect-e2e-short-credentials >"$log" 2>&1
|
||||
status=$?
|
||||
set -e
|
||||
cat "$log"
|
||||
expected='error: connect-e2e-short-credentials is restricted to debug builds'
|
||||
summary="error: could not compile \`rustfs\` (lib) due to 1 previous error"
|
||||
expected_count="$(grep -Fxc "$expected" "$log" || true)"
|
||||
summary_count="$(grep -Fc "$summary" "$log" || true)"
|
||||
error_count="$(grep -Ec '^error(:|\[)' "$log" || true)"
|
||||
if [ "$status" -ne 101 ] || [ "$expected_count" -ne 1 ] \
|
||||
|| [ "$summary_count" -ne 1 ] || [ "$error_count" -ne 2 ]; then
|
||||
echo "feature gate did not fail solely at the expected compile_error" >&2
|
||||
rm -f "$log"
|
||||
exit 1
|
||||
fi
|
||||
rm -f "$log"
|
||||
run: python3 scripts/check_connect_release_boundary.py
|
||||
|
||||
- name: Check offline enrollment E2E root boundary
|
||||
run: ./scripts/check_offline_enrollment_e2e.sh
|
||||
|
||||
fips-provider:
|
||||
name: FIPS provider boundaries
|
||||
if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name)
|
||||
needs: [ quick-checks, classify-changes ]
|
||||
# This builds the RustFS library test target, which exceeds hosted-runner disk capacity.
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 90
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
mode: [default, fips]
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
CARGO_BUILD_JOBS: "2"
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
with:
|
||||
rust-version: stable
|
||||
cache-shared-key: ci-fips-provider
|
||||
cache-save-if: 'false'
|
||||
install-build-packaging-tools: 'false'
|
||||
|
||||
- name: Install FIPS build tools
|
||||
if: matrix.mode == 'fips'
|
||||
run: sudo apt-get install -y cmake golang clang libclang-dev
|
||||
|
||||
- name: Verify provider installation
|
||||
shell: bash
|
||||
run: |
|
||||
args=()
|
||||
if [[ '${{ matrix.mode }}' == 'fips' ]]; then
|
||||
args+=(--features fips)
|
||||
fi
|
||||
cargo nextest run --locked -p rustfs-tls-runtime --lib "${args[@]}" --no-tests fail -E 'test(provider_policy_isolated)'
|
||||
cargo nextest run --locked -p rustfs-s3-client --lib "${args[@]}" --no-tests fail -E 'test(overflowing_closed_range_is_rejected_before_network_io)'
|
||||
cargo nextest run --locked -p rustfs-targets --lib "${args[@]}" --no-tests fail -E 'test(validate_pulsar)'
|
||||
cargo nextest run --locked -p rustfs-targets --lib "${args[@]}" --no-tests fail -E 'test(webhook_provider_policy_isolated)'
|
||||
if [[ '${{ matrix.mode }}' == 'fips' ]]; then
|
||||
cargo nextest run --locked -p rustfs-protocols --lib --features ftps,webdav,fips --no-tests fail -E 'test(protocols_reject_incompatible_host_provider)'
|
||||
cargo nextest run --locked -p rustfs-targets --lib --features fips --no-tests fail -E 'test(fips_postgres_rejects_incompatible_host_provider)'
|
||||
cargo nextest run --locked -p rustfs-s3-client --lib --features fips --no-tests fail -E 'test(fips_transition_rejects_incompatible_host_provider)'
|
||||
fi
|
||||
tests=(provider_installation_isolated prepared_generation_shares_crypto_and_serves_sni_with_mtls)
|
||||
if [[ '${{ matrix.mode }}' == 'fips' ]]; then
|
||||
tests+=(fips_provider_builds_fips_tls_configs fips_provider_tls_handshake_accepts_and_rejects_algorithms)
|
||||
fi
|
||||
for test_name in "${tests[@]}"; do
|
||||
cargo nextest run --locked -p rustfs --lib "${args[@]}" --no-tests fail -E "test($test_name)"
|
||||
done
|
||||
cargo clippy --locked -p rustfs --lib "${args[@]}" -- -D warnings
|
||||
|
||||
test-and-lint-rio-v2:
|
||||
name: Test and Lint (rio-v2)
|
||||
# Alternate feature configurations run on the nightly schedule, merge
|
||||
# groups and manual dispatch. The default workspace remains the PR gate and
|
||||
# the per-push main baseline: repeating four feature builds on every merge
|
||||
# bought nothing the nightly run does not report within a day.
|
||||
if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["merge_group", "schedule", "workflow_dispatch"]'), github.event_name)
|
||||
# Specialized boundary, hardware and alternate-feature lanes run only on
|
||||
# the nightly schedule and manual dispatch. They do not gate a merge: the
|
||||
# nightly run reports a regression within a day, and repeating these builds
|
||||
# on every PR revision and merge group bought no earlier signal worth the
|
||||
# runner occupancy. Workspace lint and tests, the core contract, the debug
|
||||
# build, E2E smoke and the S3 lanes remain the pull-request gate.
|
||||
if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name)
|
||||
needs: [ quick-checks, classify-changes ]
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 90
|
||||
@@ -441,29 +608,29 @@ jobs:
|
||||
# --profile ci so the quarantine list (and its junit flaky markers)
|
||||
# covers this leg too; the default profile is the local no-retry
|
||||
# profile and silently ignored quarantined flakes here (rustfs#6703).
|
||||
mkdir -p artifacts/test-and-lint
|
||||
mkdir -p artifacts/workspace
|
||||
rm -f target/nextest/ci/junit.xml
|
||||
./scripts/ci/resource_sampler.sh start nextest-rio-v2
|
||||
trap './scripts/ci/resource_sampler.sh stop' EXIT
|
||||
cargo nextest run --profile ci -p rustfs -p rustfs-ecstore --features rio-v2 \
|
||||
2>&1 | tee artifacts/test-and-lint/nextest.log
|
||||
2>&1 | tee artifacts/workspace/nextest.log
|
||||
cargo test -p rustfs --doc --features rio-v2
|
||||
|
||||
- name: Upload rio-v2 test reports and diagnostics
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: junit-test-and-lint-rio-v2-${{ github.run_number }}
|
||||
name: junit-test-and-lint-rio-v2-${{ github.run_number }}-${{ github.run_attempt }}
|
||||
path: |
|
||||
target/nextest/ci/junit.xml
|
||||
artifacts/test-and-lint
|
||||
artifacts/workspace
|
||||
retention-days: 3
|
||||
if-no-files-found: error
|
||||
|
||||
test-and-lint-protocols:
|
||||
name: "Test and Lint (${{ matrix.features.name }})"
|
||||
# Same cadence as test-and-lint-rio-v2 above.
|
||||
if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["merge_group", "schedule", "workflow_dispatch"]'), github.event_name)
|
||||
if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name)
|
||||
needs: [ quick-checks, classify-changes ]
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 90
|
||||
@@ -532,7 +699,7 @@ jobs:
|
||||
path: |
|
||||
target/nextest/ci/junit.xml
|
||||
artifacts/protocol-tests
|
||||
artifacts/test-and-lint
|
||||
artifacts/workspace
|
||||
retention-days: 3
|
||||
if-no-files-found: warn
|
||||
|
||||
@@ -541,7 +708,7 @@ jobs:
|
||||
if: contains(fromJSON('["full", "e2e"]'), needs.classify-changes.outputs.mode) && (github.event_name != 'pull_request' || github.event.action != 'closed')
|
||||
needs: [ quick-checks, classify-changes ]
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 30
|
||||
timeout-minutes: 60
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
steps:
|
||||
@@ -609,6 +776,7 @@ jobs:
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: rustfs-debug-binary
|
||||
overwrite: true
|
||||
path: |
|
||||
target/debug/rustfs
|
||||
target/debug/rustfs.e2e.json
|
||||
@@ -616,6 +784,24 @@ jobs:
|
||||
if-no-files-found: error
|
||||
retention-days: 1
|
||||
|
||||
- name: Archive E2E test binaries
|
||||
env:
|
||||
CARGO_BUILD_JOBS: "2"
|
||||
run: python3 scripts/ci_test_artifact.py build --configuration e2e --archive "$RUNNER_TEMP/e2e-tests.tar.zst"
|
||||
|
||||
- name: Run E2E documentation tests
|
||||
run: cargo test --locked -p e2e_test --doc
|
||||
|
||||
- name: Upload E2E test binaries
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: e2e-test-binaries
|
||||
overwrite: true
|
||||
path: ${{ runner.temp }}/e2e-tests.tar.zst*
|
||||
compression-level: 0
|
||||
if-no-files-found: error
|
||||
retention-days: 1
|
||||
|
||||
build-rustfs-debug-binary-rio-v2:
|
||||
name: Build RustFS Debug Binary (rio-v2)
|
||||
# Dormant rio-v2 variant (rustfs/backlog#1835): the feature ships in no
|
||||
@@ -650,6 +836,7 @@ jobs:
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: rustfs-debug-binary-rio-v2
|
||||
overwrite: true
|
||||
path: |
|
||||
target/debug/rustfs
|
||||
target/debug/rustfs.e2e.json
|
||||
@@ -664,7 +851,7 @@ jobs:
|
||||
# job had neither, so each closed/merged PR really ran the whole io_uring
|
||||
# suite (measured 4m17s / 7m19s / 7m31s on runs 30678272341 / 30678117601 /
|
||||
# 30662728539) and kept the cancellation run in progress for minutes.
|
||||
if: needs.classify-changes.outputs.mode == 'full' && (github.event_name != 'pull_request' || github.event.action != 'closed')
|
||||
if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name)
|
||||
needs: [ quick-checks, classify-changes ]
|
||||
# GitHub-hosted ubuntu-latest runs a recent kernel with io_uring and, unlike
|
||||
# a container, applies no seccomp filter that would block io_uring_setup — so
|
||||
@@ -732,10 +919,51 @@ jobs:
|
||||
# scripts/check_uring_lane_lib_only.sh guards the --lib precondition.
|
||||
run: cargo test -p rustfs-ecstore --lib uring_ -- --test-threads=1 --nocapture
|
||||
|
||||
core-contract:
|
||||
name: Core contract
|
||||
needs: [ build-rustfs-debug-binary ]
|
||||
# Compile-time fixture paths must match the producer until the pilot passes.
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: ./.github/actions/test-runtime
|
||||
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: rustfs-debug-binary
|
||||
path: target/debug
|
||||
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: e2e-test-binaries
|
||||
path: ${{ runner.temp }}
|
||||
- name: Run core storage and authorization contract
|
||||
env:
|
||||
RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/core-contract-logs
|
||||
run: |
|
||||
chmod +x target/debug/rustfs
|
||||
python3 scripts/ci_test_artifact.py verify --configuration e2e --archive "$RUNNER_TEMP/e2e-tests.tar.zst"
|
||||
cargo nextest list --profile core-contract --archive-file "$RUNNER_TEMP/e2e-tests.tar.zst" --workspace-remap "$GITHUB_WORKSPACE" --extract-to "$GITHUB_WORKSPACE" --extract-overwrite --message-format json > "$RUNNER_TEMP/core-contract-list.json"
|
||||
python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run --profile core-contract --cargo-metadata target/nextest/cargo-metadata.json --binaries-metadata target/nextest/binaries-metadata.json --workspace-remap "$GITHUB_WORKSPACE" --target-dir-remap "$GITHUB_WORKSPACE/target" --no-tests fail
|
||||
python3 scripts/ci_test_artifact.py evidence --core "$RUNNER_TEMP/core-contract-list.json" target/nextest/core-contract/junit.xml
|
||||
- name: Upload core contract evidence
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: core-contract-${{ github.run_attempt }}
|
||||
path: |
|
||||
target/nextest/core-contract/junit.xml
|
||||
${{ runner.temp }}/core-contract-list.json
|
||||
${{ runner.temp }}/core-contract-logs
|
||||
if-no-files-found: error
|
||||
retention-days: 3
|
||||
|
||||
e2e-tests:
|
||||
name: End-to-End Tests
|
||||
needs: [ build-rustfs-debug-binary ]
|
||||
runs-on: sm-standard-2
|
||||
# Retain the producer image and fixture paths until the runtime pilot passes.
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
@@ -743,16 +971,17 @@ jobs:
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
# Full setup with dependency caching: the smoke-suite step below
|
||||
# compiles the e2e_test crate, which pulls in most of the workspace.
|
||||
# Without a restored cache this was a cold multi-GB build on every run.
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
- name: Install verification runtime
|
||||
uses: ./.github/actions/test-runtime
|
||||
|
||||
- name: Download E2E test binaries
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
rust-version: stable
|
||||
cache-shared-key: ci-dev
|
||||
cache-save-if: 'false'
|
||||
install-build-packaging-tools: 'false'
|
||||
name: e2e-test-binaries
|
||||
path: ${{ runner.temp }}
|
||||
|
||||
- name: Verify E2E archive identity
|
||||
run: python3 scripts/ci_test_artifact.py verify --configuration e2e --archive "$RUNNER_TEMP/e2e-tests.tar.zst"
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
||||
@@ -781,20 +1010,15 @@ jobs:
|
||||
# Build the e2e test graph once. The archive is reused by the smoke
|
||||
# selection guard, security exact-count check, and run below, avoiding a
|
||||
# second compile of the same e2e_test target on cold runners (backlog#1645).
|
||||
- name: Archive e2e smoke test binaries
|
||||
- name: Verify e2e smoke test membership
|
||||
env:
|
||||
NEXTEST_ARCHIVE: ${{ runner.temp }}/rustfs-e2e-smoke.tar.zst
|
||||
NEXTEST_ARCHIVE: ${{ runner.temp }}/e2e-tests.tar.zst
|
||||
NEXTEST_LISTING: ${{ runner.temp }}/rustfs-e2e-smoke-list.json
|
||||
run: |
|
||||
cargo nextest archive --profile e2e-smoke -p e2e_test --archive-file "${NEXTEST_ARCHIVE}"
|
||||
cargo nextest list --profile e2e-smoke --archive-file "${NEXTEST_ARCHIVE}" --message-format json > "${NEXTEST_LISTING}"
|
||||
cargo nextest list --profile e2e-smoke --archive-file "${NEXTEST_ARCHIVE}" --workspace-remap "$GITHUB_WORKSPACE" --extract-to "$GITHUB_WORKSPACE" --extract-overwrite --message-format json > "${NEXTEST_LISTING}"
|
||||
python3 ./scripts/check_test_wiring.py --check-profile e2e-smoke "${NEXTEST_LISTING}"
|
||||
./scripts/check_security_smoke_count.sh check "${NEXTEST_LISTING}"
|
||||
|
||||
# Keep E2E documentation coverage with its already-built package graph.
|
||||
- name: Run E2E documentation tests
|
||||
run: cargo test -p e2e_test --doc
|
||||
|
||||
# PR smoke subset of the in-repo e2e suite (backlog#1149 ci-4). The
|
||||
# profile.e2e-smoke default-filter in .config/nextest.toml is the single
|
||||
# wiring mechanism for e2e tests in CI — extend that filter instead of
|
||||
@@ -802,34 +1026,40 @@ jobs:
|
||||
# random port and reuses the downloaded debug binary above.
|
||||
- name: Run e2e smoke suite
|
||||
env:
|
||||
NEXTEST_ARCHIVE: ${{ runner.temp }}/rustfs-e2e-smoke.tar.zst
|
||||
NEXTEST_ARCHIVE: ${{ runner.temp }}/e2e-tests.tar.zst
|
||||
RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-e2e-smoke-logs
|
||||
SAMPLER_INTERVAL_SECS: "15"
|
||||
run: |
|
||||
./scripts/ci/resource_sampler.sh start e2e-smoke
|
||||
trap './scripts/ci/resource_sampler.sh stop' EXIT
|
||||
python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run --profile e2e-smoke --archive-file "${NEXTEST_ARCHIVE}" \
|
||||
python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run --profile e2e-smoke --cargo-metadata target/nextest/cargo-metadata.json --binaries-metadata target/nextest/binaries-metadata.json --workspace-remap "$GITHUB_WORKSPACE" --target-dir-remap "$GITHUB_WORKSPACE/target" \
|
||||
--status-level all --final-status-level all --failure-output final
|
||||
|
||||
- name: Verify e2e smoke execution completeness
|
||||
run: python3 scripts/ci_test_artifact.py evidence "$RUNNER_TEMP/rustfs-e2e-smoke-list.json" target/nextest/e2e-smoke/junit.xml
|
||||
|
||||
- name: Upload e2e smoke diagnostics
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: e2e-smoke-diagnostics-${{ github.run_number }}
|
||||
name: e2e-smoke-diagnostics-${{ github.run_number }}-${{ github.run_attempt }}
|
||||
path: |
|
||||
${{ runner.temp }}/rustfs-e2e-smoke-logs/
|
||||
${{ runner.temp }}/rustfs-e2e-smoke-list.json
|
||||
artifacts/test-and-lint
|
||||
artifacts/workspace
|
||||
if-no-files-found: warn
|
||||
|
||||
- name: Upload e2e smoke JUnit report
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: e2e-smoke-junit-${{ github.run_number }}
|
||||
name: e2e-smoke-junit-${{ github.run_number }}-${{ github.run_attempt }}
|
||||
path: target/nextest/e2e-smoke/junit.xml
|
||||
if-no-files-found: warn
|
||||
|
||||
- name: Install tool build dependencies
|
||||
run: sudo apt-get install -y build-essential cmake pkg-config
|
||||
|
||||
- name: Install s3s-e2e test tool
|
||||
uses: taiki-e/cache-cargo-install-action@7447f04c51f2ba27ca35e7f1e28fab848c5b3ba7 # v2
|
||||
with:
|
||||
@@ -851,7 +1081,7 @@ jobs:
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: e2e-test-logs-${{ github.run_number }}
|
||||
name: e2e-test-logs-${{ github.run_number }}-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/rustfs-e2e-*/rustfs.log
|
||||
retention-days: 3
|
||||
|
||||
@@ -868,7 +1098,8 @@ jobs:
|
||||
github.event_name == 'merge_group' ||
|
||||
(github.event_name == 'push' && github.ref == 'refs/heads/main')
|
||||
needs: [ build-rustfs-debug-binary ]
|
||||
runs-on: sm-standard-2
|
||||
# Retain the producer image and fixture paths until the runtime pilot passes.
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 55
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
@@ -876,13 +1107,17 @@ jobs:
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
- name: Install verification runtime
|
||||
uses: ./.github/actions/test-runtime
|
||||
|
||||
- name: Download E2E test binaries
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
rust-version: stable
|
||||
cache-shared-key: ci-dev
|
||||
cache-save-if: 'false'
|
||||
install-build-packaging-tools: 'false'
|
||||
name: e2e-test-binaries
|
||||
path: ${{ runner.temp }}
|
||||
|
||||
- name: Verify E2E archive identity
|
||||
run: python3 scripts/ci_test_artifact.py verify --configuration e2e --archive "$RUNNER_TEMP/e2e-tests.tar.zst"
|
||||
|
||||
- name: Install network fault-injection tools
|
||||
run: |
|
||||
@@ -910,11 +1145,16 @@ jobs:
|
||||
|
||||
- name: Install mc
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
MC_VERSION: RELEASE.2025-08-13T08-35-41Z
|
||||
MC_SHA256: 01f866e9c5f9b87c2b09116fa5d7c06695b106242d829a8bb32990c00312e891
|
||||
run: |
|
||||
if ! command -v gh >/dev/null 2>&1; then
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y gh
|
||||
fi
|
||||
MC_BINARY="mc.linux-amd64.${MC_VERSION}"
|
||||
curl -fsSLo "$RUNNER_TEMP/mc" "https://github.com/minio/mc/releases/download/${MC_VERSION}/${MC_BINARY}"
|
||||
python3 .github/actions/setup/download_release_asset.py --repository minio/mc --tag "$MC_VERSION" --asset "$MC_BINARY" --output "$RUNNER_TEMP/mc" --sha256 "$MC_SHA256"
|
||||
echo "${MC_SHA256} $RUNNER_TEMP/mc" | sha256sum --check --status
|
||||
chmod +x "$RUNNER_TEMP/mc"
|
||||
echo "$RUNNER_TEMP" >> "$GITHUB_PATH"
|
||||
@@ -981,7 +1221,7 @@ jobs:
|
||||
env:
|
||||
NEXTEST_LISTING: ${{ runner.temp }}/rustfs-e2e-full-list.json
|
||||
run: |
|
||||
cargo nextest list --profile e2e-full -p e2e_test --message-format json > "${NEXTEST_LISTING}"
|
||||
cargo nextest list --profile e2e-full --archive-file "$RUNNER_TEMP/e2e-tests.tar.zst" --workspace-remap "$GITHUB_WORKSPACE" --extract-to "$GITHUB_WORKSPACE" --extract-overwrite --message-format json > "${NEXTEST_LISTING}"
|
||||
python3 ./scripts/check_test_wiring.py --check-profile e2e-full "${NEXTEST_LISTING}"
|
||||
|
||||
# Full single-node e2e lane (backlog#1149 ci-5). The e2e-full
|
||||
@@ -995,13 +1235,16 @@ jobs:
|
||||
RUSTFS_E2E_STARTUP_CAS_BUILD_MANIFEST: ${{ runner.temp }}/rustfs-startup-cas-input/rustfs.e2e-startup-cas-build.json
|
||||
RUSTFS_E2E_STARTUP_CAS_ARTIFACT_DIR: ${{ runner.temp }}/rustfs-startup-cas-evidence
|
||||
RUSTFS_HEAL_CHAOS_LOG_ROOT: ${{ runner.temp }}/rustfs-heal-chaos-logs
|
||||
run: python3 scripts/e2e_binary.py run --binary "$RUSTFS_E2E_STARTUP_CAS_BINARY" --features e2e-test-hooks -- cargo nextest run --profile e2e-full -p e2e_test
|
||||
run: python3 scripts/e2e_binary.py run --binary "$RUSTFS_E2E_STARTUP_CAS_BINARY" --features e2e-test-hooks -- cargo nextest run --profile e2e-full --cargo-metadata target/nextest/cargo-metadata.json --binaries-metadata target/nextest/binaries-metadata.json --workspace-remap "$GITHUB_WORKSPACE" --target-dir-remap "$GITHUB_WORKSPACE/target"
|
||||
|
||||
- name: Verify e2e full execution completeness
|
||||
run: python3 scripts/ci_test_artifact.py evidence "$RUNNER_TEMP/rustfs-e2e-full-list.json" target/nextest/e2e-full/junit.xml
|
||||
|
||||
- name: Upload junit
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: e2e-full-junit-${{ github.run_number }}
|
||||
name: e2e-full-junit-${{ github.run_number }}-${{ github.run_attempt }}
|
||||
path: |
|
||||
target/nextest/e2e-full/junit.xml
|
||||
${{ runner.temp }}/rustfs-e2e-full-list.json
|
||||
@@ -1011,7 +1254,7 @@ jobs:
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: fresh-startup-cas-evidence-${{ github.run_number }}
|
||||
name: fresh-startup-cas-evidence-${{ github.run_number }}-${{ github.run_attempt }}
|
||||
path: |
|
||||
${{ runner.temp }}/rustfs-startup-cas-evidence
|
||||
${{ runner.temp }}/rustfs-startup-cas-input/rustfs.e2e-startup-cas-build.json
|
||||
@@ -1022,7 +1265,7 @@ jobs:
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: e2e-full-heal-logs-${{ github.run_number }}
|
||||
name: e2e-full-heal-logs-${{ github.run_number }}-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/rustfs-heal-chaos-logs
|
||||
if-no-files-found: warn
|
||||
retention-days: 7
|
||||
@@ -1082,13 +1325,13 @@ jobs:
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: e2e-test-logs-rio-v2-${{ github.run_number }}
|
||||
name: e2e-test-logs-rio-v2-${{ github.run_number }}-${{ github.run_attempt }}
|
||||
path: /tmp/rustfs.log
|
||||
retention-days: 3
|
||||
|
||||
s3-implemented-tests:
|
||||
name: S3 Implemented Tests
|
||||
needs: [ build-rustfs-debug-binary, e2e-tests ]
|
||||
needs: [ build-rustfs-debug-binary ]
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
@@ -1125,7 +1368,7 @@ jobs:
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: s3tests-implemented-${{ github.run_number }}
|
||||
name: s3tests-implemented-${{ github.run_number }}-${{ github.run_attempt }}
|
||||
path: artifacts/s3tests-single/**
|
||||
if-no-files-found: ignore
|
||||
retention-days: 3
|
||||
@@ -1162,7 +1405,7 @@ jobs:
|
||||
# prebuilt debug binary (no cargo build), and s3-implemented-tests — which
|
||||
# already waits on e2e-tests — finishes later anyway, so a green PR's total
|
||||
# wall clock is unchanged.
|
||||
needs: [ build-rustfs-debug-binary, e2e-tests ]
|
||||
needs: [ build-rustfs-debug-binary ]
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
@@ -1207,7 +1450,7 @@ jobs:
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: s3tests-lifecycle-behavior-${{ github.run_number }}
|
||||
name: s3tests-lifecycle-behavior-${{ github.run_number }}-${{ github.run_attempt }}
|
||||
path: artifacts/s3tests-single/**
|
||||
if-no-files-found: ignore
|
||||
retention-days: 3
|
||||
@@ -1219,8 +1462,13 @@ jobs:
|
||||
- classify-changes
|
||||
- typos
|
||||
- quick-checks
|
||||
- test-and-lint
|
||||
- workspace-lint-and-build
|
||||
- workspace-tests
|
||||
- workspace-shards
|
||||
- workspace-shard-evidence
|
||||
- core-contract
|
||||
- connect-feature-boundaries
|
||||
- fips-provider
|
||||
- test-and-lint-rio-v2
|
||||
- test-and-lint-protocols
|
||||
- build-rustfs-debug-binary
|
||||
@@ -1248,10 +1496,13 @@ jobs:
|
||||
needs:
|
||||
- classify-changes
|
||||
- connect-feature-boundaries
|
||||
- fips-provider
|
||||
- required-checks
|
||||
- typos
|
||||
- quick-checks
|
||||
- test-and-lint
|
||||
- workspace-lint-and-build
|
||||
- workspace-tests
|
||||
- core-contract
|
||||
- test-and-lint-rio-v2
|
||||
- test-and-lint-protocols
|
||||
- build-rustfs-debug-binary
|
||||
|
||||
@@ -95,7 +95,7 @@ jobs:
|
||||
|
||||
- name: Run CLA Bot
|
||||
if: github.event_name != 'merge_group'
|
||||
uses: overtrue/cla-bot@5e5929846a60cfb42b16d0d38e135ef67d94fc88 # v0.0.9
|
||||
uses: overtrue/cla-bot@4bd1387c19691168c42fcf5edfa17b15819b2ab3 # v0.1.1
|
||||
with:
|
||||
github-token: ${{ github.token }}
|
||||
registry-token: ${{ steps.registry-token.outputs.token }}
|
||||
|
||||
@@ -17,8 +17,8 @@
|
||||
# Each selected test starts a real localhost cluster via
|
||||
# `RustFSTestClusterEnvironment` (4 processes; 4 drives per node unless the
|
||||
# case is a two-site 4-node 1-drive pair or a 4-node upgrade). Membership is
|
||||
# `[profile.e2e-distributed]` in `.config/nextest.toml`. Storage-sensitive PRs,
|
||||
# nightly runs, and manual dispatches all execute the same fail-closed suite.
|
||||
# `[profile.e2e-distributed]` in `.config/nextest.toml`. The nightly schedule
|
||||
# and manual dispatches execute the same fail-closed suite.
|
||||
# Upgrade cases download the same pinned previous release as e2e-upgrade.yml.
|
||||
#
|
||||
# Isolated pool filesystems: expand/decommission/rebalance cases require
|
||||
@@ -33,32 +33,6 @@
|
||||
name: e2e-distributed
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- "Cargo.lock"
|
||||
- "Cargo.toml"
|
||||
- ".config/nextest.toml"
|
||||
- ".config/e2e-distributed-selection.txt"
|
||||
- ".github/workflows/e2e-distributed.yml"
|
||||
- "crates/audit/**"
|
||||
- "crates/common/**"
|
||||
- "crates/config/**"
|
||||
- "crates/e2e_test/**"
|
||||
- "crates/ecstore/**"
|
||||
- "crates/filemeta/**"
|
||||
- "crates/heal/**"
|
||||
- "crates/iam/**"
|
||||
- "crates/lock/**"
|
||||
- "crates/madmin/**"
|
||||
- "crates/notify/**"
|
||||
- "crates/replication/**"
|
||||
- "crates/s3-client/**"
|
||||
- "crates/s3-ops/**"
|
||||
- "crates/s3-types/**"
|
||||
- "crates/scanner/**"
|
||||
- "crates/storage-api/**"
|
||||
- "crates/utils/**"
|
||||
- "rustfs/**"
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
filter:
|
||||
|
||||
@@ -12,30 +12,19 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Single-node upgrade and rollback lane.
|
||||
#
|
||||
# The weekly Monday schedule, version tags, and workflow_dispatch run this
|
||||
# suite. Distributed upgrade cases stay on the nightly e2e-distributed lane.
|
||||
|
||||
name: Upgrade Compatibility
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- ".github/workflows/e2e-upgrade.yml"
|
||||
- "crates/e2e_test/src/common.rs"
|
||||
- "crates/e2e_test/src/distributed/**"
|
||||
- "crates/e2e_test/src/fake_s3_target/**"
|
||||
- "crates/e2e_test/src/lib.rs"
|
||||
- "crates/e2e_test/src/replication_extension_test.rs"
|
||||
- "crates/e2e_test/src/rpc_fault_proxy.rs"
|
||||
- "crates/e2e_test/src/upgrade_compatibility_test.rs"
|
||||
- "crates/ecstore/**"
|
||||
- "crates/filemeta/**"
|
||||
- "crates/heal/**"
|
||||
- "crates/kms/**"
|
||||
- "crates/storage-api/**"
|
||||
- "rustfs/**"
|
||||
- "Cargo.lock"
|
||||
push:
|
||||
tags:
|
||||
- "[0-9]*.[0-9]*.[0-9]*"
|
||||
schedule:
|
||||
# 03:17 UTC every Monday.
|
||||
- cron: "17 3 * * 1"
|
||||
workflow_dispatch:
|
||||
|
||||
@@ -44,7 +33,7 @@ permissions:
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
|
||||
@@ -12,44 +12,13 @@ concurrency:
|
||||
jobs:
|
||||
collect:
|
||||
if: github.repository == 'rustfs/rustfs'
|
||||
runs-on: smoke-testing
|
||||
# Health remains available when either laboratory fleet is offline.
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Ensure GitHub CLI
|
||||
shell: bash
|
||||
run: |
|
||||
if command -v gh >/dev/null 2>&1; then
|
||||
gh --version
|
||||
exit 0
|
||||
fi
|
||||
|
||||
gh_version="2.101.0"
|
||||
case "$(uname -m)" in
|
||||
x86_64) gh_arch="amd64" ;;
|
||||
aarch64|arm64) gh_arch="arm64" ;;
|
||||
*)
|
||||
echo "Unsupported Linux architecture for GitHub CLI: $(uname -m)" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
gh_install_root="${RUNNER_TEMP:-$PWD}/gh-cli"
|
||||
rm -rf "$gh_install_root"
|
||||
mkdir -p "$gh_install_root/bin"
|
||||
archive="$gh_install_root/gh.tar.gz"
|
||||
curl --fail --location --retry 3 \
|
||||
"https://github.com/cli/cli/releases/download/v${gh_version}/gh_${gh_version}_linux_${gh_arch}.tar.gz" \
|
||||
--output "$archive"
|
||||
tar -xzf "$archive" -C "$gh_install_root"
|
||||
install -m 0755 \
|
||||
"$gh_install_root/gh_${gh_version}_linux_${gh_arch}/bin/gh" \
|
||||
"$gh_install_root/bin/gh"
|
||||
echo "$gh_install_root/bin" >> "$GITHUB_PATH"
|
||||
export PATH="$gh_install_root/bin:$PATH"
|
||||
gh --version
|
||||
- name: Collect and publish verified chain health
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
|
||||
@@ -87,18 +87,206 @@ jobs:
|
||||
install-build-packaging-tools: 'false'
|
||||
install-test-tools: 'false'
|
||||
|
||||
- name: Download static console assets
|
||||
shell: bash
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
# Packaging tools are off on this lane, so setup does not install unzip.
|
||||
if ! command -v unzip >/dev/null 2>&1; then
|
||||
sudo apt-get update -qq
|
||||
sudo apt-get install -y -qq unzip
|
||||
fi
|
||||
if ! command -v unzip >/dev/null 2>&1; then
|
||||
echo "unzip is required to extract console assets" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p ./rustfs/static
|
||||
|
||||
verify_sha256() {
|
||||
local expected="$1"
|
||||
local file="$2"
|
||||
local actual=""
|
||||
local actual_line=""
|
||||
|
||||
if command -v sha256sum >/dev/null 2>&1; then
|
||||
actual_line=$(sha256sum -- "$file") || return 1
|
||||
elif command -v shasum >/dev/null 2>&1; then
|
||||
actual_line=$(shasum -a 256 -- "$file") || return 1
|
||||
else
|
||||
echo "No SHA-256 verification tool found" >&2
|
||||
return 1
|
||||
fi
|
||||
actual="${actual_line%%[[:space:]]*}"
|
||||
|
||||
if [ "$actual" = "$expected" ]; then
|
||||
echo "SHA256 verified OK: $actual"
|
||||
return 0
|
||||
else
|
||||
echo "SHA256 mismatch: expected=$expected actual=$actual" >&2
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
download_console_assets() {
|
||||
local curl_bin="curl"
|
||||
local python_bin="python3"
|
||||
local console_api="https://api.github.com/repos/rustfs/console/releases/latest"
|
||||
local console_json="console-release.json"
|
||||
local console_url
|
||||
local console_sha256
|
||||
local curl_auth_args=()
|
||||
console_tag=""
|
||||
|
||||
chmod +w ./rustfs/static/LICENSE || true
|
||||
|
||||
if ! command -v "$python_bin" >/dev/null 2>&1; then
|
||||
python_bin="python"
|
||||
fi
|
||||
if ! command -v "$python_bin" >/dev/null 2>&1; then
|
||||
echo "No Python interpreter found for release metadata parsing" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [[ -n "${GITHUB_TOKEN:-}" ]]; then
|
||||
curl_auth_args=(-H "Authorization: Bearer ${GITHUB_TOKEN}" -H "X-GitHub-Api-Version: 2022-11-28")
|
||||
fi
|
||||
|
||||
"$curl_bin" "${curl_auth_args[@]}" --fail -L "$console_api" \
|
||||
-o "$console_json" --retry 3 --retry-delay 5 --max-time 300 || return 1
|
||||
|
||||
read -r console_tag console_url console_sha256 < <("$python_bin" - "$console_json" <<'PY'
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
|
||||
with open(sys.argv[1], encoding="utf-8") as handle:
|
||||
release = json.load(handle)
|
||||
|
||||
tag = release.get("tag_name", "") or "unknown"
|
||||
|
||||
for asset in release.get("assets", []):
|
||||
name = asset.get("name", "")
|
||||
digest = asset.get("digest", "")
|
||||
url = asset.get("browser_download_url", "")
|
||||
if name.startswith("rustfs-console-") and name.endswith(".zip") and digest.startswith("sha256:") and url:
|
||||
sha256 = digest.split(":", 1)[1]
|
||||
if not re.fullmatch(r"[0-9a-fA-F]{64}", sha256):
|
||||
raise SystemExit(f"console zip asset has invalid sha256 digest: {sha256}")
|
||||
sys.stdout.buffer.write(f"{tag} {url} {sha256}\n".encode("utf-8"))
|
||||
break
|
||||
else:
|
||||
raise SystemExit("no console zip asset with sha256 digest found")
|
||||
PY
|
||||
) || return 1
|
||||
|
||||
if [[ -z "$console_url" || -z "$console_sha256" ]]; then
|
||||
echo "Console release metadata is missing URL or SHA-256 digest" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
echo "Console release: ${console_tag}"
|
||||
echo "Downloading console asset: ${console_url}"
|
||||
"$curl_bin" --fail -L "$console_url" -o console.zip --retry 3 --retry-delay 5 --max-time 300 || return 1
|
||||
verify_sha256 "$console_sha256" console.zip || return 2
|
||||
unzip -o console.zip -d ./rustfs/static || return 2
|
||||
}
|
||||
|
||||
if download_console_assets; then
|
||||
rm -f console.zip console-release.json
|
||||
else
|
||||
status=$?
|
||||
rm -f console.zip console-release.json
|
||||
if [[ "$status" -eq 2 ]]; then
|
||||
echo "Console asset integrity verification failed" >&2
|
||||
fi
|
||||
echo "Failed to download verified console assets" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ! -s ./rustfs/static/index.html ]]; then
|
||||
echo "Console asset archive is missing static/index.html" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
asset_count=$(find ./rustfs/static -type f | wc -l | tr -d '[:space:]')
|
||||
echo "Console assets ready: version=${console_tag:-unknown}, ${asset_count} files extracted to ./rustfs/static"
|
||||
|
||||
- name: Build RustFS
|
||||
# Pin the glibc baseline (`<target>.<glibc>` for cargo-zigbuild) so
|
||||
# nightlies run on older distros such as RHEL/Rocky 9 (glibc 2.34); #8062.
|
||||
run: cargo zigbuild --release --locked --target x86_64-unknown-linux-gnu.2.31 -p rustfs --bins
|
||||
shell: bash
|
||||
run: |
|
||||
# Force rebuild by touching build.rs so a restored cache cannot
|
||||
# reuse a rustfs artifact compiled before these console assets existed.
|
||||
touch rustfs/build.rs
|
||||
cargo zigbuild --release --locked --target x86_64-unknown-linux-gnu.2.31 -p rustfs --bin rustfs
|
||||
|
||||
- name: Verify glibc symbol baseline
|
||||
run: |
|
||||
python3 scripts/release/check_glibc.py 2.31 \
|
||||
target/x86_64-unknown-linux-gnu/release/rustfs \
|
||||
target/x86_64-unknown-linux-gnu/release/rustfs-cli
|
||||
target/x86_64-unknown-linux-gnu/release/rustfs
|
||||
target/x86_64-unknown-linux-gnu/release/rustfs --version | grep -Fx 'build target : x86_64-unknown-linux-gnu'
|
||||
|
||||
- name: Verify console
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
find_free_port() {
|
||||
python3 - <<'PY'
|
||||
import socket
|
||||
|
||||
with socket.socket() as sock:
|
||||
sock.bind(("127.0.0.1", 0))
|
||||
print(sock.getsockname()[1])
|
||||
PY
|
||||
}
|
||||
|
||||
binary="target/x86_64-unknown-linux-gnu/release/rustfs"
|
||||
data_dir="$(mktemp -d)"
|
||||
log_file="${RUNNER_TEMP}/rustfs-console-smoke.log"
|
||||
server_pid=""
|
||||
trap '
|
||||
if [[ -n "${server_pid:-}" ]]; then
|
||||
kill "$server_pid" 2>/dev/null || true
|
||||
wait "$server_pid" 2>/dev/null || true
|
||||
fi
|
||||
rm -rf "$data_dir"
|
||||
' EXIT
|
||||
|
||||
if [[ ! -x "$binary" ]]; then
|
||||
echo "rustfs binary not found: ${binary}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
api_port="$(find_free_port)"
|
||||
console_port="$(find_free_port)"
|
||||
console_url="http://127.0.0.1:${console_port}/rustfs/console/"
|
||||
|
||||
"$binary" server \
|
||||
--address "127.0.0.1:${api_port}" \
|
||||
--console-enable \
|
||||
--console-address "127.0.0.1:${console_port}" \
|
||||
--access-key console-smoke \
|
||||
--secret-key console-smoke-secret \
|
||||
"$data_dir" >"$log_file" 2>&1 &
|
||||
server_pid=$!
|
||||
|
||||
response=""
|
||||
for _ in {1..40}; do
|
||||
response="$(curl --silent --output /dev/null --write-out '%{http_code} %{content_type}' "$console_url" || true)"
|
||||
if [[ "$response" == 200\ text/html* ]]; then
|
||||
exit 0
|
||||
fi
|
||||
sleep 0.25
|
||||
done
|
||||
|
||||
echo "Console endpoint did not return 200 text/html: $response" >&2
|
||||
cat "$log_file"
|
||||
exit 1
|
||||
|
||||
- name: Build DEB package
|
||||
id: deb
|
||||
shell: bash
|
||||
@@ -346,12 +534,12 @@ jobs:
|
||||
path: ${{ steps.publish.outputs.candidate_file }}
|
||||
if-no-files-found: error
|
||||
|
||||
# Publish the deb/rpm pair to the auto-testing repo's `assets` branch so
|
||||
# engineers can download and install the nightly directly. The branch is
|
||||
# a single-commit orphan rewritten on every build, which keeps the repo
|
||||
# small while the latest files stay reachable at stable raw URLs.
|
||||
# Publish the deb/rpm pair as assets of the rolling `nightly` release on
|
||||
# rustfs/auto-testing (see scripts/release/publish_nightly_assets.sh).
|
||||
# The dispatch below runs only after this upload and the immutable R2
|
||||
# candidate both succeed. It does not replace rustfs-functional-chain.yml,
|
||||
# which still starts from this workflow's workflow_run completion.
|
||||
# kms-vault-lane and kms-vault-ha-failover stay parallel with this job.
|
||||
- name: Publish packages to auto-testing release assets
|
||||
env:
|
||||
ASSETS_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
@@ -360,6 +548,90 @@ jobs:
|
||||
DEB_DATE: ${{ steps.deb.outputs.deb_date }}
|
||||
BUILD_REF: ${{ env.NIGHTLY_BUILD_REF }}
|
||||
run: bash scripts/release/publish_nightly_assets.sh
|
||||
|
||||
# Cross-repo repository_dispatch cannot use GITHUB_TOKEN. The secret is a
|
||||
# PAT or GitHub App token for rustfs/auto-testing: classic `repo` scope,
|
||||
# or fine-grained Contents write (the permission GitHub checks for
|
||||
# POST /repos/{owner}/{repo}/dispatches). A dispatch failure fails this
|
||||
# step, and therefore this job, so the scheduled failure alert fires.
|
||||
# GitHub allows at most 10 top-level client_payload properties, so the
|
||||
# schema 2 candidate object is nested under `candidate` and the release
|
||||
# fields sit beside it.
|
||||
- name: Dispatch nightly candidate to auto-testing
|
||||
if: ${{ steps.publish.outputs.candidate_file != '' }}
|
||||
env:
|
||||
DISPATCH_TOKEN: ${{ secrets.AUTO_TESTING_DISPATCH_TOKEN }}
|
||||
CANDIDATE_FILE: ${{ steps.publish.outputs.candidate_file }}
|
||||
DEB_DATE: ${{ steps.deb.outputs.deb_date }}
|
||||
SOURCE_REPO: ${{ github.repository }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
: "${DISPATCH_TOKEN:?AUTO_TESTING_DISPATCH_TOKEN is required to dispatch rustfs/auto-testing}"
|
||||
: "${CANDIDATE_FILE:?nightly candidate manifest is required}"
|
||||
: "${DEB_DATE:?DEB_DATE is required}"
|
||||
[[ "${DEB_DATE}" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]]
|
||||
[[ "${SOURCE_REPO}" == "rustfs/rustfs" ]]
|
||||
|
||||
payload="${RUNNER_TEMP}/nightly-candidate-dispatch-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.json"
|
||||
jq -n --slurpfile candidate "${CANDIDATE_FILE}" \
|
||||
--arg event_type "nightly-candidate-ready" \
|
||||
--arg source_repo "${SOURCE_REPO}" \
|
||||
--arg latest_alias_url "https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb" \
|
||||
--arg auto_testing_release "https://github.com/rustfs/auto-testing/releases/tag/nightly" \
|
||||
--arg channel "nightly" \
|
||||
--arg deb_date "${DEB_DATE}" \
|
||||
--arg nightly_workflow ".github/workflows/nightly-gnu.yml" \
|
||||
'{
|
||||
event_type: $event_type,
|
||||
source_repo: $source_repo,
|
||||
latest_alias_url: $latest_alias_url,
|
||||
auto_testing_release: $auto_testing_release,
|
||||
channel: $channel,
|
||||
deb_date: $deb_date,
|
||||
nightly_workflow: $nightly_workflow,
|
||||
candidate: $candidate[0]
|
||||
}' > "${payload}"
|
||||
if ! jq -e '
|
||||
.event_type == "nightly-candidate-ready"
|
||||
and .source_repo == "rustfs/rustfs"
|
||||
and .channel == "nightly"
|
||||
and .deb_date == env.DEB_DATE
|
||||
and .nightly_workflow == ".github/workflows/nightly-gnu.yml"
|
||||
and .latest_alias_url == "https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb"
|
||||
and .auto_testing_release == "https://github.com/rustfs/auto-testing/releases/tag/nightly"
|
||||
and .candidate.schema == 2
|
||||
and ((.candidate | keys | sort) == ["build_run_attempt","build_run_id","package_sha256","package_url","schema","source_ref","source_sha","workflow_sha"])
|
||||
and ((keys | sort) == ["auto_testing_release","candidate","channel","deb_date","event_type","latest_alias_url","nightly_workflow","source_repo"])
|
||||
' "${payload}" >/dev/null; then
|
||||
echo "nightly dispatch payload does not match the nightly-candidate-ready contract" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
body="${RUNNER_TEMP}/nightly-candidate-dispatch-body-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.json"
|
||||
jq -n --slurpfile client_payload "${payload}" \
|
||||
'{event_type: "nightly-candidate-ready", client_payload: $client_payload[0]}' > "${body}"
|
||||
response="${RUNNER_TEMP}/nightly-candidate-dispatch-response-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.txt"
|
||||
status="$(curl -sS --retry 3 --connect-timeout 15 --max-time 60 \
|
||||
-o "${response}" -w '%{http_code}' \
|
||||
-X POST \
|
||||
-H "Authorization: Bearer ${DISPATCH_TOKEN}" \
|
||||
-H "Accept: application/vnd.github+json" \
|
||||
-H "X-GitHub-Api-Version: 2022-11-28" \
|
||||
-H "Content-Type: application/json" \
|
||||
--data-binary @"${body}" \
|
||||
https://api.github.com/repos/rustfs/auto-testing/dispatches)" || {
|
||||
echo "repository_dispatch request to rustfs/auto-testing failed before an HTTP status" >&2
|
||||
exit 1
|
||||
}
|
||||
if [[ "${status}" != "204" ]]; then
|
||||
echo "repository_dispatch to rustfs/auto-testing failed: HTTP ${status}" >&2
|
||||
if [[ -s "${response}" ]]; then
|
||||
cat "${response}" >&2
|
||||
fi
|
||||
exit 1
|
||||
fi
|
||||
echo "dispatched nightly-candidate-ready to rustfs/auto-testing (deb_date=${DEB_DATE}, run=${GITHUB_RUN_ID}, attempt=${GITHUB_RUN_ATTEMPT})"
|
||||
# Live-Vault lane for the rustfs-kms suite (rustfs/backlog#1774).
|
||||
#
|
||||
# RUSTFS_KMS_VAULT_TOKEN is the single switch that adds the Vault KV2 and
|
||||
|
||||
@@ -93,6 +93,12 @@ jobs:
|
||||
printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
|
||||
} >> "${GITHUB_ENV}"
|
||||
|
||||
- name: Checkout environment guard
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
path: rustfs-tools
|
||||
|
||||
- name: Checkout auto-testing scripts
|
||||
timeout-minutes: 5
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
@@ -118,6 +124,13 @@ jobs:
|
||||
echo "auto-testing: ${AUTO_TESTING_SHA}"
|
||||
df -h /data | tail -1
|
||||
|
||||
- name: Admit clean shared environment
|
||||
id: environment
|
||||
timeout-minutes: 5
|
||||
env:
|
||||
FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-matrix.json
|
||||
run: python3 rustfs-tools/scripts/functional_environment.py enter --suite fault-tolerance
|
||||
|
||||
- name: Cleanup environment (before)
|
||||
timeout-minutes: 5
|
||||
if: ${{ inputs.cleanup_before != 'false' }}
|
||||
@@ -215,7 +228,24 @@ jobs:
|
||||
if-no-files-found: warn
|
||||
|
||||
- name: Cleanup environment (after)
|
||||
id: environment_cleanup
|
||||
timeout-minutes: 5
|
||||
if: ${{ always() && inputs.cleanup_after != 'false' }}
|
||||
if: ${{ always() && inputs.cleanup_after != 'false' && steps.environment.outcome == 'success' }}
|
||||
run: |
|
||||
./auto-testing/rustfs-fault-tolerance-test.sh --cleanup -y --log-file "${LOG_FILE}" || true
|
||||
./auto-testing/rustfs-fault-tolerance-test.sh --cleanup -y --log-file "${LOG_FILE}"
|
||||
|
||||
- name: Release environment only after successful cleanup
|
||||
if: ${{ always() && steps.environment.outcome == 'success' }}
|
||||
timeout-minutes: 5
|
||||
env:
|
||||
FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-matrix.json
|
||||
CLEANUP_OUTCOME: ${{ steps.environment_cleanup.outcome }}
|
||||
run: python3 rustfs-tools/scripts/functional_environment.py finish --suite fault-tolerance --cleanup-outcome "$CLEANUP_OUTCOME"
|
||||
|
||||
- name: Retain environment ownership and cleanup receipt
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: functional-environment-matrix-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/environment-matrix.json
|
||||
if-no-files-found: error
|
||||
|
||||
@@ -50,10 +50,6 @@ on:
|
||||
description: 'Reset the nodes after the test (DESTROYS test data/config)'
|
||||
type: boolean
|
||||
default: true
|
||||
repository_dispatch:
|
||||
# Chain handoff: dispatched when the replication suite finishes, ahead of
|
||||
# the performance suite.
|
||||
types: [rustfs-chain-fault-tolerance]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -80,7 +76,7 @@ jobs:
|
||||
fault-tolerance-test:
|
||||
runs-on: smoke-testing
|
||||
timeout-minutes: 60
|
||||
if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
|
||||
if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' }}
|
||||
steps:
|
||||
- name: Checkout repository (for report parser)
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
@@ -136,6 +132,13 @@ jobs:
|
||||
echo "auto-testing: ${AUTO_TESTING_SHA}"
|
||||
df -h /data | tail -1
|
||||
|
||||
- name: Admit clean shared environment
|
||||
id: environment
|
||||
timeout-minutes: 5
|
||||
env:
|
||||
FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-fault-tolerance.json
|
||||
run: python3 scripts/functional_environment.py enter --suite fault-tolerance
|
||||
|
||||
- name: Cleanup environment (before)
|
||||
timeout-minutes: 5
|
||||
if: ${{ inputs.cleanup_before != 'false' }}
|
||||
@@ -194,21 +197,21 @@ jobs:
|
||||
jq -e --arg testing_sha "${AUTO_TESTING_SHA}" '
|
||||
.schema_version == 1 and
|
||||
.auto_testing_sha == $testing_sha and
|
||||
.selected_scenarios == ["A", "B", "C", "C2", "D", "E"] and
|
||||
.selected_scenarios == ["A", "B", "C", "C2", "D", "E", "F"] and
|
||||
.complete == true and
|
||||
.counts.expected == 44 and
|
||||
.counts.seen == 44 and
|
||||
.counts.expected == 54 and
|
||||
.counts.seen == 54 and
|
||||
.counts.harness_errors == 0 and
|
||||
(.expected_cases | length) == 44 and
|
||||
(.expected_cases | unique | length) == 44 and
|
||||
(.seen_cases | length) == 44 and
|
||||
(.seen_cases | unique | length) == 44 and
|
||||
(.expected_cases | length) == 54 and
|
||||
(.expected_cases | unique | length) == 54 and
|
||||
(.seen_cases | length) == 54 and
|
||||
(.seen_cases | unique | length) == 54 and
|
||||
(.seen_cases | sort) == (.expected_cases | sort) and
|
||||
(.missing_cases | length) == 0 and
|
||||
(.harness_errors | length) == 0 and
|
||||
(.cases | length) == 44 and
|
||||
(.cases | length) == 54 and
|
||||
([.cases[].case_id] | sort) == (.expected_cases | sort) and
|
||||
([.cases[].case_id] | unique | length) == 44 and
|
||||
([.cases[].case_id] | unique | length) == 54 and
|
||||
all(.cases[]; .verdict == "pass" or .verdict == "known-divergence" or .verdict == "UNEXPECTED") and
|
||||
.counts.unexpected == ([.cases[] | select(.verdict == "UNEXPECTED")] | length) and
|
||||
.counts.known_divergence == ([.cases[] | select(.verdict == "known-divergence")] | length)
|
||||
@@ -305,58 +308,27 @@ jobs:
|
||||
if-no-files-found: warn
|
||||
|
||||
- name: Cleanup environment (after)
|
||||
id: environment_cleanup
|
||||
timeout-minutes: 5
|
||||
if: ${{ always() && inputs.cleanup_after != 'false' }}
|
||||
if: ${{ always() && inputs.cleanup_after != 'false' && steps.environment.outcome == 'success' }}
|
||||
run: |
|
||||
./auto-testing/rustfs-fault-tolerance-test.sh --cleanup -y --log-file "${LOG_FILE}" || true
|
||||
./auto-testing/rustfs-fault-tolerance-test.sh --cleanup -y --log-file "${LOG_FILE}"
|
||||
|
||||
- name: "Continue functional chain (next: Performance)"
|
||||
# Only chain-triggered runs forward to the next suite; standalone
|
||||
# workflow_dispatch runs stop after their own cleanup. A failed
|
||||
# handoff retries, then files an alert issue in rustfs/backlog.
|
||||
if: ${{ always() && github.event_name == 'repository_dispatch' }}
|
||||
continue-on-error: true
|
||||
- name: Release environment only after successful cleanup
|
||||
if: ${{ always() && steps.environment.outcome == 'success' }}
|
||||
timeout-minutes: 5
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
run: |
|
||||
set -uo pipefail
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
|
||||
exit 1
|
||||
fi
|
||||
DISPATCHED=0
|
||||
for attempt in 1 2 3; do
|
||||
if gh api --method POST repos/rustfs/rustfs/dispatches \
|
||||
-f event_type='rustfs-chain-performance' \
|
||||
-F 'client_payload[from_suite]=fault-tolerance'; then
|
||||
echo "dispatched next suite Performance (attempt ${attempt})"
|
||||
DISPATCHED=1
|
||||
break
|
||||
fi
|
||||
echo "dispatch attempt ${attempt} failed; retrying in ${attempt}0s" >&2
|
||||
sleep "${attempt}0"
|
||||
done
|
||||
if [ "${DISPATCHED:-0}" -ne 1 ]; then
|
||||
echo "ERROR: functional chain stalled: could not dispatch Performance after 3 attempts" >&2
|
||||
TITLE="[functional][chain] stalled after fault-tolerance (run ${GITHUB_RUN_ID})"
|
||||
BODY_FILE="$(mktemp)"
|
||||
{
|
||||
echo "The functional chain could not hand off from **fault-tolerance** to **Performance** after 3 attempts."
|
||||
echo ""
|
||||
echo "- Failed suite job: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
|
||||
echo "- Expected next event: 'rustfs-chain-performance'"
|
||||
echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
|
||||
echo "- Recovery: re-dispatch manually with"
|
||||
FENCE="$(printf "\x60\x60\x60")"; echo " ${FENCE}"
|
||||
echo " gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-performance'"
|
||||
FENCE="$(printf "\x60\x60\x60")"; echo " ${FENCE}"
|
||||
} > "${BODY_FILE}"
|
||||
gh issue create -R rustfs/backlog --title "${TITLE}" \
|
||||
--body-file "${BODY_FILE}" --label functional-test \
|
||||
|| gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}" \
|
||||
|| echo "could not file the stall alert issue either; check the token" >&2
|
||||
exit 1
|
||||
fi
|
||||
FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-fault-tolerance.json
|
||||
CLEANUP_OUTCOME: ${{ steps.environment_cleanup.outcome }}
|
||||
run: python3 scripts/functional_environment.py finish --suite fault-tolerance --cleanup-outcome "$CLEANUP_OUTCOME"
|
||||
|
||||
- name: Retain environment ownership and cleanup receipt
|
||||
if: ${{ always() }}
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: functional-environment-fault-tolerance-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/environment-fault-tolerance.json
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Notify on failure
|
||||
if: failure()
|
||||
@@ -391,3 +363,13 @@ jobs:
|
||||
name: functional-chain-fault-tolerance-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/chain-fault-tolerance-${{ github.run_id }}-${{ github.run_attempt }}/fault-tolerance.json
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Enforce suite and report result
|
||||
if: ${{ always() }}
|
||||
env:
|
||||
SUITE_OUTCOME: ${{ steps.test.outcome }}
|
||||
REPORT_OUTCOME: ${{ steps.chain_report.outcome }}
|
||||
run: |
|
||||
test "$SUITE_OUTCOME" = success
|
||||
test "$REPORT_OUTCOME" = success
|
||||
python3 scripts/functional_chain_evidence.py verify-report --suite fault-tolerance --report "${FUNCTIONAL_ARTIFACTS_DIR}/suite.log"
|
||||
|
||||
@@ -13,12 +13,23 @@
|
||||
# limitations under the License.
|
||||
|
||||
# Reusable workflows run from this driver's commit in one Actions run. Each
|
||||
# suite still runs after an earlier suite fails; the final job requires all twelve.
|
||||
# selected suite still runs after an earlier failure; the final job checks its plan.
|
||||
name: RustFS Functional Chain
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
validation_plan:
|
||||
description: Registered cadence plan (existing suite frequencies are preserved)
|
||||
type: choice
|
||||
options: [nightly, weekly, monthly]
|
||||
required: false
|
||||
default: nightly
|
||||
suites:
|
||||
description: Comma-separated suite IDs, or empty for the complete registered plan
|
||||
type: string
|
||||
required: false
|
||||
default: ''
|
||||
build_run_id:
|
||||
description: Successful nightly or validation package run; empty for package-only build
|
||||
type: string
|
||||
@@ -216,10 +227,25 @@ jobs:
|
||||
outputs:
|
||||
manifest: ${{ steps.candidate.outputs.manifest }}
|
||||
performance_ready: ${{ steps.perf_probe.outputs.performance_ready }}
|
||||
plan: ${{ steps.selection.outputs.plan }}
|
||||
suites: ${{ steps.selection.outputs.suites }}
|
||||
optional_suites: ${{ steps.selection.outputs.optional_suites }}
|
||||
steps:
|
||||
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Select registered suites
|
||||
id: selection
|
||||
env:
|
||||
VALIDATION_PLAN: ${{ inputs.validation_plan || 'nightly' }}
|
||||
VALIDATION_SUITES: ${{ inputs.suites }}
|
||||
run: python3 scripts/validation_registry.py select --output "${RUNNER_TEMP}/suite-plan.json"
|
||||
- name: Retain selected responsibilities
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: functional-plan-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/suite-plan.json
|
||||
if-no-files-found: error
|
||||
- name: Resolve published candidate
|
||||
id: candidate
|
||||
env:
|
||||
@@ -239,6 +265,7 @@ jobs:
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Check shared functional fleet runner before scheduling suites
|
||||
if: ${{ contains(fromJSON(steps.selection.outputs.environments), 'functional-fleet') }}
|
||||
# The eleven functional suites run on the smoke-testing fleet; a hard
|
||||
# requirement. The performance lane runs on pf-testing, which is a
|
||||
# separate fleet - probe it without failing the whole chain so an
|
||||
@@ -248,15 +275,26 @@ jobs:
|
||||
run: python3 scripts/check_functional_runners.py smoke-testing
|
||||
|
||||
- name: Probe performance fleet runner
|
||||
if: ${{ contains(fromJSON(steps.selection.outputs.suites), 'performance') }}
|
||||
id: perf_probe
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
PERFORMANCE_OPTIONAL: ${{ contains(fromJSON(steps.selection.outputs.optional_suites), 'performance') }}
|
||||
run: |
|
||||
if python3 scripts/check_functional_runners.py pf-testing; then
|
||||
echo "performance_ready=online" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "performance_ready=offline" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
# TEMPORARY: the pf-testing environment is broken, so the
|
||||
# performance lane is disabled until it is repaired. Forcing
|
||||
# offline keeps the chain on its designed skip path: the lane job
|
||||
# is skipped, aggregate receives --allow-skipped performance
|
||||
# (required=false in scheduled-validations.json), and the report
|
||||
# records the optional-lane skip with its reason.
|
||||
# Restore the probe below once the environment is healthy again.
|
||||
# if python3 scripts/check_functional_runners.py pf-testing; then
|
||||
# echo "performance_ready=online" >> "$GITHUB_OUTPUT"
|
||||
# else
|
||||
# echo "performance_ready=offline" >> "$GITHUB_OUTPUT"
|
||||
# test "$PERFORMANCE_OPTIONAL" = true
|
||||
# fi
|
||||
echo "performance_ready=offline" >> "$GITHUB_OUTPUT"
|
||||
|
||||
validation:
|
||||
if: ${{ github.event_name == 'workflow_dispatch' && !inputs.validation_runner_inspect_only && inputs.validation_retain_evidence_run_id == '' && inputs.validation_package_sha == '' && (inputs.validation_case != '' || inputs.validation_tools_only || inputs.validation_native_tools || inputs.validation_shared_tools || inputs.validation_manager_tools || inputs.validation_cleanup_run_id != '' || inputs.validation_cleanup_retained_evidence) }}
|
||||
@@ -580,6 +618,10 @@ jobs:
|
||||
> "${VALIDATION_ROOT}/shared-fixture-manifest.json"
|
||||
printf 'VALIDATION_HEAL_SHARED_DIRECTORY=%s\n' "${SHARED_DIRECTORY}" >> "${GITHUB_ENV}"
|
||||
|
||||
- name: Reject quarantined suite environment before canonical validation
|
||||
if: ${{ !inputs.validation_tools_only && inputs.validation_cleanup_run_id == '' && inputs.validation_case != 'environment.inspect' }}
|
||||
run: python3 scripts/functional_environment.py check --suite heal
|
||||
|
||||
- name: Run selected canonical validation case
|
||||
if: ${{ !inputs.validation_tools_only && inputs.validation_cleanup_run_id == '' && inputs.validation_case != 'environment.inspect' }}
|
||||
env:
|
||||
@@ -665,7 +707,7 @@ jobs:
|
||||
|
||||
upgrade:
|
||||
needs: [prepare]
|
||||
if: ${{ always() && needs.prepare.result == 'success' }}
|
||||
if: ${{ always() && needs.prepare.result == 'success' && contains(fromJSON(needs.prepare.outputs.suites || '[]'), 'upgrade') }}
|
||||
uses: ./.github/workflows/rustfs-upgrade-test.yml
|
||||
with:
|
||||
chain_manifest: ${{ needs.prepare.outputs.manifest }}
|
||||
@@ -673,7 +715,7 @@ jobs:
|
||||
|
||||
s3:
|
||||
needs: [prepare, upgrade]
|
||||
if: ${{ always() && needs.prepare.result == 'success' }}
|
||||
if: ${{ always() && needs.prepare.result == 'success' && contains(fromJSON(needs.prepare.outputs.suites || '[]'), 's3') }}
|
||||
uses: ./.github/workflows/rustfs-s3-compat-test.yml
|
||||
with:
|
||||
chain_manifest: ${{ needs.prepare.outputs.manifest }}
|
||||
@@ -681,7 +723,7 @@ jobs:
|
||||
|
||||
kms:
|
||||
needs: [prepare, s3]
|
||||
if: ${{ always() && needs.prepare.result == 'success' }}
|
||||
if: ${{ always() && needs.prepare.result == 'success' && contains(fromJSON(needs.prepare.outputs.suites || '[]'), 'kms') }}
|
||||
uses: ./.github/workflows/rustfs-kms-test.yml
|
||||
with:
|
||||
chain_manifest: ${{ needs.prepare.outputs.manifest }}
|
||||
@@ -689,7 +731,7 @@ jobs:
|
||||
|
||||
tier:
|
||||
needs: [prepare, kms]
|
||||
if: ${{ always() && needs.prepare.result == 'success' }}
|
||||
if: ${{ always() && needs.prepare.result == 'success' && contains(fromJSON(needs.prepare.outputs.suites || '[]'), 'tier') }}
|
||||
uses: ./.github/workflows/rustfs-tier-test.yml
|
||||
with:
|
||||
chain_manifest: ${{ needs.prepare.outputs.manifest }}
|
||||
@@ -697,7 +739,7 @@ jobs:
|
||||
|
||||
storage:
|
||||
needs: [prepare, tier]
|
||||
if: ${{ always() && needs.prepare.result == 'success' }}
|
||||
if: ${{ always() && needs.prepare.result == 'success' && contains(fromJSON(needs.prepare.outputs.suites || '[]'), 'storage') }}
|
||||
uses: ./.github/workflows/rustfs-storage-test.yml
|
||||
with:
|
||||
chain_manifest: ${{ needs.prepare.outputs.manifest }}
|
||||
@@ -705,7 +747,7 @@ jobs:
|
||||
|
||||
heal:
|
||||
needs: [prepare, storage]
|
||||
if: ${{ always() && needs.prepare.result == 'success' }}
|
||||
if: ${{ always() && needs.prepare.result == 'success' && contains(fromJSON(needs.prepare.outputs.suites || '[]'), 'heal') }}
|
||||
uses: ./.github/workflows/rustfs-heal-test.yml
|
||||
with:
|
||||
chain_manifest: ${{ needs.prepare.outputs.manifest }}
|
||||
@@ -713,7 +755,7 @@ jobs:
|
||||
|
||||
pool:
|
||||
needs: [prepare, heal]
|
||||
if: ${{ always() && needs.prepare.result == 'success' }}
|
||||
if: ${{ always() && needs.prepare.result == 'success' && contains(fromJSON(needs.prepare.outputs.suites || '[]'), 'pool') }}
|
||||
uses: ./.github/workflows/rustfs-pool-expand-test.yml
|
||||
with:
|
||||
chain_manifest: ${{ needs.prepare.outputs.manifest }}
|
||||
@@ -721,7 +763,7 @@ jobs:
|
||||
|
||||
security:
|
||||
needs: [prepare, pool]
|
||||
if: ${{ always() && needs.prepare.result == 'success' }}
|
||||
if: ${{ always() && needs.prepare.result == 'success' && contains(fromJSON(needs.prepare.outputs.suites || '[]'), 'security') }}
|
||||
uses: ./.github/workflows/rustfs-security-test.yml
|
||||
with:
|
||||
chain_manifest: ${{ needs.prepare.outputs.manifest }}
|
||||
@@ -729,7 +771,7 @@ jobs:
|
||||
|
||||
replication:
|
||||
needs: [prepare, security]
|
||||
if: ${{ always() && needs.prepare.result == 'success' }}
|
||||
if: ${{ always() && needs.prepare.result == 'success' && contains(fromJSON(needs.prepare.outputs.suites || '[]'), 'replication') }}
|
||||
uses: ./.github/workflows/rustfs-replication-test.yml
|
||||
with:
|
||||
chain_manifest: ${{ needs.prepare.outputs.manifest }}
|
||||
@@ -737,7 +779,7 @@ jobs:
|
||||
|
||||
fault-tolerance:
|
||||
needs: [prepare, replication]
|
||||
if: ${{ always() && needs.prepare.result == 'success' }}
|
||||
if: ${{ always() && needs.prepare.result == 'success' && contains(fromJSON(needs.prepare.outputs.suites || '[]'), 'fault-tolerance') }}
|
||||
uses: ./.github/workflows/rustfs-fault-tolerance-test.yml
|
||||
with:
|
||||
chain_manifest: ${{ needs.prepare.outputs.manifest }}
|
||||
@@ -745,15 +787,15 @@ jobs:
|
||||
|
||||
table:
|
||||
needs: [prepare, fault-tolerance]
|
||||
if: ${{ always() && needs.prepare.result == 'success' }}
|
||||
if: ${{ always() && needs.prepare.result == 'success' && contains(fromJSON(needs.prepare.outputs.suites || '[]'), 'table') }}
|
||||
uses: ./.github/workflows/rustfs-table-test.yml
|
||||
with:
|
||||
chain_manifest: ${{ needs.prepare.outputs.manifest }}
|
||||
secrets: inherit
|
||||
|
||||
performance:
|
||||
needs: [prepare, table]
|
||||
if: ${{ always() && needs.prepare.result == 'success' && needs.prepare.outputs.performance_ready == 'online' }}
|
||||
needs: [prepare]
|
||||
if: ${{ always() && needs.prepare.result == 'success' && contains(fromJSON(needs.prepare.outputs.suites || '[]'), 'performance') && needs.prepare.outputs.performance_ready == 'online' }}
|
||||
uses: ./.github/workflows/rustfs-performance-test.yml
|
||||
with:
|
||||
chain_manifest: ${{ needs.prepare.outputs.manifest }}
|
||||
@@ -781,8 +823,10 @@ jobs:
|
||||
env:
|
||||
CHAIN_MANIFEST: ${{ needs.prepare.outputs.manifest }}
|
||||
CHAIN_NEEDS: ${{ toJSON(needs) }}
|
||||
CHAIN_PLAN: ${{ needs.prepare.outputs.plan }}
|
||||
run: >-
|
||||
python3 scripts/functional_chain_evidence.py summarize
|
||||
--allow-skipped "${{ contains(fromJSON(needs.prepare.outputs.optional_suites || '[]'), 'performance') && needs.prepare.outputs.performance_ready != 'online' && 'performance' || '' }}"
|
||||
--directory "${RUNNER_TEMP}/chain-evidence"
|
||||
--output "${RUNNER_TEMP}/chain-report.json"
|
||||
- name: Retain chain report regardless of test verdict
|
||||
@@ -799,9 +843,10 @@ jobs:
|
||||
env:
|
||||
CHAIN_MANIFEST: ${{ needs.prepare.outputs.manifest }}
|
||||
CHAIN_NEEDS: ${{ toJSON(needs) }}
|
||||
CHAIN_PLAN: ${{ needs.prepare.outputs.plan }}
|
||||
run: >-
|
||||
python3 scripts/functional_chain_evidence.py aggregate
|
||||
--allow-skipped "${{ needs.prepare.outputs.performance_ready == 'online' && '' || 'performance' }}"
|
||||
--allow-skipped "${{ contains(fromJSON(needs.prepare.outputs.optional_suites || '[]'), 'performance') && needs.prepare.outputs.performance_ready != 'online' && 'performance' || '' }}"
|
||||
--directory "${RUNNER_TEMP}/chain-evidence"
|
||||
--output "${RUNNER_TEMP}/chain-complete.json"
|
||||
- name: Upload complete-chain evidence
|
||||
|
||||
@@ -29,11 +29,6 @@ on:
|
||||
description: 'Reset the nodes after the test (DESTROYS test data/config)'
|
||||
type: boolean
|
||||
default: true
|
||||
repository_dispatch:
|
||||
# Chain handoff: dispatched when the storage suite finishes. Heal runs
|
||||
# exactly once per chain; the pool expansion workflow no longer embeds
|
||||
# its own heal pass.
|
||||
types: [rustfs-chain-heal]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -60,10 +55,11 @@ env:
|
||||
jobs:
|
||||
heal-test:
|
||||
runs-on: smoke-testing
|
||||
timeout-minutes: 60
|
||||
# 90-minute heal run (see the step below) plus setup, report and cleanup.
|
||||
timeout-minutes: 120
|
||||
# Standalone manual run, or one link of the nightly functional chain
|
||||
# (storage -> heal -> pool). Pool expansion no longer re-runs heal.
|
||||
if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
|
||||
if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' }}
|
||||
steps:
|
||||
- name: Checkout chain tooling
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
@@ -109,6 +105,13 @@ jobs:
|
||||
warp --version || true
|
||||
df -h /data | tail -1
|
||||
|
||||
- name: Admit clean shared environment
|
||||
id: environment
|
||||
timeout-minutes: 5
|
||||
env:
|
||||
FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-heal.json
|
||||
run: python3 scripts/functional_environment.py enter --suite heal
|
||||
|
||||
- name: Cleanup environment (before)
|
||||
timeout-minutes: 5
|
||||
if: ${{ inputs.cleanup_before != 'false' }}
|
||||
@@ -152,10 +155,9 @@ jobs:
|
||||
./auto-testing/rustfs_heal_test.sh "${ARGS[@]}" --log-file "${LOG_FILE}"
|
||||
|
||||
- name: Run heal test (write -> outage -> heal -> verify)
|
||||
timeout-minutes: 45
|
||||
timeout-minutes: 90
|
||||
id: test
|
||||
# Case failures keep the run green: the report and the backlog
|
||||
# issue manager carry the product signal.
|
||||
# Collect reports and backlog evidence before the final suite verdict.
|
||||
continue-on-error: true
|
||||
run: |
|
||||
./auto-testing/rustfs_heal_test.sh \
|
||||
@@ -241,9 +243,8 @@ jobs:
|
||||
RESULT=success
|
||||
fi
|
||||
|
||||
# Product gate: failing steps keep the run green — they are reported
|
||||
# below and tracked in rustfs/backlog. Only harness/environment
|
||||
# breakdowns turn the workflow red.
|
||||
# Preserve harness completion separately; the final verdict checks
|
||||
# every case after retaining reports and cleanup evidence.
|
||||
STEPS_TOTAL="$(grep -cE '^\[HEAL-STEP\]' "${LOG_FILE}" 2>/dev/null || true)"
|
||||
STEPS_FAIL="$(grep -cE '^\[HEAL-STEP\].* FAIL$' "${LOG_FILE}" 2>/dev/null || true)"
|
||||
STEPS_TOTAL=$(( ${STEPS_TOTAL:-0} + 0 )); STEPS_FAIL=$(( ${STEPS_FAIL:-0} + 0 ))
|
||||
@@ -281,7 +282,7 @@ jobs:
|
||||
fi
|
||||
} | tee "${REPORT_FILE}"
|
||||
cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
|
||||
# Red only for harness/environment breakdowns; step failures stay green.
|
||||
# Preserve the harness verdict separately from the final case verdict.
|
||||
[ "${HARNESS_OK}" = "1" ]
|
||||
|
||||
- name: Upload functional report to dashboard
|
||||
@@ -377,8 +378,9 @@ jobs:
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Cleanup environment (after)
|
||||
id: environment_cleanup
|
||||
timeout-minutes: 5
|
||||
if: ${{ always() && inputs.cleanup_after != 'false' }}
|
||||
if: ${{ always() && inputs.cleanup_after != 'false' && steps.environment.outcome == 'success' }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
|
||||
@@ -396,54 +398,21 @@ jobs:
|
||||
'
|
||||
done
|
||||
|
||||
- name: "Continue functional chain (next: Pool expansion)"
|
||||
# Only chain-triggered runs forward to the next suite; standalone
|
||||
# workflow_dispatch runs stop after their own cleanup. A failed
|
||||
# handoff must never pass silently: it retries, then files an alert
|
||||
# issue in rustfs/backlog so a stalled chain is visible.
|
||||
if: ${{ always() && github.event_name == 'repository_dispatch' }}
|
||||
continue-on-error: true
|
||||
- name: Release environment only after successful cleanup
|
||||
if: ${{ always() && steps.environment.outcome == 'success' }}
|
||||
timeout-minutes: 5
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
run: |
|
||||
set -uo pipefail
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
|
||||
exit 1
|
||||
fi
|
||||
DISPATCHED=0
|
||||
for attempt in 1 2 3; do
|
||||
if gh api --method POST repos/rustfs/rustfs/dispatches \
|
||||
-f event_type='rustfs-chain-pool' \
|
||||
-F 'client_payload[from_suite]=heal'; then
|
||||
echo "dispatched next suite Pool expansion (attempt ${attempt})"
|
||||
DISPATCHED=1
|
||||
break
|
||||
fi
|
||||
echo "dispatch attempt ${attempt} failed; retrying in ${attempt}0s" >&2
|
||||
sleep "${attempt}0"
|
||||
done
|
||||
if [ "${DISPATCHED:-0}" -ne 1 ]; then
|
||||
echo "ERROR: functional chain stalled: could not dispatch Pool expansion after 3 attempts" >&2
|
||||
TITLE="[functional][chain] stalled after heal (run ${GITHUB_RUN_ID})"
|
||||
BODY_FILE="$(mktemp)"
|
||||
{
|
||||
echo "The functional chain could not hand off from **heal** to **Pool expansion** after 3 attempts."
|
||||
echo ""
|
||||
echo "- Failed suite job: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
|
||||
echo "- Expected next event: 'rustfs-chain-pool'"
|
||||
echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
|
||||
echo "- Recovery: re-dispatch manually with"
|
||||
FENCE="$(printf "\x60\x60\x60")"; echo " ${FENCE}"
|
||||
echo " gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-pool'"
|
||||
FENCE="$(printf "\x60\x60\x60")"; echo " ${FENCE}"
|
||||
} > "${BODY_FILE}"
|
||||
gh issue create -R rustfs/backlog --title "${TITLE}" \
|
||||
--body-file "${BODY_FILE}" --label functional-test \
|
||||
|| gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}" \
|
||||
|| echo "could not file the stall alert issue either; check the token" >&2
|
||||
exit 1
|
||||
fi
|
||||
FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-heal.json
|
||||
CLEANUP_OUTCOME: ${{ steps.environment_cleanup.outcome }}
|
||||
run: python3 scripts/functional_environment.py finish --suite heal --cleanup-outcome "$CLEANUP_OUTCOME"
|
||||
|
||||
- name: Retain environment ownership and cleanup receipt
|
||||
if: ${{ always() }}
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: functional-environment-heal-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/environment-heal.json
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Notify on failure
|
||||
if: failure()
|
||||
@@ -472,3 +441,13 @@ jobs:
|
||||
name: functional-chain-heal-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/chain-heal-${{ github.run_id }}-${{ github.run_attempt }}/heal.json
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Enforce suite and report result
|
||||
if: ${{ always() }}
|
||||
env:
|
||||
SUITE_OUTCOME: ${{ steps.test.outcome }}
|
||||
REPORT_OUTCOME: ${{ steps.chain_report.outcome }}
|
||||
run: |
|
||||
test "$SUITE_OUTCOME" = success
|
||||
test "$REPORT_OUTCOME" = success
|
||||
python3 scripts/functional_chain_evidence.py verify-report --suite heal --report "${FUNCTIONAL_ARTIFACTS_DIR}/steps.md"
|
||||
|
||||
@@ -29,9 +29,6 @@ on:
|
||||
description: 'Set RUSTFS_KMS_CONFIG_SECRET (runs KMS-107 config sealing)'
|
||||
required: false
|
||||
type: string
|
||||
repository_dispatch:
|
||||
# Chain handoff: dispatched when the S3 compatibility suite finishes.
|
||||
types: [rustfs-chain-kms]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -56,7 +53,7 @@ jobs:
|
||||
kms-test:
|
||||
runs-on: smoke-testing
|
||||
timeout-minutes: 60
|
||||
if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
|
||||
if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' }}
|
||||
steps:
|
||||
- name: Checkout repository (for report parser)
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
@@ -100,6 +97,13 @@ jobs:
|
||||
openssl version
|
||||
docker --version || true
|
||||
|
||||
- name: Admit clean shared environment
|
||||
id: environment
|
||||
timeout-minutes: 5
|
||||
env:
|
||||
FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-kms.json
|
||||
run: python3 scripts/functional_environment.py enter --suite kms
|
||||
|
||||
- name: Cleanup environment (before)
|
||||
timeout-minutes: 5
|
||||
run: |
|
||||
@@ -131,8 +135,7 @@ jobs:
|
||||
- name: Run KMS suite
|
||||
id: test
|
||||
timeout-minutes: 45
|
||||
# Case failures keep the run green: the report and the backlog
|
||||
# issue manager carry the product signal.
|
||||
# Collect reports and backlog evidence before the final suite verdict.
|
||||
continue-on-error: true
|
||||
run: |
|
||||
set -euo pipefail
|
||||
@@ -185,10 +188,8 @@ jobs:
|
||||
RESULT=success
|
||||
fi
|
||||
|
||||
# Product gate: failing cases keep the run green — they are reported
|
||||
# below and tracked in rustfs/backlog. Only harness/environment
|
||||
# breakdowns (the suite never reached case level, or a wholesale
|
||||
# failure with zero passes) turn the workflow red.
|
||||
# Report harness completion here; the final verdict checks every
|
||||
# case result after diagnostics and cleanup have been retained.
|
||||
CASES_TOTAL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| (PASS|FAIL|UNSUPPORTED|RUNNING) \|' "${CASE_TABLE}" 2>/dev/null || true)"
|
||||
CASES_FAIL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| FAIL \|' "${CASE_TABLE}" 2>/dev/null || true)"
|
||||
CASES_TOTAL=$(( ${CASES_TOTAL:-0} + 0 )); CASES_FAIL=$(( ${CASES_FAIL:-0} + 0 ))
|
||||
@@ -226,7 +227,7 @@ jobs:
|
||||
fi
|
||||
} | tee "${REPORT_FILE}"
|
||||
cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
|
||||
# Red only for harness/environment breakdowns; case failures stay green.
|
||||
# Preserve the harness verdict separately from the final case verdict.
|
||||
[ "${HARNESS_OK}" = "1" ]
|
||||
|
||||
- name: Upload functional report to dashboard
|
||||
@@ -325,8 +326,9 @@ jobs:
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Cleanup environment (after)
|
||||
id: environment_cleanup
|
||||
timeout-minutes: 5
|
||||
if: always()
|
||||
if: ${{ always() && steps.environment.outcome == 'success' }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
|
||||
@@ -344,54 +346,21 @@ jobs:
|
||||
'
|
||||
done
|
||||
|
||||
- name: "Continue functional chain (next: Tier)"
|
||||
# Only chain-triggered runs forward to the next suite; standalone
|
||||
# workflow_dispatch runs stop after their own cleanup. A failed
|
||||
# handoff must never pass silently: it retries, then files an alert
|
||||
# issue in rustfs/backlog so a stalled chain is visible.
|
||||
if: ${{ always() && github.event_name == 'repository_dispatch' }}
|
||||
continue-on-error: true
|
||||
- name: Release environment only after successful cleanup
|
||||
if: ${{ always() && steps.environment.outcome == 'success' }}
|
||||
timeout-minutes: 5
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
run: |
|
||||
set -uo pipefail
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
|
||||
exit 1
|
||||
fi
|
||||
DISPATCHED=0
|
||||
for attempt in 1 2 3; do
|
||||
if gh api --method POST repos/rustfs/rustfs/dispatches \
|
||||
-f event_type='rustfs-chain-tier' \
|
||||
-F 'client_payload[from_suite]=kms'; then
|
||||
echo "dispatched next suite Tier (attempt ${attempt})"
|
||||
DISPATCHED=1
|
||||
break
|
||||
fi
|
||||
echo "dispatch attempt ${attempt} failed; retrying in ${attempt}0s" >&2
|
||||
sleep "${attempt}0"
|
||||
done
|
||||
if [ "${DISPATCHED:-0}" -ne 1 ]; then
|
||||
echo "ERROR: functional chain stalled: could not dispatch Tier after 3 attempts" >&2
|
||||
TITLE="[functional][chain] stalled after kms (run ${GITHUB_RUN_ID})"
|
||||
BODY_FILE="$(mktemp)"
|
||||
{
|
||||
echo "The functional chain could not hand off from **kms** to **Tier** after 3 attempts."
|
||||
echo ""
|
||||
echo "- Failed suite job: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
|
||||
echo "- Expected next event: 'rustfs-chain-tier'"
|
||||
echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
|
||||
echo "- Recovery: re-dispatch manually with"
|
||||
FENCE="$(printf "\x60\x60\x60")"; echo " ${FENCE}"
|
||||
echo " gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-tier'"
|
||||
FENCE="$(printf "\x60\x60\x60")"; echo " ${FENCE}"
|
||||
} > "${BODY_FILE}"
|
||||
gh issue create -R rustfs/backlog --title "${TITLE}" \
|
||||
--body-file "${BODY_FILE}" --label functional-test \
|
||||
|| gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}" \
|
||||
|| echo "could not file the stall alert issue either; check the token" >&2
|
||||
exit 1
|
||||
fi
|
||||
FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-kms.json
|
||||
CLEANUP_OUTCOME: ${{ steps.environment_cleanup.outcome }}
|
||||
run: python3 scripts/functional_environment.py finish --suite kms --cleanup-outcome "$CLEANUP_OUTCOME"
|
||||
|
||||
- name: Retain environment ownership and cleanup receipt
|
||||
if: ${{ always() }}
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: functional-environment-kms-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/environment-kms.json
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Notify on failure
|
||||
if: failure()
|
||||
@@ -419,3 +388,13 @@ jobs:
|
||||
name: functional-chain-kms-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/chain-kms-${{ github.run_id }}-${{ github.run_attempt }}/kms.json
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Enforce suite and report result
|
||||
if: ${{ always() }}
|
||||
env:
|
||||
SUITE_OUTCOME: ${{ steps.test.outcome }}
|
||||
REPORT_OUTCOME: ${{ steps.chain_report.outcome }}
|
||||
run: |
|
||||
test "$SUITE_OUTCOME" = success
|
||||
test "$REPORT_OUTCOME" = success
|
||||
python3 scripts/functional_chain_evidence.py verify-report --suite kms --report "${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"
|
||||
|
||||
@@ -54,9 +54,6 @@ on:
|
||||
description: 'Reset the nodes after the test (DESTROYS test data/config)'
|
||||
type: boolean
|
||||
default: true
|
||||
repository_dispatch:
|
||||
# Chain handoff: dispatched when the replication suite finishes.
|
||||
types: [rustfs-chain-performance]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -89,7 +86,7 @@ env:
|
||||
|
||||
jobs:
|
||||
runner-check:
|
||||
if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
|
||||
if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' }}
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
@@ -113,7 +110,7 @@ jobs:
|
||||
RUSTFS_WARP_CONCURRENCY: ${{ inputs.warp_concurrency || '64' }}
|
||||
# Run on manual dispatch, or when the nightly build completed successfully.
|
||||
# Skipped when nightly failed.
|
||||
if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
|
||||
if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' }}
|
||||
steps:
|
||||
- name: Checkout chain tooling
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
@@ -159,6 +156,13 @@ jobs:
|
||||
warp --version || true
|
||||
df -h /data | tail -1
|
||||
|
||||
- name: Admit clean shared environment
|
||||
id: environment
|
||||
timeout-minutes: 5
|
||||
env:
|
||||
FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-performance.json
|
||||
run: python3 scripts/functional_environment.py enter --suite performance
|
||||
|
||||
- name: Reset test environment (before)
|
||||
if: ${{ inputs.cleanup_before != 'false' }}
|
||||
run: |
|
||||
@@ -323,10 +327,27 @@ jobs:
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Reset test environment (after)
|
||||
if: ${{ always() && inputs.cleanup_after != 'false' }}
|
||||
id: environment_cleanup
|
||||
if: ${{ always() && inputs.cleanup_after != 'false' && steps.environment.outcome == 'success' }}
|
||||
run: |
|
||||
./auto-testing/rustfs_performance_test.sh --step 7 -y --log-file "${LOG_FILE:-/dev/null}"
|
||||
|
||||
- name: Release environment only after successful cleanup
|
||||
if: ${{ always() && steps.environment.outcome == 'success' }}
|
||||
timeout-minutes: 5
|
||||
env:
|
||||
FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-performance.json
|
||||
CLEANUP_OUTCOME: ${{ steps.environment_cleanup.outcome }}
|
||||
run: python3 scripts/functional_environment.py finish --suite performance --cleanup-outcome "$CLEANUP_OUTCOME"
|
||||
|
||||
- name: Retain environment ownership and cleanup receipt
|
||||
if: ${{ always() }}
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: functional-environment-performance-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/environment-performance.json
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Notify on failure
|
||||
if: failure()
|
||||
run: |
|
||||
@@ -354,3 +375,13 @@ jobs:
|
||||
name: functional-chain-performance-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/chain-performance-${{ github.run_id }}-${{ github.run_attempt }}/performance.json
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Enforce suite and report result
|
||||
if: ${{ always() }}
|
||||
env:
|
||||
SUITE_OUTCOME: ${{ steps.benchmark.outcome }}
|
||||
REPORT_OUTCOME: ${{ steps.chain_report.outcome }}
|
||||
run: |
|
||||
test "$SUITE_OUTCOME" = success
|
||||
test "$REPORT_OUTCOME" = success
|
||||
python3 scripts/functional_chain_evidence.py verify-report --suite performance --report "${RUSTFS_RESULT_DIR}/summary.tsv"
|
||||
|
||||
@@ -56,9 +56,6 @@ on:
|
||||
description: 'Reset the nodes after the test (DESTROYS test data/config)'
|
||||
type: boolean
|
||||
default: true
|
||||
repository_dispatch:
|
||||
# Chain handoff: dispatched when the heal suite finishes.
|
||||
types: [rustfs-chain-pool]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -86,14 +83,13 @@ env:
|
||||
RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
|
||||
|
||||
jobs:
|
||||
# Pool expansion: dispatched by the heal suite's chain handoff. Heal
|
||||
# itself lives in rustfs-heal-test.yml and runs exactly once per chain.
|
||||
# Pool expansion owns an independent reusable suite.
|
||||
pool-expansion-test:
|
||||
name: Pool expansion / decommission test
|
||||
runs-on: smoke-testing
|
||||
# Leave setup and finalizer headroom beyond the default 240-minute test.
|
||||
timeout-minutes: 360
|
||||
if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
|
||||
if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' }}
|
||||
env:
|
||||
RUSTFS_POOL_ADMIN_ENDPOINT: ${{ secrets.RUSTFS_POOL_ADMIN_ENDPOINT || vars.RUSTFS_POOL_ADMIN_ENDPOINT || 'http://rustfs-node1:9000' }}
|
||||
RUSTFS_POOL_PROXY_ENDPOINT: http://127.0.0.1:19000
|
||||
@@ -149,6 +145,13 @@ jobs:
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Admit clean shared environment
|
||||
id: environment
|
||||
timeout-minutes: 5
|
||||
env:
|
||||
FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-pool.json
|
||||
run: python3 scripts/functional_environment.py enter --suite pool
|
||||
|
||||
- name: Cleanup environment (before)
|
||||
timeout-minutes: 5
|
||||
if: ${{ inputs.cleanup_before != 'false' }}
|
||||
@@ -270,8 +273,7 @@ jobs:
|
||||
# timeout-minutes requires; the bare || form yielded the string '240'
|
||||
# and the runner rejected the template (Unexpected value '240').
|
||||
timeout-minutes: ${{ fromJSON(inputs.pool_timeout_minutes || '240') }}
|
||||
# Case failures keep the run green: the report and the backlog
|
||||
# issue manager carry the product signal.
|
||||
# Collect reports and backlog evidence before the final suite verdict.
|
||||
continue-on-error: true
|
||||
run: |
|
||||
set -o pipefail
|
||||
@@ -698,8 +700,9 @@ jobs:
|
||||
./auto-testing/rustfs_pool_nginx_stage.sh cleanup
|
||||
|
||||
- name: Cleanup environment (after)
|
||||
id: environment_cleanup
|
||||
timeout-minutes: 5
|
||||
if: ${{ always() && steps.topology.outcome == 'success' && inputs.cleanup_after != 'false' }}
|
||||
if: ${{ always() && steps.topology.outcome == 'success' && inputs.cleanup_after != 'false' && steps.environment.outcome == 'success' }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
|
||||
@@ -717,54 +720,21 @@ jobs:
|
||||
'
|
||||
done
|
||||
|
||||
- name: "Continue functional chain (next: Security)"
|
||||
# Only chain-triggered runs forward to the next suite; standalone
|
||||
# workflow_dispatch runs stop after their own cleanup. A failed
|
||||
# handoff must never pass silently: it retries, then files an alert
|
||||
# issue in rustfs/backlog so a stalled chain is visible.
|
||||
if: ${{ always() && github.event_name == 'repository_dispatch' }}
|
||||
continue-on-error: true
|
||||
- name: Release environment only after successful cleanup
|
||||
if: ${{ always() && steps.environment.outcome == 'success' }}
|
||||
timeout-minutes: 5
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
run: |
|
||||
set -uo pipefail
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
|
||||
exit 1
|
||||
fi
|
||||
DISPATCHED=0
|
||||
for attempt in 1 2 3; do
|
||||
if gh api --method POST repos/rustfs/rustfs/dispatches \
|
||||
-f event_type='rustfs-chain-security' \
|
||||
-F 'client_payload[from_suite]=pool'; then
|
||||
echo "dispatched next suite Security (attempt ${attempt})"
|
||||
DISPATCHED=1
|
||||
break
|
||||
fi
|
||||
echo "dispatch attempt ${attempt} failed; retrying in ${attempt}0s" >&2
|
||||
sleep "${attempt}0"
|
||||
done
|
||||
if [ "${DISPATCHED:-0}" -ne 1 ]; then
|
||||
echo "ERROR: functional chain stalled: could not dispatch Security after 3 attempts" >&2
|
||||
TITLE="[functional][chain] stalled after pool (run ${GITHUB_RUN_ID})"
|
||||
BODY_FILE="$(mktemp)"
|
||||
{
|
||||
echo "The functional chain could not hand off from **pool** to **Security** after 3 attempts."
|
||||
echo ""
|
||||
echo "- Failed suite job: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
|
||||
echo "- Expected next event: 'rustfs-chain-security'"
|
||||
echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
|
||||
echo "- Recovery: re-dispatch manually with"
|
||||
FENCE="$(printf "\x60\x60\x60")"; echo " ${FENCE}"
|
||||
echo " gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-security'"
|
||||
FENCE="$(printf "\x60\x60\x60")"; echo " ${FENCE}"
|
||||
} > "${BODY_FILE}"
|
||||
gh issue create -R rustfs/backlog --title "${TITLE}" \
|
||||
--body-file "${BODY_FILE}" --label functional-test \
|
||||
|| gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}" \
|
||||
|| echo "could not file the stall alert issue either; check the token" >&2
|
||||
exit 1
|
||||
fi
|
||||
FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-pool.json
|
||||
CLEANUP_OUTCOME: ${{ steps.environment_cleanup.outcome }}
|
||||
run: python3 scripts/functional_environment.py finish --suite pool --cleanup-outcome "$CLEANUP_OUTCOME"
|
||||
|
||||
- name: Retain environment ownership and cleanup receipt
|
||||
if: ${{ always() }}
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: functional-environment-pool-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/environment-pool.json
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Notify on failure
|
||||
if: failure()
|
||||
@@ -793,3 +763,13 @@ jobs:
|
||||
name: functional-chain-pool-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/chain-pool-${{ github.run_id }}-${{ github.run_attempt }}/pool.json
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Enforce suite and report result
|
||||
if: ${{ always() }}
|
||||
env:
|
||||
SUITE_OUTCOME: ${{ steps.pool_test.outcome }}
|
||||
REPORT_OUTCOME: ${{ steps.chain_report.outcome }}
|
||||
run: |
|
||||
test "$SUITE_OUTCOME" = success
|
||||
test "$REPORT_OUTCOME" = success
|
||||
python3 scripts/functional_chain_evidence.py verify-report --suite pool --report "${POOL_ARTIFACT_DIR}/pool-steps.md"
|
||||
|
||||
@@ -39,9 +39,6 @@ on:
|
||||
- bucket
|
||||
- site
|
||||
default: all
|
||||
repository_dispatch:
|
||||
# Chain handoff: dispatched when the security suite finishes.
|
||||
types: [rustfs-chain-replication]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -68,7 +65,7 @@ jobs:
|
||||
replication-test:
|
||||
runs-on: smoke-testing
|
||||
timeout-minutes: 60
|
||||
if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
|
||||
if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' }}
|
||||
steps:
|
||||
- name: Checkout repository (for report parser)
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
@@ -113,6 +110,13 @@ jobs:
|
||||
aws --version
|
||||
df -h /data | tail -1 || true
|
||||
|
||||
- name: Admit clean shared environment
|
||||
id: environment
|
||||
timeout-minutes: 5
|
||||
env:
|
||||
FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-replication.json
|
||||
run: python3 scripts/functional_environment.py enter --suite replication
|
||||
|
||||
- name: Cleanup environment (before)
|
||||
timeout-minutes: 5
|
||||
run: |
|
||||
@@ -135,8 +139,7 @@ jobs:
|
||||
- name: Run replication suite
|
||||
id: test
|
||||
timeout-minutes: 45
|
||||
# Case failures keep the run green: the report and the backlog
|
||||
# issue manager carry the product signal.
|
||||
# Collect reports and backlog evidence before the final suite verdict.
|
||||
continue-on-error: true
|
||||
run: |
|
||||
set -euo pipefail
|
||||
@@ -192,10 +195,8 @@ jobs:
|
||||
RESULT=success
|
||||
fi
|
||||
|
||||
# Product gate: failing cases keep the run green — they are reported
|
||||
# below and tracked in rustfs/backlog. Only harness/environment
|
||||
# breakdowns (the suite never reached case level, or a wholesale
|
||||
# failure with zero passes) turn the workflow red.
|
||||
# Report harness completion here; the final verdict checks every
|
||||
# case result after diagnostics and cleanup have been retained.
|
||||
CASES_TOTAL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| (PASS|FAIL|UNSUPPORTED|RUNNING) \|' "${CASE_TABLE}" 2>/dev/null || true)"
|
||||
CASES_FAIL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| FAIL \|' "${CASE_TABLE}" 2>/dev/null || true)"
|
||||
CASES_TOTAL=$(( ${CASES_TOTAL:-0} + 0 )); CASES_FAIL=$(( ${CASES_FAIL:-0} + 0 ))
|
||||
@@ -234,7 +235,7 @@ jobs:
|
||||
fi
|
||||
} | tee "${REPORT_FILE}"
|
||||
cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
|
||||
# Red only for harness/environment breakdowns; case failures stay green.
|
||||
# Preserve the harness verdict separately from the final case verdict.
|
||||
[ "${HARNESS_OK}" = "1" ]
|
||||
|
||||
- name: Upload functional report to dashboard
|
||||
@@ -333,8 +334,9 @@ jobs:
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Cleanup environment (after)
|
||||
id: environment_cleanup
|
||||
timeout-minutes: 5
|
||||
if: always()
|
||||
if: ${{ always() && steps.environment.outcome == 'success' }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
|
||||
@@ -352,50 +354,21 @@ jobs:
|
||||
'
|
||||
done
|
||||
|
||||
- name: "Continue functional chain (next: Fault tolerance)"
|
||||
if: ${{ always() && github.event_name == 'repository_dispatch' }}
|
||||
- name: Release environment only after successful cleanup
|
||||
if: ${{ always() && steps.environment.outcome == 'success' }}
|
||||
timeout-minutes: 5
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
run: |
|
||||
set -uo pipefail
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
|
||||
exit 1
|
||||
fi
|
||||
DISPATCHED=0
|
||||
for attempt in 1 2 3; do
|
||||
if gh api --method POST repos/rustfs/rustfs/dispatches \
|
||||
-f event_type='rustfs-chain-fault-tolerance' \
|
||||
-F 'client_payload[from_suite]=replication'; then
|
||||
echo "dispatched next suite Fault tolerance (attempt ${attempt})"
|
||||
DISPATCHED=1
|
||||
break
|
||||
fi
|
||||
echo "dispatch attempt ${attempt} failed; retrying in ${attempt}0s" >&2
|
||||
sleep "${attempt}0"
|
||||
done
|
||||
if [ "${DISPATCHED:-0}" -ne 1 ]; then
|
||||
echo "ERROR: functional chain stalled: could not dispatch Fault tolerance after 3 attempts" >&2
|
||||
TITLE="[functional][chain] stalled after replication (run ${GITHUB_RUN_ID})"
|
||||
BODY_FILE="$(mktemp)"
|
||||
trap 'rm -f "${BODY_FILE}"' EXIT
|
||||
{
|
||||
echo "The functional chain could not hand off from **replication** to **Fault tolerance** after 3 attempts."
|
||||
echo ""
|
||||
echo "- Failed suite job: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
|
||||
echo "- Expected next event: 'rustfs-chain-fault-tolerance'"
|
||||
echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
|
||||
echo "- Recovery: re-dispatch manually with"
|
||||
FENCE="$(printf "\x60\x60\x60")"; echo " ${FENCE}"
|
||||
echo " gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-fault-tolerance'"
|
||||
FENCE="$(printf "\x60\x60\x60")"; echo " ${FENCE}"
|
||||
} > "${BODY_FILE}"
|
||||
gh issue create -R rustfs/backlog --title "${TITLE}" \
|
||||
--body-file "${BODY_FILE}" --label functional-test \
|
||||
|| gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}" \
|
||||
|| echo "could not file the stall alert issue either; check the token" >&2
|
||||
exit 1
|
||||
fi
|
||||
FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-replication.json
|
||||
CLEANUP_OUTCOME: ${{ steps.environment_cleanup.outcome }}
|
||||
run: python3 scripts/functional_environment.py finish --suite replication --cleanup-outcome "$CLEANUP_OUTCOME"
|
||||
|
||||
- name: Retain environment ownership and cleanup receipt
|
||||
if: ${{ always() }}
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: functional-environment-replication-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/environment-replication.json
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Notify on failure
|
||||
if: failure()
|
||||
@@ -424,3 +397,13 @@ jobs:
|
||||
name: functional-chain-replication-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/chain-replication-${{ github.run_id }}-${{ github.run_attempt }}/replication.json
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Enforce suite and report result
|
||||
if: ${{ always() }}
|
||||
env:
|
||||
SUITE_OUTCOME: ${{ steps.test.outcome }}
|
||||
REPORT_OUTCOME: ${{ steps.chain_report.outcome }}
|
||||
run: |
|
||||
test "$SUITE_OUTCOME" = success
|
||||
test "$REPORT_OUTCOME" = success
|
||||
python3 scripts/functional_chain_evidence.py verify-report --suite replication --report "${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"
|
||||
|
||||
@@ -17,9 +17,6 @@ on:
|
||||
description: 'Direct .deb URL (nightly/R2/dev). Overrides rustfs_version.'
|
||||
required: false
|
||||
type: string
|
||||
repository_dispatch:
|
||||
# Chain handoff: dispatched when the upgrade suite finishes.
|
||||
types: [rustfs-chain-s3]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -44,7 +41,7 @@ jobs:
|
||||
s3-compat-test:
|
||||
runs-on: smoke-testing
|
||||
timeout-minutes: 60
|
||||
if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
|
||||
if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' }}
|
||||
steps:
|
||||
- name: Checkout repository (for report parser)
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
@@ -88,6 +85,13 @@ jobs:
|
||||
openssl version
|
||||
df -h /data | tail -1
|
||||
|
||||
- name: Admit clean shared environment
|
||||
id: environment
|
||||
timeout-minutes: 5
|
||||
env:
|
||||
FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-s3.json
|
||||
run: python3 scripts/functional_environment.py enter --suite s3
|
||||
|
||||
- name: Cleanup environment (before)
|
||||
timeout-minutes: 5
|
||||
run: |
|
||||
@@ -110,8 +114,7 @@ jobs:
|
||||
- name: Run S3 compatibility suite
|
||||
id: test
|
||||
timeout-minutes: 45
|
||||
# Case failures keep the run green: the report and the backlog
|
||||
# issue manager carry the product signal.
|
||||
# Collect reports and backlog evidence before the final suite verdict.
|
||||
continue-on-error: true
|
||||
run: |
|
||||
set -euo pipefail
|
||||
@@ -161,10 +164,8 @@ jobs:
|
||||
RESULT=success
|
||||
fi
|
||||
|
||||
# Product gate: failing cases keep the run green — they are reported
|
||||
# below and tracked in rustfs/backlog. Only harness/environment
|
||||
# breakdowns (the suite never reached case level, or a wholesale
|
||||
# failure with zero passes) turn the workflow red.
|
||||
# Report harness completion here; the final verdict checks every
|
||||
# case result after diagnostics and cleanup have been retained.
|
||||
CASES_TOTAL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| (PASS|FAIL|UNSUPPORTED|RUNNING) \|' "${CASE_TABLE}" 2>/dev/null || true)"
|
||||
CASES_FAIL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| FAIL \|' "${CASE_TABLE}" 2>/dev/null || true)"
|
||||
CASES_TOTAL=$(( ${CASES_TOTAL:-0} + 0 )); CASES_FAIL=$(( ${CASES_FAIL:-0} + 0 ))
|
||||
@@ -203,7 +204,7 @@ jobs:
|
||||
fi
|
||||
} | tee "${REPORT_FILE}"
|
||||
cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
|
||||
# Red only for harness/environment breakdowns; case failures stay green.
|
||||
# Preserve the harness verdict separately from the final case verdict.
|
||||
[ "${HARNESS_OK}" = "1" ]
|
||||
|
||||
- name: Upload functional report to dashboard
|
||||
@@ -302,8 +303,9 @@ jobs:
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Cleanup environment (after)
|
||||
id: environment_cleanup
|
||||
timeout-minutes: 5
|
||||
if: always()
|
||||
if: ${{ always() && steps.environment.outcome == 'success' }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
|
||||
@@ -321,54 +323,21 @@ jobs:
|
||||
'
|
||||
done
|
||||
|
||||
- name: "Continue functional chain (next: KMS)"
|
||||
# Only chain-triggered runs forward to the next suite; standalone
|
||||
# workflow_dispatch runs stop after their own cleanup. A failed
|
||||
# handoff must never pass silently: it retries, then files an alert
|
||||
# issue in rustfs/backlog so a stalled chain is visible.
|
||||
if: ${{ always() && github.event_name == 'repository_dispatch' }}
|
||||
continue-on-error: true
|
||||
- name: Release environment only after successful cleanup
|
||||
if: ${{ always() && steps.environment.outcome == 'success' }}
|
||||
timeout-minutes: 5
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
run: |
|
||||
set -uo pipefail
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
|
||||
exit 1
|
||||
fi
|
||||
DISPATCHED=0
|
||||
for attempt in 1 2 3; do
|
||||
if gh api --method POST repos/rustfs/rustfs/dispatches \
|
||||
-f event_type='rustfs-chain-kms' \
|
||||
-F 'client_payload[from_suite]=s3'; then
|
||||
echo "dispatched next suite KMS (attempt ${attempt})"
|
||||
DISPATCHED=1
|
||||
break
|
||||
fi
|
||||
echo "dispatch attempt ${attempt} failed; retrying in ${attempt}0s" >&2
|
||||
sleep "${attempt}0"
|
||||
done
|
||||
if [ "${DISPATCHED:-0}" -ne 1 ]; then
|
||||
echo "ERROR: functional chain stalled: could not dispatch KMS after 3 attempts" >&2
|
||||
TITLE="[functional][chain] stalled after s3 (run ${GITHUB_RUN_ID})"
|
||||
BODY_FILE="$(mktemp)"
|
||||
{
|
||||
echo "The functional chain could not hand off from **s3** to **KMS** after 3 attempts."
|
||||
echo ""
|
||||
echo "- Failed suite job: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
|
||||
echo "- Expected next event: 'rustfs-chain-kms'"
|
||||
echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
|
||||
echo "- Recovery: re-dispatch manually with"
|
||||
FENCE="$(printf "\x60\x60\x60")"; echo " ${FENCE}"
|
||||
echo " gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-kms'"
|
||||
FENCE="$(printf "\x60\x60\x60")"; echo " ${FENCE}"
|
||||
} > "${BODY_FILE}"
|
||||
gh issue create -R rustfs/backlog --title "${TITLE}" \
|
||||
--body-file "${BODY_FILE}" --label functional-test \
|
||||
|| gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}" \
|
||||
|| echo "could not file the stall alert issue either; check the token" >&2
|
||||
exit 1
|
||||
fi
|
||||
FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-s3.json
|
||||
CLEANUP_OUTCOME: ${{ steps.environment_cleanup.outcome }}
|
||||
run: python3 scripts/functional_environment.py finish --suite s3 --cleanup-outcome "$CLEANUP_OUTCOME"
|
||||
|
||||
- name: Retain environment ownership and cleanup receipt
|
||||
if: ${{ always() }}
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: functional-environment-s3-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/environment-s3.json
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Notify on failure
|
||||
if: failure()
|
||||
@@ -396,3 +365,13 @@ jobs:
|
||||
name: functional-chain-s3-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/chain-s3-${{ github.run_id }}-${{ github.run_attempt }}/s3.json
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Enforce suite and report result
|
||||
if: ${{ always() }}
|
||||
env:
|
||||
SUITE_OUTCOME: ${{ steps.test.outcome }}
|
||||
REPORT_OUTCOME: ${{ steps.chain_report.outcome }}
|
||||
run: |
|
||||
test "$SUITE_OUTCOME" = success
|
||||
test "$REPORT_OUTCOME" = success
|
||||
python3 scripts/functional_chain_evidence.py verify-report --suite s3 --report "${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"
|
||||
|
||||
@@ -52,9 +52,6 @@ on:
|
||||
description: 'Reset the nodes after the test (DESTROYS test data/config)'
|
||||
type: boolean
|
||||
default: true
|
||||
repository_dispatch:
|
||||
# Chain handoff: dispatched when the pool expansion suite finishes.
|
||||
types: [rustfs-chain-security]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -81,7 +78,7 @@ jobs:
|
||||
security-test:
|
||||
runs-on: smoke-testing
|
||||
timeout-minutes: 60
|
||||
if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
|
||||
if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' }}
|
||||
steps:
|
||||
# Checkout the repository into its own subdirectory. Checking out at
|
||||
# the workspace root would wipe the auto-testing clone above (that is
|
||||
@@ -126,6 +123,13 @@ jobs:
|
||||
docker --version || true
|
||||
df -h /data | tail -1
|
||||
|
||||
- name: Admit clean shared environment
|
||||
id: environment
|
||||
timeout-minutes: 5
|
||||
env:
|
||||
FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-security.json
|
||||
run: python3 rustfs-repo/scripts/functional_environment.py enter --suite security
|
||||
|
||||
- name: Cleanup environment (before)
|
||||
timeout-minutes: 5
|
||||
if: ${{ inputs.cleanup_before != 'false' || github.event_name != 'workflow_dispatch' }}
|
||||
@@ -149,8 +153,7 @@ jobs:
|
||||
- name: Run security suite
|
||||
id: test
|
||||
timeout-minutes: 45
|
||||
# Case failures keep the run green: the report and the backlog
|
||||
# issue manager carry the product signal.
|
||||
# Collect reports and backlog evidence before the final suite verdict.
|
||||
continue-on-error: true
|
||||
env:
|
||||
REPORT_FILE: ${{ env.SECURITY_ARTIFACTS_DIR }}/suite-report.md
|
||||
@@ -194,9 +197,8 @@ jobs:
|
||||
RESULT=success
|
||||
fi
|
||||
|
||||
# Product gate: failing cases keep the run green — they are reported
|
||||
# below and tracked in rustfs/backlog. Only harness/environment
|
||||
# breakdowns turn the workflow red. The suite log lines carry ANSI
|
||||
# The final verdict checks cases after diagnostics and cleanup.
|
||||
# The suite log lines carry ANSI
|
||||
# color escapes both before and after the verdict tag
|
||||
# (e.g. "\e[1;31m[FAIL]\e[0m STS-105 ..."), so the pattern allows
|
||||
# any number of escapes on each side of it.
|
||||
@@ -236,7 +238,7 @@ jobs:
|
||||
fi
|
||||
} > "${SECURITY_ARTIFACTS_DIR}/report.md"
|
||||
cat "${SECURITY_ARTIFACTS_DIR}/report.md" >> "${GITHUB_STEP_SUMMARY}"
|
||||
# Red only for harness/environment breakdowns; case failures stay green.
|
||||
# Preserve the harness verdict separately from the final case verdict.
|
||||
[ "${HARNESS_OK}" = "1" ]
|
||||
|
||||
- name: Upload functional report to dashboard
|
||||
@@ -336,8 +338,9 @@ jobs:
|
||||
retention-days: 3
|
||||
|
||||
- name: Cleanup environment (after)
|
||||
id: environment_cleanup
|
||||
timeout-minutes: 5
|
||||
if: ${{ always() && (inputs.cleanup_after != 'false' || github.event_name != 'workflow_dispatch') }}
|
||||
if: ${{ always() && (inputs.cleanup_after != 'false' || github.event_name != 'workflow_dispatch') && steps.environment.outcome == 'success' }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
|
||||
@@ -355,23 +358,21 @@ jobs:
|
||||
'
|
||||
done
|
||||
|
||||
- name: "Continue functional chain (next: Replication)"
|
||||
# Only chain-triggered runs forward to the next suite; standalone
|
||||
# workflow_dispatch runs stop after their own cleanup.
|
||||
if: ${{ always() && github.event_name == 'repository_dispatch' }}
|
||||
continue-on-error: true
|
||||
- name: Release environment only after successful cleanup
|
||||
if: ${{ always() && steps.environment.outcome == 'success' }}
|
||||
timeout-minutes: 5
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "Dispatching next functional suite: Replication"
|
||||
gh api --method POST repos/rustfs/rustfs/dispatches \
|
||||
-f event_type='rustfs-chain-replication' \
|
||||
-F 'client_payload[from_suite]=security'
|
||||
FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-security.json
|
||||
CLEANUP_OUTCOME: ${{ steps.environment_cleanup.outcome }}
|
||||
run: python3 rustfs-repo/scripts/functional_environment.py finish --suite security --cleanup-outcome "$CLEANUP_OUTCOME"
|
||||
|
||||
- name: Retain environment ownership and cleanup receipt
|
||||
if: ${{ always() }}
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: functional-environment-security-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/environment-security.json
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Notify on failure
|
||||
if: failure()
|
||||
@@ -400,3 +401,13 @@ jobs:
|
||||
name: functional-chain-security-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/chain-security-${{ github.run_id }}-${{ github.run_attempt }}/security.json
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Enforce suite and report result
|
||||
if: ${{ always() }}
|
||||
env:
|
||||
SUITE_OUTCOME: ${{ steps.test.outcome }}
|
||||
REPORT_OUTCOME: ${{ steps.report.outcome }}
|
||||
run: |
|
||||
test "$SUITE_OUTCOME" = success
|
||||
test "$REPORT_OUTCOME" = success
|
||||
python3 rustfs-repo/scripts/functional_chain_evidence.py verify-report --suite security --report "${SECURITY_ARTIFACTS_DIR}/suite-report.md"
|
||||
|
||||
@@ -26,9 +26,6 @@ on:
|
||||
- single-multi
|
||||
- multi-multi
|
||||
default: all
|
||||
repository_dispatch:
|
||||
# Chain handoff: dispatched when the tier suite finishes.
|
||||
types: [rustfs-chain-storage]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -53,7 +50,7 @@ jobs:
|
||||
storage-test:
|
||||
runs-on: smoke-testing
|
||||
timeout-minutes: 60
|
||||
if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
|
||||
if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' }}
|
||||
steps:
|
||||
- name: Checkout repository (for report parser)
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
@@ -97,6 +94,13 @@ jobs:
|
||||
openssl version
|
||||
df -h /data | tail -1
|
||||
|
||||
- name: Admit clean shared environment
|
||||
id: environment
|
||||
timeout-minutes: 5
|
||||
env:
|
||||
FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-storage.json
|
||||
run: python3 scripts/functional_environment.py enter --suite storage
|
||||
|
||||
- name: Cleanup environment (before)
|
||||
timeout-minutes: 5
|
||||
run: |
|
||||
@@ -119,8 +123,7 @@ jobs:
|
||||
- name: Run storage engine suite
|
||||
id: test
|
||||
timeout-minutes: 45
|
||||
# Case failures keep the run green: the report and the backlog
|
||||
# issue manager carry the product signal.
|
||||
# Collect reports and backlog evidence before the final suite verdict.
|
||||
continue-on-error: true
|
||||
run: |
|
||||
set -euo pipefail
|
||||
@@ -176,10 +179,8 @@ jobs:
|
||||
RESULT=success
|
||||
fi
|
||||
|
||||
# Product gate: failing cases keep the run green — they are reported
|
||||
# below and tracked in rustfs/backlog. Only harness/environment
|
||||
# breakdowns (the suite never reached case level, or a wholesale
|
||||
# failure with zero passes) turn the workflow red.
|
||||
# Report harness completion here; the final verdict checks every
|
||||
# case result after diagnostics and cleanup have been retained.
|
||||
CASES_TOTAL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| (PASS|FAIL|UNSUPPORTED|RUNNING) \|' "${CASE_TABLE}" 2>/dev/null || true)"
|
||||
CASES_FAIL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| FAIL \|' "${CASE_TABLE}" 2>/dev/null || true)"
|
||||
CASES_TOTAL=$(( ${CASES_TOTAL:-0} + 0 )); CASES_FAIL=$(( ${CASES_FAIL:-0} + 0 ))
|
||||
@@ -218,7 +219,7 @@ jobs:
|
||||
fi
|
||||
} | tee "${REPORT_FILE}"
|
||||
cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
|
||||
# Red only for harness/environment breakdowns; case failures stay green.
|
||||
# Preserve the harness verdict separately from the final case verdict.
|
||||
[ "${HARNESS_OK}" = "1" ]
|
||||
|
||||
- name: Upload functional report to dashboard
|
||||
@@ -317,8 +318,9 @@ jobs:
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Cleanup environment (after)
|
||||
id: environment_cleanup
|
||||
timeout-minutes: 5
|
||||
if: always()
|
||||
if: ${{ always() && steps.environment.outcome == 'success' }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
|
||||
@@ -336,54 +338,21 @@ jobs:
|
||||
'
|
||||
done
|
||||
|
||||
- name: "Continue functional chain (next: Heal)"
|
||||
# Only chain-triggered runs forward to the next suite; standalone
|
||||
# workflow_dispatch runs stop after their own cleanup. A failed
|
||||
# handoff must never pass silently: it retries, then files an alert
|
||||
# issue in rustfs/backlog so a stalled chain is visible.
|
||||
if: ${{ always() && github.event_name == 'repository_dispatch' }}
|
||||
continue-on-error: true
|
||||
- name: Release environment only after successful cleanup
|
||||
if: ${{ always() && steps.environment.outcome == 'success' }}
|
||||
timeout-minutes: 5
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
run: |
|
||||
set -uo pipefail
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
|
||||
exit 1
|
||||
fi
|
||||
DISPATCHED=0
|
||||
for attempt in 1 2 3; do
|
||||
if gh api --method POST repos/rustfs/rustfs/dispatches \
|
||||
-f event_type='rustfs-chain-heal' \
|
||||
-F 'client_payload[from_suite]=storage'; then
|
||||
echo "dispatched next suite Heal (attempt ${attempt})"
|
||||
DISPATCHED=1
|
||||
break
|
||||
fi
|
||||
echo "dispatch attempt ${attempt} failed; retrying in ${attempt}0s" >&2
|
||||
sleep "${attempt}0"
|
||||
done
|
||||
if [ "${DISPATCHED:-0}" -ne 1 ]; then
|
||||
echo "ERROR: functional chain stalled: could not dispatch Heal after 3 attempts" >&2
|
||||
TITLE="[functional][chain] stalled after storage (run ${GITHUB_RUN_ID})"
|
||||
BODY_FILE="$(mktemp)"
|
||||
{
|
||||
echo "The functional chain could not hand off from **storage** to **Heal** after 3 attempts."
|
||||
echo ""
|
||||
echo "- Failed suite job: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
|
||||
echo "- Expected next event: 'rustfs-chain-heal'"
|
||||
echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
|
||||
echo "- Recovery: re-dispatch manually with"
|
||||
FENCE="$(printf "\x60\x60\x60")"; echo " ${FENCE}"
|
||||
echo " gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-heal'"
|
||||
FENCE="$(printf "\x60\x60\x60")"; echo " ${FENCE}"
|
||||
} > "${BODY_FILE}"
|
||||
gh issue create -R rustfs/backlog --title "${TITLE}" \
|
||||
--body-file "${BODY_FILE}" --label functional-test \
|
||||
|| gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}" \
|
||||
|| echo "could not file the stall alert issue either; check the token" >&2
|
||||
exit 1
|
||||
fi
|
||||
FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-storage.json
|
||||
CLEANUP_OUTCOME: ${{ steps.environment_cleanup.outcome }}
|
||||
run: python3 scripts/functional_environment.py finish --suite storage --cleanup-outcome "$CLEANUP_OUTCOME"
|
||||
|
||||
- name: Retain environment ownership and cleanup receipt
|
||||
if: ${{ always() }}
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: functional-environment-storage-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/environment-storage.json
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Notify on failure
|
||||
if: failure()
|
||||
@@ -411,3 +380,13 @@ jobs:
|
||||
name: functional-chain-storage-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/chain-storage-${{ github.run_id }}-${{ github.run_attempt }}/storage.json
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Enforce suite and report result
|
||||
if: ${{ always() }}
|
||||
env:
|
||||
SUITE_OUTCOME: ${{ steps.test.outcome }}
|
||||
REPORT_OUTCOME: ${{ steps.chain_report.outcome }}
|
||||
run: |
|
||||
test "$SUITE_OUTCOME" = success
|
||||
test "$REPORT_OUTCOME" = success
|
||||
python3 scripts/functional_chain_evidence.py verify-report --suite storage --report "${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"
|
||||
|
||||
@@ -89,6 +89,13 @@ jobs:
|
||||
aws --version
|
||||
python3 --version
|
||||
|
||||
- name: Admit clean shared environment
|
||||
id: environment
|
||||
timeout-minutes: 5
|
||||
env:
|
||||
FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-table.json
|
||||
run: python3 scripts/functional_environment.py enter --suite table
|
||||
|
||||
- name: Cleanup environment (before)
|
||||
timeout-minutes: 5
|
||||
continue-on-error: true
|
||||
@@ -162,10 +169,8 @@ jobs:
|
||||
RESULT=success
|
||||
fi
|
||||
|
||||
# Product gate: failing cases keep the run green — they are reported
|
||||
# below and tracked in rustfs/backlog. Only harness/environment
|
||||
# breakdowns (the suite never reached case level, or a wholesale
|
||||
# failure with zero passes) turn the workflow red.
|
||||
# Report harness completion here; the final verdict checks every
|
||||
# case result after diagnostics and cleanup have been retained.
|
||||
CASES_TOTAL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| (PASS|FAIL|UNSUPPORTED|RUNNING) \|' "${CASE_TABLE}" 2>/dev/null || true)"
|
||||
CASES_FAIL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| FAIL \|' "${CASE_TABLE}" 2>/dev/null || true)"
|
||||
CASES_TOTAL=$(( ${CASES_TOTAL:-0} + 0 )); CASES_FAIL=$(( ${CASES_FAIL:-0} + 0 ))
|
||||
@@ -203,7 +208,7 @@ jobs:
|
||||
fi
|
||||
} | tee "${REPORT_FILE}"
|
||||
cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
|
||||
# Red only for harness/environment breakdowns; case failures stay green.
|
||||
# Preserve the harness verdict separately from the final case verdict.
|
||||
[ "${HARNESS_OK}" = "1" ]
|
||||
|
||||
- name: Upload functional report to dashboard
|
||||
@@ -292,11 +297,12 @@ jobs:
|
||||
retention-days: 30
|
||||
|
||||
- name: Cleanup environment (after)
|
||||
id: environment_cleanup
|
||||
timeout-minutes: 5
|
||||
if: always()
|
||||
continue-on-error: true
|
||||
if: ${{ always() && steps.environment.outcome == 'success' }}
|
||||
run: |
|
||||
set -uo pipefail
|
||||
set -euo pipefail
|
||||
cleanup_status=0
|
||||
for node in ${RUSTFS_NODES}; do
|
||||
ssh -o BatchMode=yes -o ConnectTimeout=10 "${RUSTFS_SSH_USER}@${node}" '
|
||||
set -euo pipefail
|
||||
@@ -307,26 +313,25 @@ jobs:
|
||||
fi
|
||||
for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
|
||||
${SUDO} rm -f /etc/default/rustfs
|
||||
' || echo "cleanup failed on ${node} (continuing)"
|
||||
' || cleanup_status=$?
|
||||
done
|
||||
exit "${cleanup_status}"
|
||||
|
||||
- name: "Continue functional chain (next: Performance)"
|
||||
# Only chain-triggered runs forward to the next suite; standalone
|
||||
# workflow_dispatch runs stop after their own cleanup.
|
||||
if: ${{ always() && github.event_name == 'repository_dispatch' }}
|
||||
continue-on-error: true
|
||||
- name: Release environment only after successful cleanup
|
||||
if: ${{ always() && steps.environment.outcome == 'success' }}
|
||||
timeout-minutes: 5
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "Dispatching next functional suite: Performance"
|
||||
gh api --method POST repos/rustfs/rustfs/dispatches \
|
||||
-f event_type='rustfs-chain-performance' \
|
||||
-F 'client_payload[from_suite]=table'
|
||||
FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-table.json
|
||||
CLEANUP_OUTCOME: ${{ steps.environment_cleanup.outcome }}
|
||||
run: python3 scripts/functional_environment.py finish --suite table --cleanup-outcome "$CLEANUP_OUTCOME"
|
||||
|
||||
- name: Retain environment ownership and cleanup receipt
|
||||
if: ${{ always() }}
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: functional-environment-table-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/environment-table.json
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Notify on failure
|
||||
if: failure()
|
||||
@@ -358,3 +363,13 @@ jobs:
|
||||
name: functional-chain-table-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/chain-table-${{ github.run_id }}-${{ github.run_attempt }}/table.json
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Enforce suite and report result
|
||||
if: ${{ always() }}
|
||||
env:
|
||||
SUITE_OUTCOME: ${{ steps.test.outcome }}
|
||||
REPORT_OUTCOME: ${{ steps.chain_report.outcome }}
|
||||
run: |
|
||||
test "$SUITE_OUTCOME" = success
|
||||
test "$REPORT_OUTCOME" = success
|
||||
python3 scripts/functional_chain_evidence.py verify-report --suite table --report "${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"
|
||||
|
||||
@@ -37,9 +37,6 @@ on:
|
||||
required: false
|
||||
default: false
|
||||
type: boolean
|
||||
repository_dispatch:
|
||||
# Chain handoff: dispatched when the KMS suite finishes.
|
||||
types: [rustfs-chain-tier]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -68,7 +65,7 @@ jobs:
|
||||
tier-test:
|
||||
runs-on: smoke-testing
|
||||
timeout-minutes: 60
|
||||
if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
|
||||
if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' }}
|
||||
steps:
|
||||
- name: Checkout chain tooling
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
@@ -216,6 +213,13 @@ jobs:
|
||||
openssl version
|
||||
df -h /data | tail -1
|
||||
|
||||
- name: Admit clean shared environment
|
||||
id: environment
|
||||
timeout-minutes: 5
|
||||
env:
|
||||
FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-tier.json
|
||||
run: python3 scripts/functional_environment.py enter --suite tier
|
||||
|
||||
- name: Cleanup environment (before)
|
||||
timeout-minutes: 5
|
||||
run: |
|
||||
@@ -269,8 +273,7 @@ jobs:
|
||||
- name: Run tier suite
|
||||
id: test
|
||||
timeout-minutes: 45
|
||||
# Case failures keep the run green: the report and the backlog
|
||||
# issue manager carry the product signal.
|
||||
# Collect reports and backlog evidence before the final suite verdict.
|
||||
continue-on-error: true
|
||||
env:
|
||||
PACKAGE_URL_INPUT: ${{ inputs.package_url }}
|
||||
@@ -467,8 +470,9 @@ jobs:
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Cleanup environment (after)
|
||||
id: environment_cleanup
|
||||
timeout-minutes: 5
|
||||
if: always()
|
||||
if: ${{ always() && steps.environment.outcome == 'success' }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
sudo docker rm -f rustfs-test-mqtt >/dev/null 2>&1 || true
|
||||
@@ -488,6 +492,22 @@ jobs:
|
||||
'
|
||||
done
|
||||
|
||||
- name: Release environment only after successful cleanup
|
||||
if: ${{ always() && steps.environment.outcome == 'success' }}
|
||||
timeout-minutes: 5
|
||||
env:
|
||||
FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-tier.json
|
||||
CLEANUP_OUTCOME: ${{ steps.environment_cleanup.outcome }}
|
||||
run: python3 scripts/functional_environment.py finish --suite tier --cleanup-outcome "$CLEANUP_OUTCOME"
|
||||
|
||||
- name: Retain environment ownership and cleanup receipt
|
||||
if: ${{ always() }}
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: functional-environment-tier-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/environment-tier.json
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Cleanup pinned RustFS CLI
|
||||
timeout-minutes: 2
|
||||
if: always()
|
||||
@@ -664,55 +684,6 @@ jobs:
|
||||
--package-source "${PACKAGE_SOURCE}" \
|
||||
--date "$(date -u +%Y-%m-%d)"
|
||||
|
||||
- name: "Continue functional chain (next: Storage engine)"
|
||||
# Only chain-triggered runs forward to the next suite; standalone
|
||||
# workflow_dispatch runs stop after their own cleanup. A failed
|
||||
# handoff must never pass silently: it retries, then files an alert
|
||||
# issue in rustfs/backlog so a stalled chain is visible.
|
||||
if: ${{ always() && github.event_name == 'repository_dispatch' }}
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
run: |
|
||||
set -uo pipefail
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
|
||||
exit 1
|
||||
fi
|
||||
DISPATCHED=0
|
||||
for attempt in 1 2 3; do
|
||||
if gh api --method POST repos/rustfs/rustfs/dispatches \
|
||||
-f event_type='rustfs-chain-storage' \
|
||||
-F 'client_payload[from_suite]=tier'; then
|
||||
echo "dispatched next suite Storage engine (attempt ${attempt})"
|
||||
DISPATCHED=1
|
||||
break
|
||||
fi
|
||||
echo "dispatch attempt ${attempt} failed; retrying in ${attempt}0s" >&2
|
||||
sleep "${attempt}0"
|
||||
done
|
||||
if [ "${DISPATCHED:-0}" -ne 1 ]; then
|
||||
echo "ERROR: functional chain stalled: could not dispatch Storage engine after 3 attempts" >&2
|
||||
TITLE="[functional][chain] stalled after tier (run ${GITHUB_RUN_ID})"
|
||||
BODY_FILE="$(mktemp)"
|
||||
{
|
||||
echo "The functional chain could not hand off from **tier** to **Storage engine** after 3 attempts."
|
||||
echo ""
|
||||
echo "- Failed suite job: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
|
||||
echo "- Expected next event: 'rustfs-chain-storage'"
|
||||
echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
|
||||
echo "- Recovery: re-dispatch manually with"
|
||||
FENCE="$(printf "\x60\x60\x60")"; echo " ${FENCE}"
|
||||
echo " gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-storage'"
|
||||
FENCE="$(printf "\x60\x60\x60")"; echo " ${FENCE}"
|
||||
} > "${BODY_FILE}"
|
||||
gh issue create -R rustfs/backlog --title "${TITLE}" \
|
||||
--body-file "${BODY_FILE}" --label functional-test \
|
||||
|| gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}" \
|
||||
|| echo "could not file the stall alert issue either; check the token" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Notify on failure
|
||||
if: failure()
|
||||
run: |
|
||||
@@ -739,3 +710,13 @@ jobs:
|
||||
name: functional-chain-tier-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/chain-tier-${{ github.run_id }}-${{ github.run_attempt }}/tier.json
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Enforce suite and report result
|
||||
if: ${{ always() }}
|
||||
env:
|
||||
SUITE_OUTCOME: ${{ steps.test.outcome }}
|
||||
REPORT_OUTCOME: ${{ steps.chain_report.outcome }}
|
||||
run: |
|
||||
test "$SUITE_OUTCOME" = success
|
||||
test "$REPORT_OUTCOME" = success
|
||||
python3 scripts/functional_chain_evidence.py verify-report --suite tier --report "${TIER_ARTIFACTS_DIR}/rustfs-tier-cases.md"
|
||||
|
||||
@@ -59,9 +59,6 @@ on:
|
||||
description: 'Reset the nodes after the test (DESTROYS test data/config)'
|
||||
type: boolean
|
||||
default: true
|
||||
repository_dispatch:
|
||||
# Functional-chain entry: dispatched by rustfs-functional-chain.yml.
|
||||
types: [rustfs-chain-upgrade]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -86,7 +83,7 @@ jobs:
|
||||
upgrade-test:
|
||||
runs-on: smoke-testing
|
||||
timeout-minutes: 60
|
||||
if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
|
||||
if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' }}
|
||||
steps:
|
||||
- name: Checkout repository (for report parser)
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
@@ -132,6 +129,13 @@ jobs:
|
||||
docker --version || true
|
||||
df -h /data | tail -1
|
||||
|
||||
- name: Admit clean shared environment
|
||||
id: environment
|
||||
timeout-minutes: 5
|
||||
env:
|
||||
FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-upgrade.json
|
||||
run: python3 scripts/functional_environment.py enter --suite upgrade
|
||||
|
||||
- name: Cleanup environment (before)
|
||||
timeout-minutes: 5
|
||||
if: ${{ inputs.cleanup_before != 'false' || github.event_name != 'workflow_dispatch' }}
|
||||
@@ -165,8 +169,7 @@ jobs:
|
||||
- name: Run upgrade compatibility suite
|
||||
id: test
|
||||
timeout-minutes: 45
|
||||
# Case failures keep the run green: the report and the backlog
|
||||
# issue manager carry the product signal.
|
||||
# Collect reports and backlog evidence before the final suite verdict.
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
@@ -258,10 +261,8 @@ jobs:
|
||||
RESULT=success
|
||||
fi
|
||||
|
||||
# Product gate: failing cases keep the run green — they are reported
|
||||
# below and tracked in rustfs/backlog. Only harness/environment
|
||||
# breakdowns (the suite never reached case level, or a wholesale
|
||||
# failure with zero passes) turn the workflow red.
|
||||
# Report harness completion here; the final verdict checks every
|
||||
# case result after diagnostics and cleanup have been retained.
|
||||
CASES_TOTAL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| (PASS|FAIL|UNSUPPORTED|RUNNING) \|' "${CASE_TABLE}" 2>/dev/null || true)"
|
||||
CASES_FAIL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| FAIL \|' "${CASE_TABLE}" 2>/dev/null || true)"
|
||||
CASES_TOTAL=$(( ${CASES_TOTAL:-0} + 0 )); CASES_FAIL=$(( ${CASES_FAIL:-0} + 0 ))
|
||||
@@ -304,7 +305,7 @@ jobs:
|
||||
fi
|
||||
} | tee "${REPORT_FILE}"
|
||||
cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
|
||||
# Red only for harness/environment breakdowns; case failures stay green.
|
||||
# Preserve the harness verdict separately from the final case verdict.
|
||||
[ "${HARNESS_OK}" = "1" ]
|
||||
|
||||
- name: Upload functional report to dashboard
|
||||
@@ -412,8 +413,9 @@ jobs:
|
||||
retention-days: 3
|
||||
|
||||
- name: Cleanup environment (after)
|
||||
id: environment_cleanup
|
||||
timeout-minutes: 5
|
||||
if: ${{ always() && (inputs.cleanup_after != 'false' || github.event_name != 'workflow_dispatch') }}
|
||||
if: ${{ always() && (inputs.cleanup_after != 'false' || github.event_name != 'workflow_dispatch') && steps.environment.outcome == 'success' }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
|
||||
@@ -431,54 +433,21 @@ jobs:
|
||||
'
|
||||
done
|
||||
|
||||
- name: "Continue functional chain (next: S3 compatibility)"
|
||||
# Only chain-triggered runs forward to the next suite; standalone
|
||||
# workflow_dispatch runs stop after their own cleanup. A failed
|
||||
# handoff must never pass silently: it retries, then files an alert
|
||||
# issue in rustfs/backlog so a stalled chain is visible.
|
||||
if: ${{ always() && github.event_name == 'repository_dispatch' }}
|
||||
continue-on-error: true
|
||||
- name: Release environment only after successful cleanup
|
||||
if: ${{ always() && steps.environment.outcome == 'success' }}
|
||||
timeout-minutes: 5
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
run: |
|
||||
set -uo pipefail
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
|
||||
exit 1
|
||||
fi
|
||||
DISPATCHED=0
|
||||
for attempt in 1 2 3; do
|
||||
if gh api --method POST repos/rustfs/rustfs/dispatches \
|
||||
-f event_type='rustfs-chain-s3' \
|
||||
-F 'client_payload[from_suite]=upgrade'; then
|
||||
echo "dispatched next suite S3 compatibility (attempt ${attempt})"
|
||||
DISPATCHED=1
|
||||
break
|
||||
fi
|
||||
echo "dispatch attempt ${attempt} failed; retrying in ${attempt}0s" >&2
|
||||
sleep "${attempt}0"
|
||||
done
|
||||
if [ "${DISPATCHED:-0}" -ne 1 ]; then
|
||||
echo "ERROR: functional chain stalled: could not dispatch S3 compatibility after 3 attempts" >&2
|
||||
TITLE="[functional][chain] stalled after upgrade (run ${GITHUB_RUN_ID})"
|
||||
BODY_FILE="$(mktemp)"
|
||||
{
|
||||
echo "The functional chain could not hand off from **upgrade** to **S3 compatibility** after 3 attempts."
|
||||
echo ""
|
||||
echo "- Failed suite job: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
|
||||
echo "- Expected next event: 'rustfs-chain-s3'"
|
||||
echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
|
||||
echo "- Recovery: re-dispatch manually with"
|
||||
FENCE="$(printf "\x60\x60\x60")"; echo " ${FENCE}"
|
||||
echo " gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-s3'"
|
||||
FENCE="$(printf "\x60\x60\x60")"; echo " ${FENCE}"
|
||||
} > "${BODY_FILE}"
|
||||
gh issue create -R rustfs/backlog --title "${TITLE}" \
|
||||
--body-file "${BODY_FILE}" --label functional-test \
|
||||
|| gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}" \
|
||||
|| echo "could not file the stall alert issue either; check the token" >&2
|
||||
exit 1
|
||||
fi
|
||||
FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-upgrade.json
|
||||
CLEANUP_OUTCOME: ${{ steps.environment_cleanup.outcome }}
|
||||
run: python3 scripts/functional_environment.py finish --suite upgrade --cleanup-outcome "$CLEANUP_OUTCOME"
|
||||
|
||||
- name: Retain environment ownership and cleanup receipt
|
||||
if: ${{ always() }}
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: functional-environment-upgrade-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/environment-upgrade.json
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Notify on failure
|
||||
if: failure()
|
||||
@@ -508,3 +477,13 @@ jobs:
|
||||
name: functional-chain-upgrade-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/chain-upgrade-${{ github.run_id }}-${{ github.run_attempt }}/upgrade.json
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Enforce suite and report result
|
||||
if: ${{ always() }}
|
||||
env:
|
||||
SUITE_OUTCOME: ${{ steps.test.outcome }}
|
||||
REPORT_OUTCOME: ${{ steps.chain_report.outcome }}
|
||||
run: |
|
||||
test "$SUITE_OUTCOME" = success
|
||||
test "$REPORT_OUTCOME" = success
|
||||
python3 scripts/functional_chain_evidence.py verify-report --suite upgrade --report "${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"
|
||||
|
||||
@@ -0,0 +1,457 @@
|
||||
# Copyright 2024 RustFS Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
name: RustFS Validation Sweep
|
||||
|
||||
# Executes the migrated canonical validation cases (rustfs/auto-testing#113)
|
||||
# on the shared functional fleet. Everything with a lane executor is swept in
|
||||
# rotating nightly batches so the whole set is covered roughly weekly; a
|
||||
# single case or one module can be dispatched manually. That is the
|
||||
# implemented catalog (66 today) plus the eight planned cases whose dedicated
|
||||
# executors already exist and have live PASS evidence - fresh nightly
|
||||
# evidence feeds their acceptance review. heal.* cases stay on the
|
||||
# interactive validation flow: they require reviewed native/tool collectors
|
||||
# built by separate validation-tools runs before fleet execution.
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# After the nightly functional chain settles (chain starts ~16:48 UTC and
|
||||
# runs for about two hours).
|
||||
- cron: "30 19 * * *"
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
case:
|
||||
description: 'Single canonical validation case id (overrides module and rotation), e.g. pool.decommission'
|
||||
type: string
|
||||
required: false
|
||||
default: ''
|
||||
module:
|
||||
description: 'Restrict the sweep to one catalog module, e.g. bucket-replication'
|
||||
type: string
|
||||
required: false
|
||||
default: ''
|
||||
limit:
|
||||
description: 'Maximum cases per sweep (0 = run everything selected)'
|
||||
type: number
|
||||
required: false
|
||||
default: 12
|
||||
build_run_id:
|
||||
description: 'Successful nightly package run; empty resolves the latest'
|
||||
type: string
|
||||
required: false
|
||||
default: ''
|
||||
build_run_attempt:
|
||||
description: 'Attempt of that package run'
|
||||
type: string
|
||||
required: false
|
||||
default: '1'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: rustfs-validation-sweep
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES || 'vm000 vm001 vm002 vm003' }}
|
||||
RUSTFS_SSH_USER: ${{ secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER || 'azureuser' }}
|
||||
RUSTFS_ACCESS_KEY: ${{ secrets.RUSTFS_ACCESS_KEY }}
|
||||
RUSTFS_SECRET_KEY: ${{ secrets.RUSTFS_SECRET_KEY }}
|
||||
RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
|
||||
PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
|
||||
jobs:
|
||||
sweep:
|
||||
if: github.repository == 'rustfs/rustfs'
|
||||
runs-on: smoke-testing
|
||||
timeout-minutes: 150
|
||||
steps:
|
||||
- name: Checkout chain tooling
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Resolve nightly candidate
|
||||
id: candidate
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
BUILD_RUN_ID: ${{ inputs.build_run_id }}
|
||||
BUILD_RUN_ATTEMPT: ${{ inputs.build_run_attempt }}
|
||||
CHAIN_OUTPUT: ${{ runner.temp }}/validation-sweep-candidate.json
|
||||
run: |
|
||||
set -euo pipefail
|
||||
# Scheduled runs carry no inputs and the resolver only derives a
|
||||
# build from a workflow_run payload (which a schedule cannot
|
||||
# provide), so resolve the newest successful nightly build
|
||||
# explicitly before invoking it.
|
||||
if [ -z "${BUILD_RUN_ID:-}" ]; then
|
||||
LATEST="$(gh api "repos/rustfs/rustfs/actions/workflows/nightly-gnu.yml/runs?branch=main&status=success&per_page=1" --jq '.workflow_runs[0]')"
|
||||
BUILD_RUN_ID="$(printf '%s' "${LATEST}" | jq -r '.id')"
|
||||
BUILD_RUN_ATTEMPT="$(printf '%s' "${LATEST}" | jq -r '.run_attempt')"
|
||||
export BUILD_RUN_ID BUILD_RUN_ATTEMPT
|
||||
echo "scheduled sweep pinned to nightly ${BUILD_RUN_ID} attempt ${BUILD_RUN_ATTEMPT}"
|
||||
fi
|
||||
python3 scripts/resolve_functional_candidate.py
|
||||
python3 - <<'PY'
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
root = Path(os.environ["RUNNER_TEMP"]) / "validation-sweep"
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
chain = json.loads((Path(os.environ["RUNNER_TEMP"]) / "validation-sweep-candidate.json").read_text())
|
||||
manifest = chain["candidate"]["manifest"]
|
||||
(root / "candidate.json").write_text(json.dumps({
|
||||
"package_url": manifest["package_url"],
|
||||
"package_sha256": manifest["package_sha256"],
|
||||
"source_sha": manifest["source_sha"],
|
||||
}, sort_keys=True) + "\n")
|
||||
with open(os.environ["GITHUB_OUTPUT"], "a") as output:
|
||||
output.write("testing_sha=" + chain["testing_sha"] + "\n")
|
||||
output.write("package_url=" + manifest["package_url"] + "\n")
|
||||
output.write("source_sha=" + manifest["source_sha"] + "\n")
|
||||
PY
|
||||
|
||||
- name: Checkout auto-testing scripts
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
repository: rustfs/auto-testing
|
||||
ref: ${{ steps.candidate.outputs.testing_sha }}
|
||||
token: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
path: auto-testing
|
||||
persist-credentials: false
|
||||
|
||||
- name: Prepare validation Python runtime
|
||||
# The pool / ilm / replication validation runners require a managed
|
||||
# Python, exactly like the chain's validation job provides.
|
||||
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- name: Install validation Python requirements
|
||||
# Case scripts call python3 and import boto3 at startup. Install the
|
||||
# functional chain's client into a per-run env so that interpreter
|
||||
# sees it without changing the runner's shared Python.
|
||||
run: |
|
||||
set -euo pipefail
|
||||
validation_python="${RUNNER_TEMP}/rustfs-validation-python-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
|
||||
test ! -e "${validation_python}"
|
||||
python3 -m venv "${validation_python}"
|
||||
"${validation_python}/bin/python" -m pip install \
|
||||
-r auto-testing/scripts/validation_ilm/requirements.txt
|
||||
printf '%s\n' "${validation_python}/bin" >> "${GITHUB_PATH}"
|
||||
|
||||
- name: Select sweep cases
|
||||
id: select
|
||||
env:
|
||||
SELECTED_CASE: ${{ inputs.case }}
|
||||
SELECTED_MODULE: ${{ inputs.module }}
|
||||
CASE_LIMIT: ${{ inputs.limit }}
|
||||
SWEEP_EVENT: ${{ github.event_name }}
|
||||
SWEEP_ROOT: ${{ runner.temp }}/validation-sweep
|
||||
run: |
|
||||
set -euo pipefail
|
||||
python3 - <<'PY'
|
||||
import datetime
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
sweep_root = Path(os.environ["SWEEP_ROOT"])
|
||||
|
||||
catalog = json.loads(Path("auto-testing/validation-cases.json").read_text())
|
||||
def sweepable(case):
|
||||
status = case.get("source_status")
|
||||
if status == "implemented":
|
||||
return True
|
||||
# The remaining planned non-heal cases route to the lane
|
||||
# executors and already have live PASS evidence; sweeping them
|
||||
# nightly produces fresh evidence for their acceptance review.
|
||||
return status == "planned" and not case["id"].startswith("heal.")
|
||||
|
||||
selected = [(case["id"], module["id"]) for module in catalog["modules"]
|
||||
for case in module["cases"]
|
||||
if sweepable(case)]
|
||||
if not selected:
|
||||
raise SystemExit("catalog has no sweepable cases")
|
||||
|
||||
one_case = os.environ.get("SELECTED_CASE", "").strip()
|
||||
module_filter = os.environ.get("SELECTED_MODULE", "").strip()
|
||||
known = {case_id for case_id, _ in selected}
|
||||
modules = {module_id for _, module_id in selected}
|
||||
if one_case:
|
||||
if one_case not in known:
|
||||
raise SystemExit(f"unknown case or no lane executor for it: {one_case}")
|
||||
selected = [(one_case, dict(selected)[one_case])]
|
||||
elif module_filter:
|
||||
if module_filter not in modules:
|
||||
raise SystemExit(f"unknown module or no implemented cases: {module_filter}")
|
||||
selected = [pair for pair in selected if pair[1] == module_filter]
|
||||
elif os.environ.get("SWEEP_EVENT") == "schedule":
|
||||
# Stateless rotation: every night take one window of the catalog
|
||||
# so a full sweep needs len(selected)/limit nights and no state
|
||||
# has to be carried between runs.
|
||||
limit = max(1, int(os.environ.get("CASE_LIMIT") or 12))
|
||||
day = datetime.datetime.now(datetime.timezone.utc).timetuple().tm_yday
|
||||
start = (day * limit) % len(selected)
|
||||
selected = (selected[start:] + selected[:start])[:limit]
|
||||
limit = int(os.environ.get("CASE_LIMIT") or 0)
|
||||
if one_case or module_filter or (not os.environ.get("SWEEP_EVENT") == "schedule"):
|
||||
if limit > 0:
|
||||
selected = selected[:limit]
|
||||
|
||||
(sweep_root / "sweep-cases.txt").write_text("".join(f"{case_id}\n" for case_id, _ in selected))
|
||||
with open(os.environ["GITHUB_OUTPUT"], "a") as output:
|
||||
output.write(f"count={len(selected)}\n")
|
||||
print(f"selected {len(selected)} case(s):", ", ".join(c for c, _ in selected))
|
||||
PY
|
||||
|
||||
- name: Admit clean shared environment
|
||||
id: environment
|
||||
timeout-minutes: 5
|
||||
env:
|
||||
FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-validation-sweep.json
|
||||
run: python3 scripts/functional_environment.py enter --suite heal
|
||||
|
||||
- name: Run validation cases
|
||||
id: run_cases
|
||||
timeout-minutes: 120
|
||||
env:
|
||||
AUTO_TESTING_SHA: ${{ steps.candidate.outputs.testing_sha }}
|
||||
SWEEP_ROOT: ${{ runner.temp }}/validation-sweep
|
||||
run: |
|
||||
set -uo pipefail
|
||||
mkdir -p "${SWEEP_ROOT}/results"
|
||||
: > "${SWEEP_ROOT}/results.txt"
|
||||
while read -r CASE_ID; do
|
||||
[ -n "${CASE_ID}" ] || continue
|
||||
echo "::group::case ${CASE_ID}"
|
||||
rm -rf "${SWEEP_ROOT}/run"
|
||||
if ./auto-testing/scripts/run_validation_case.sh \
|
||||
"${CASE_ID}" "${SWEEP_ROOT}/candidate.json" \
|
||||
"${AUTO_TESTING_SHA}" "${SWEEP_ROOT}/run" \
|
||||
> "${SWEEP_ROOT}/results/${CASE_ID//\//_}.log" 2>&1; then
|
||||
printf '%s\tPASS\n' "${CASE_ID}" >> "${SWEEP_ROOT}/results.txt"
|
||||
echo "case ${CASE_ID}: PASS"
|
||||
else
|
||||
printf '%s\tFAIL\n' "${CASE_ID}" >> "${SWEEP_ROOT}/results.txt"
|
||||
echo "case ${CASE_ID}: FAIL (log retained in the sweep artifact)"
|
||||
fi
|
||||
if [ -f "${SWEEP_ROOT}/run/report.md" ]; then
|
||||
cp "${SWEEP_ROOT}/run/report.md" "${SWEEP_ROOT}/results/${CASE_ID//\//_}.report.md" 2>/dev/null || true
|
||||
fi
|
||||
echo "::endgroup::"
|
||||
# Fleet hygiene between cases: the case runners set up their own
|
||||
# topology, this purge guarantees the next case starts clean even
|
||||
# when the previous one died mid-run.
|
||||
for node in ${RUSTFS_NODES}; do
|
||||
# -n keeps ssh from consuming the case list this loop reads from stdin
|
||||
ssh -n -o BatchMode=yes -o ConnectTimeout=10 "${RUSTFS_SSH_USER}@${node}" '
|
||||
SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
|
||||
${SUDO} systemctl stop rustfs 2>/dev/null || true
|
||||
if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
|
||||
${SUDO} dpkg -P rustfs
|
||||
fi
|
||||
for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
|
||||
${SUDO} rm -f /etc/default/rustfs
|
||||
' || echo "inter-case cleanup failed on ${node} (continuing)"
|
||||
done
|
||||
done < "${SWEEP_ROOT}/sweep-cases.txt"
|
||||
echo "sweep finished: $(wc -l < "${SWEEP_ROOT}/results.txt") case(s) executed"
|
||||
|
||||
- name: Generate sweep report
|
||||
if: ${{ always() && steps.environment.outcome == 'success' }}
|
||||
env:
|
||||
SWEEP_ROOT: ${{ runner.temp }}/validation-sweep
|
||||
PACKAGE_URL: ${{ steps.candidate.outputs.package_url }}
|
||||
SOURCE_SHA: ${{ steps.candidate.outputs.source_sha }}
|
||||
AUTO_TESTING_SHA: ${{ steps.candidate.outputs.testing_sha }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
python3 - <<'PY'
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
root = Path(os.environ["SWEEP_ROOT"])
|
||||
catalog = json.loads(Path("auto-testing/validation-cases.json").read_text())
|
||||
module_of = {case["id"]: module["id"] for module in catalog["modules"] for case in module["cases"]}
|
||||
rows = []
|
||||
for line in (root / "results.txt").read_text().splitlines():
|
||||
if not line:
|
||||
continue
|
||||
case_id, status = line.split("\t")
|
||||
rows.append((case_id, module_of.get(case_id, "?"), status))
|
||||
passed = sum(1 for _, _, s in rows if s == "PASS")
|
||||
failed = len(rows) - passed
|
||||
run_url = (f"{os.environ.get('GITHUB_SERVER_URL', 'https://github.com')}/"
|
||||
f"{os.environ.get('GITHUB_REPOSITORY', 'rustfs/rustfs')}/actions/runs/"
|
||||
f"{os.environ.get('GITHUB_RUN_ID', '')}")
|
||||
|
||||
report = [
|
||||
"# RustFS validation sweep report", "",
|
||||
f"- Run: {run_url}",
|
||||
f"- Attempt: {os.environ.get('GITHUB_RUN_ATTEMPT', '')}",
|
||||
f"- Trigger: {os.environ.get('GITHUB_EVENT_NAME', '')}",
|
||||
f"- Package: {os.environ.get('PACKAGE_URL', 'nightly (R2 latest)')}",
|
||||
f"- Source SHA: {os.environ.get('SOURCE_SHA', '')}",
|
||||
f"- auto-testing: {os.environ.get('AUTO_TESTING_SHA', '')}", "",
|
||||
"Canonical validation cases (rustfs/auto-testing#113) executed by the sweep, grouped by catalog module.", "",
|
||||
]
|
||||
# Group the executed cases by catalog module (the major category:
|
||||
# pool lifecycle, heal, ILM expiration, bucket/site replication, kms,
|
||||
# data services, identity, s3 compatibility). Per-module subtotals
|
||||
# deliberately use "- Subtotal:" so the dashboard overview regexes
|
||||
# (which anchor on "- Total:") keep reading the grand totals below.
|
||||
by_module = {}
|
||||
module_order = []
|
||||
for case_id, module, status in rows:
|
||||
if module not in by_module:
|
||||
by_module[module] = []
|
||||
module_order.append(module)
|
||||
by_module[module].append((case_id, status))
|
||||
report += ["## Case Summary", ""]
|
||||
for module in module_order:
|
||||
cases = by_module[module]
|
||||
mod_pass = sum(1 for _, st in cases if st == "PASS")
|
||||
mod_fail = len(cases) - mod_pass
|
||||
report += [f"### {module} ({len(cases)})", "",
|
||||
"| Case | Status |", "| --- | --- |"]
|
||||
report += [f"| {case_id} | {status} |" for case_id, status in cases]
|
||||
report += ["", f"- Subtotal: {mod_pass} passed, {mod_fail} failed", ""]
|
||||
report += [
|
||||
"## Totals", "",
|
||||
f"- Total: {len(rows)}",
|
||||
f"- PASS: {passed}",
|
||||
f"- FAIL: {failed}",
|
||||
"- UNSUPPORTED: 0", "",
|
||||
"Failing cases keep this workflow red only as a report; the sweep artifact",
|
||||
"retains every case log and per-case report for diagnosis.", "",
|
||||
]
|
||||
(root / "report.md").write_text("\n".join(report))
|
||||
(root / "cases.md").write_text("\n".join(f"| {c} | {m} | {s} |" for c, m, s in rows) + "\n")
|
||||
PY
|
||||
cat "${SWEEP_ROOT}/report.md" >> "${GITHUB_STEP_SUMMARY}"
|
||||
|
||||
- name: Upload functional report to dashboard
|
||||
if: ${{ always() && steps.environment.outcome == 'success' }}
|
||||
timeout-minutes: 2
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
|
||||
REPORT_FILE: ${{ runner.temp }}/validation-sweep/report.md
|
||||
SUITE: validation
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; skipping dashboard upload"
|
||||
exit 0
|
||||
fi
|
||||
DATE="$(date -u +%Y-%m-%d)"
|
||||
REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
|
||||
B64_FILE="$(mktemp)"
|
||||
python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
|
||||
SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
|
||||
if [ -n "${SHA}" ]; then
|
||||
jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
|
||||
'{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
|
||||
else
|
||||
jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
|
||||
'{message:$msg, content:($content|rtrimstr("\n"))}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
|
||||
fi
|
||||
echo "dashboard updated: ${REPORT_PATH}"
|
||||
|
||||
- name: Retain sweep evidence
|
||||
if: ${{ always() }}
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: rustfs-validation-sweep-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: |
|
||||
${{ runner.temp }}/validation-sweep/report.md
|
||||
${{ runner.temp }}/validation-sweep/results.txt
|
||||
${{ runner.temp }}/validation-sweep/sweep-cases.txt
|
||||
${{ runner.temp }}/validation-sweep/candidate.json
|
||||
${{ runner.temp }}/validation-sweep/results/
|
||||
${{ runner.temp }}/validation-sweep/run/
|
||||
# warn (not error): an early infra failure legitimately leaves no
|
||||
# per-case evidence, and the failing step is already red.
|
||||
if-no-files-found: warn
|
||||
retention-days: 30
|
||||
|
||||
- name: Cleanup environment (after)
|
||||
id: environment_cleanup
|
||||
timeout-minutes: 5
|
||||
if: ${{ always() && steps.environment.outcome == 'success' }}
|
||||
run: |
|
||||
set -uo pipefail
|
||||
cleanup_status=0
|
||||
for node in ${RUSTFS_NODES}; do
|
||||
ssh -o BatchMode=yes -o ConnectTimeout=10 "${RUSTFS_SSH_USER}@${node}" '
|
||||
set -euo pipefail
|
||||
SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
|
||||
${SUDO} systemctl stop rustfs 2>/dev/null || true
|
||||
if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
|
||||
${SUDO} dpkg -P rustfs
|
||||
fi
|
||||
for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
|
||||
${SUDO} rm -f /etc/default/rustfs
|
||||
' || cleanup_status=$?
|
||||
done
|
||||
exit "${cleanup_status}"
|
||||
|
||||
- name: Release environment only after successful cleanup
|
||||
if: ${{ always() && steps.environment.outcome == 'success' }}
|
||||
timeout-minutes: 5
|
||||
env:
|
||||
FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-validation-sweep.json
|
||||
CLEANUP_OUTCOME: ${{ steps.environment_cleanup.outcome }}
|
||||
run: python3 scripts/functional_environment.py finish --suite heal --cleanup-outcome "$CLEANUP_OUTCOME"
|
||||
|
||||
- name: Notify on failure
|
||||
if: failure()
|
||||
run: echo "RustFS validation sweep failed (harness/environment breakdown); see the sweep artifact for case logs."
|
||||
|
||||
- name: Enforce sweep result
|
||||
if: ${{ always() && steps.environment.outcome == 'success' }}
|
||||
env:
|
||||
SWEEP_ROOT: ${{ runner.temp }}/validation-sweep
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if grep -q $'\tFAIL$' "${SWEEP_ROOT}/results.txt"; then
|
||||
echo "validation sweep has failing cases:"
|
||||
grep $'\tFAIL$' "${SWEEP_ROOT}/results.txt"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
alert-on-failure:
|
||||
name: Alert on scheduled failure
|
||||
# A scheduled sweep failure usually means a canonical validation case
|
||||
# regressed or the fleet environment broke; surface it as a tracking
|
||||
# issue instead of leaving it in the Actions tab.
|
||||
needs: [sweep]
|
||||
if: github.repository == 'rustfs/rustfs' && (always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure'))
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
contents: read
|
||||
issues: write
|
||||
steps:
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v5.0.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Open or update failure-tracking issue
|
||||
uses: ./.github/actions/schedule-failure-issue
|
||||
with:
|
||||
github-token: ${{ secrets.BACKLOG_ISSUE_TOKEN }}
|
||||
@@ -34,7 +34,7 @@ jobs:
|
||||
# self-hosted sm-standard-* pool is down, which is exactly when the
|
||||
# scheduled validations it watches stall.
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
timeout-minutes: 20
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
@@ -47,6 +47,7 @@ jobs:
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
RUSTFS_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
|
||||
NIGHTLY_SOURCE_REF: ${{ vars.NIGHTLY_BRANCH || 'main' }}
|
||||
run: |
|
||||
set +e
|
||||
python3 scripts/check_scheduled_validation_freshness.py \
|
||||
|
||||
@@ -16,19 +16,8 @@ name: Scheduled Validation Watchdog
|
||||
|
||||
on:
|
||||
workflow_run:
|
||||
workflows:
|
||||
- "Security Audit"
|
||||
- "Build and Release"
|
||||
- "Continuous Integration"
|
||||
- "coverage"
|
||||
- "e2e-nightly"
|
||||
- "e2e-distributed"
|
||||
- "e2e-s3tests"
|
||||
- "Fuzz"
|
||||
- "mint"
|
||||
- "minio-interop"
|
||||
- "Nightly GNU Build"
|
||||
- "Runner Hygiene"
|
||||
# The registry filters this broad subscription; names are not duplicated here.
|
||||
workflows: ["*"]
|
||||
types: [completed]
|
||||
|
||||
permissions:
|
||||
@@ -37,11 +26,7 @@ permissions:
|
||||
jobs:
|
||||
alert-on-incomplete-run:
|
||||
name: Alert on incomplete scheduled run
|
||||
if: >-
|
||||
github.repository == 'rustfs/rustfs' &&
|
||||
github.event.workflow_run.event == 'schedule' &&
|
||||
github.event.workflow_run.conclusion != 'success' &&
|
||||
github.event.workflow_run.conclusion != 'failure'
|
||||
if: github.repository == 'rustfs/rustfs'
|
||||
# Stay on GitHub-hosted runners: this monitor must keep running when the
|
||||
# self-hosted sm-standard-* pool is down, which is exactly when the
|
||||
# scheduled validations it watches stall.
|
||||
@@ -55,7 +40,11 @@ jobs:
|
||||
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Select expected completion from registry
|
||||
id: registry
|
||||
run: python3 scripts/validation_registry.py watchdog
|
||||
- name: Open or update incomplete-run issue
|
||||
if: steps.registry.outputs.selected == 'true'
|
||||
uses: ./.github/actions/schedule-failure-issue
|
||||
with:
|
||||
github-token: ${{ secrets.BACKLOG_ISSUE_TOKEN }}
|
||||
|
||||
@@ -107,6 +107,32 @@ jobs:
|
||||
shell: pwsh
|
||||
run: cargo test -p rustfs-ecstore --lib multipart_owned_record_parent_barriers_ -- --nocapture
|
||||
|
||||
# Native multipart completion publishes its payload from a different
|
||||
# parent than its metadata, and abort renames under a precondition; both
|
||||
# need their own Windows handle discipline.
|
||||
- name: Test native multipart complete and abort publication
|
||||
shell: pwsh
|
||||
run: >-
|
||||
cargo test -p rustfs-ecstore --lib --
|
||||
native_abort_missing_state_with_actual_owned_records_never_becomes_legacy_or_missing
|
||||
native_abort_pending_intent_is_not_a_delete_certificate_and_raw_cleanup_preserves_exact_evidence
|
||||
--nocapture
|
||||
|
||||
# Owned-undo reclaim removes a directory it pinned, copy heal persists
|
||||
# fault bytes through a handle, and metadata reads meet a directory.
|
||||
- name: Test native multipart reclaim, copy heal, and directory reads
|
||||
shell: pwsh
|
||||
run: >-
|
||||
cargo test -p rustfs-ecstore --lib --
|
||||
native_multipart_owned_undo
|
||||
native_multipart_copy_heal
|
||||
orphan_multipart_cleanup_rejects_metadata_directory_on_one_disk
|
||||
--nocapture
|
||||
|
||||
- name: Test directory removal fallback
|
||||
shell: pwsh
|
||||
run: "cargo test -p rustfs-ecstore --lib disk::fs::tests::test_remove -- --nocapture"
|
||||
|
||||
- name: Test Windows path conversion
|
||||
shell: pwsh
|
||||
run: "cargo test -p rustfs-utils --no-default-features --features path --lib dunce:: -- --nocapture"
|
||||
|
||||
@@ -26,6 +26,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
- IAM snapshot/deletion replay, target-assigned delete-marker purges, timestamp ordering, and best-effort peer broadcast now close the control-plane gaps found by the R6 review ([#7195](https://github.com/rustfs/rustfs/pull/7195)).
|
||||
- A `set-remote-target?update=true&creds=true` request on a site-replication peer's bucket target is now rejected with `400 InvalidRequest`, naming the target and why it counts as peer-owned. It used to return success while keeping the stored endpoint, credentials, and TLS settings, so an endpoint edit from the Console's bucket replication form appeared to apply but did not. Change a peer's address with a site-replication edit instead; see "Changing a site's address" in `docs/operations/site-replication-operations.md`. The `sync`, `bandwidth`, `proxy`, and `path` update groups are unchanged.
|
||||
- Site-replication edits that omit the site name, such as `mc admin replicate update --deployment-id <ID> --endpoint <URL>`, now keep the stored name on the editing site and on every receiver instead of renaming the site after the new endpoint host. No persisted format changes.
|
||||
- The persisted replication MRF journal is now replayed every 5 minutes while the server runs, not only at startup. Delete-marker purges whose targets were unreachable, and replications that overflowed the worker queues, now converge within one replay interval after the target returns instead of waiting for a restart. A target that stays down is re-attempted once per retained entry per interval across the cluster: the start time of the last pass is shared through a small `config/replication/mrf.bin.replay` object, so nodes do not each replay once per interval. Older versions ignore that object; existing persisted formats are unchanged.
|
||||
- Upgrade and rollback: upgrade every node in one site consecutively and verify reconciliation before moving to the next site; do not intentionally run a site mixed-version. Target-version ledger keys are harmless on rollback, although old code cannot use their routing. Before rolling back past [#7307](https://github.com/rustfs/rustfs/pull/7307), drain or repair every pending version purge: older code can free a retained version's data directory before its remote purge is acknowledged. See `docs/operations/site-replication-operations.md`.
|
||||
|
||||
### Security
|
||||
|
||||
Generated
+513
-286
File diff suppressed because it is too large
Load Diff
+72
-73
@@ -75,7 +75,7 @@ edition = "2024"
|
||||
license = "Apache-2.0"
|
||||
repository = "https://github.com/rustfs/rustfs"
|
||||
rust-version = "1.99.0"
|
||||
version = "1.0.2"
|
||||
version = "1.0.6"
|
||||
homepage = "https://rustfs.com"
|
||||
description = "RustFS is a high-performance distributed object storage software built using Rust, one of the most popular languages worldwide. "
|
||||
keywords = ["RustFS", "Minio", "object-storage", "filesystem", "s3"]
|
||||
@@ -92,62 +92,62 @@ redundant_clone = "warn"
|
||||
|
||||
[workspace.dependencies]
|
||||
# RustFS Internal Crates
|
||||
rustfs = { path = "./rustfs", version = "1.0.2" }
|
||||
rustfs-heal = { path = "crates/heal", version = "1.0.2" }
|
||||
rustfs-heal-contracts = { path = "crates/heal-contracts", version = "1.0.2" }
|
||||
rustfs-scanner-metrics = { path = "crates/scanner-metrics", version = "1.0.2" }
|
||||
rustfs-audit = { path = "crates/audit", version = "1.0.2" }
|
||||
rustfs-checksums = { path = "crates/checksums", version = "1.0.2" }
|
||||
rustfs-common = { path = "crates/common", version = "1.0.2" }
|
||||
rustfs-data-usage = { path = "crates/data-usage", version = "1.0.2" }
|
||||
rustfs-config = { path = "./crates/config", version = "1.0.2" }
|
||||
rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.2" }
|
||||
rustfs-credentials = { path = "crates/credentials", version = "1.0.2" }
|
||||
rustfs-crypto = { path = "crates/crypto", version = "1.0.2" }
|
||||
rustfs-ecstore = { path = "crates/ecstore", version = "1.0.2" }
|
||||
rustfs-filemeta = { path = "crates/filemeta", version = "1.0.2" }
|
||||
rustfs-iam = { path = "crates/iam", version = "1.0.2" }
|
||||
rustfs-keystone = { path = "crates/keystone", version = "1.0.2" }
|
||||
rustfs-license = { path = "crates/license", version = "1.0.2" }
|
||||
rustfs-lifecycle = { path = "crates/lifecycle", version = "1.0.2" }
|
||||
rustfs-kms = { path = "crates/kms", version = "1.0.2" }
|
||||
rustfs-lock = { path = "crates/lock", version = "1.0.2" }
|
||||
rustfs-madmin = { path = "crates/madmin", version = "1.0.2" }
|
||||
rustfs-notify = { path = "crates/notify", version = "1.0.2" }
|
||||
rustfs-io-metrics = { path = "crates/io-metrics", version = "1.0.2" }
|
||||
rustfs-io-core = { path = "crates/io-core", version = "1.0.2" }
|
||||
rustfs-object-capacity = { path = "crates/object-capacity", version = "1.0.2" }
|
||||
rustfs-object-data-cache = { path = "crates/object-data-cache", version = "1.0.2", default-features = false }
|
||||
rustfs-log-analyzer = { path = "crates/log-analyzer", version = "1.0.2" }
|
||||
rustfs-obs = { path = "crates/obs", version = "1.0.2" }
|
||||
rustfs-policy = { path = "crates/policy", version = "1.0.2" }
|
||||
rustfs-protos = { path = "crates/protos", version = "1.0.2" }
|
||||
rustfs-protocols = { path = "crates/protocols", version = "1.0.2" }
|
||||
rustfs-replication = { path = "crates/replication", version = "1.0.2" }
|
||||
rustfs-rio = { path = "crates/rio", version = "1.0.2" }
|
||||
rustfs-rio-v2 = { path = "crates/rio-v2", version = "1.0.2" }
|
||||
rustfs-s3-client = { path = "crates/s3-client", version = "1.0.2" }
|
||||
rustfs-s3-types = { path = "crates/s3-types", version = "1.0.2" }
|
||||
rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.2" }
|
||||
rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.2" }
|
||||
rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.2" }
|
||||
rustfs-scanner = { path = "crates/scanner", version = "1.0.2" }
|
||||
rustfs-security-governance = { path = "crates/security-governance", version = "1.0.2" }
|
||||
rustfs-extension-schema = { path = "crates/extension-schema", version = "1.0.2" }
|
||||
rustfs-signer = { path = "crates/signer", version = "1.0.2" }
|
||||
rustfs-storage-api = { path = "crates/storage-api", version = "1.0.2" }
|
||||
rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.2" }
|
||||
rustfs-targets = { path = "crates/targets", version = "1.0.2" }
|
||||
rustfs-test-utils = { path = "crates/test-utils", version = "1.0.2" }
|
||||
rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.2" }
|
||||
rustfs-utils = { path = "crates/utils", version = "1.0.2" }
|
||||
rustfs-zip = { path = "./crates/zip", version = "1.0.2" }
|
||||
rustfs = { path = "./rustfs", version = "1.0.6" }
|
||||
rustfs-heal = { path = "crates/heal", version = "1.0.6" }
|
||||
rustfs-heal-contracts = { path = "crates/heal-contracts", version = "1.0.6" }
|
||||
rustfs-scanner-metrics = { path = "crates/scanner-metrics", version = "1.0.6" }
|
||||
rustfs-audit = { path = "crates/audit", version = "1.0.6" }
|
||||
rustfs-checksums = { path = "crates/checksums", version = "1.0.6" }
|
||||
rustfs-common = { path = "crates/common", version = "1.0.6" }
|
||||
rustfs-data-usage = { path = "crates/data-usage", version = "1.0.6" }
|
||||
rustfs-config = { path = "./crates/config", version = "1.0.6" }
|
||||
rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.6" }
|
||||
rustfs-credentials = { path = "crates/credentials", version = "1.0.6" }
|
||||
rustfs-crypto = { path = "crates/crypto", version = "1.0.6" }
|
||||
rustfs-ecstore = { path = "crates/ecstore", version = "1.0.6" }
|
||||
rustfs-filemeta = { path = "crates/filemeta", version = "1.0.6" }
|
||||
rustfs-iam = { path = "crates/iam", version = "1.0.6" }
|
||||
rustfs-keystone = { path = "crates/keystone", version = "1.0.6" }
|
||||
rustfs-license = { path = "crates/license", version = "1.0.6" }
|
||||
rustfs-lifecycle = { path = "crates/lifecycle", version = "1.0.6" }
|
||||
rustfs-kms = { path = "crates/kms", version = "1.0.6" }
|
||||
rustfs-lock = { path = "crates/lock", version = "1.0.6" }
|
||||
rustfs-madmin = { path = "crates/madmin", version = "1.0.6" }
|
||||
rustfs-notify = { path = "crates/notify", version = "1.0.6" }
|
||||
rustfs-io-metrics = { path = "crates/io-metrics", version = "1.0.6" }
|
||||
rustfs-io-core = { path = "crates/io-core", version = "1.0.6" }
|
||||
rustfs-object-capacity = { path = "crates/object-capacity", version = "1.0.6" }
|
||||
rustfs-object-data-cache = { path = "crates/object-data-cache", version = "1.0.6", default-features = false }
|
||||
rustfs-log-analyzer = { path = "crates/log-analyzer", version = "1.0.6" }
|
||||
rustfs-obs = { path = "crates/obs", version = "1.0.6" }
|
||||
rustfs-policy = { path = "crates/policy", version = "1.0.6" }
|
||||
rustfs-protos = { path = "crates/protos", version = "1.0.6" }
|
||||
rustfs-protocols = { path = "crates/protocols", version = "1.0.6" }
|
||||
rustfs-replication = { path = "crates/replication", version = "1.0.6" }
|
||||
rustfs-rio = { path = "crates/rio", version = "1.0.6" }
|
||||
rustfs-rio-v2 = { path = "crates/rio-v2", version = "1.0.6" }
|
||||
rustfs-s3-client = { path = "crates/s3-client", version = "1.0.6" }
|
||||
rustfs-s3-types = { path = "crates/s3-types", version = "1.0.6" }
|
||||
rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.6" }
|
||||
rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.6" }
|
||||
rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.6" }
|
||||
rustfs-scanner = { path = "crates/scanner", version = "1.0.6" }
|
||||
rustfs-security-governance = { path = "crates/security-governance", version = "1.0.6" }
|
||||
rustfs-extension-schema = { path = "crates/extension-schema", version = "1.0.6" }
|
||||
rustfs-signer = { path = "crates/signer", version = "1.0.6" }
|
||||
rustfs-storage-api = { path = "crates/storage-api", version = "1.0.6" }
|
||||
rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.6" }
|
||||
rustfs-targets = { path = "crates/targets", version = "1.0.6" }
|
||||
rustfs-test-utils = { path = "crates/test-utils", version = "1.0.6" }
|
||||
rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.6" }
|
||||
rustfs-utils = { path = "crates/utils", version = "1.0.6" }
|
||||
rustfs-zip = { path = "./crates/zip", version = "1.0.6" }
|
||||
|
||||
# Async Runtime and Networking
|
||||
async-channel = "2.5.0"
|
||||
async_zip = { default-features = false, version = "0.0.19" }
|
||||
mysql_async = { default-features = false, version = "0.37.1" }
|
||||
async-compression = { version = "0.4.51" }
|
||||
async-compression = { version = "0.4.52" }
|
||||
async-trait = "0.1.92"
|
||||
async-nats = { version = "0.51.0", default-features = false }
|
||||
axum = "0.8.9"
|
||||
@@ -168,16 +168,16 @@ http-body-util = "0.1.5"
|
||||
memchr = "2.8.3"
|
||||
minlz = "1.2.3"
|
||||
reqwest = "0.13.5"
|
||||
rustfs-kafka-async = { version = "1.4.0" }
|
||||
rustfs-kafka-async = { version = "1.4.1" }
|
||||
socket2 = { version = "0.6.5" }
|
||||
tokio = { version = "1.54.1" }
|
||||
tokio-rustls = { default-features = false, version = "0.26.6" }
|
||||
tokio-stream = { version = "0.1.19" }
|
||||
tokio-test = "0.4.6"
|
||||
tokio-util = { version = "0.7.20" }
|
||||
tonic = { version = "0.14.8" }
|
||||
tonic-prost = { version = "0.14.8" }
|
||||
tonic-prost-build = { version = "0.14.8" }
|
||||
tonic = { version = "0.14.9" }
|
||||
tonic-prost = { version = "0.14.9" }
|
||||
tonic-prost-build = { version = "0.14.9" }
|
||||
tower = { version = "0.5.3" }
|
||||
tower-http = { version = "0.7.1" }
|
||||
h3 = "0.0.8"
|
||||
@@ -191,6 +191,7 @@ quinn = { version = "0.11.12", default-features = false, features = [
|
||||
# Serialization and Data Formats
|
||||
apache-avro = { version = "0.22.0", features = ["snappy", "zstandard"] }
|
||||
bytes = { version = "1.12.1" }
|
||||
csv-core = "0.1.13"
|
||||
bytesize = "2.7.0"
|
||||
byteorder = "1.5.0"
|
||||
flatbuffers = "25.12.19"
|
||||
@@ -212,12 +213,13 @@ serde_urlencoded = "0.7.1"
|
||||
# releases.
|
||||
aes-gcm = { version = "0.11.1" }
|
||||
argon2 = { version = "0.6.0" }
|
||||
aws-lc-rs = { version = "1.19.0", default-features = false }
|
||||
blake2 = "0.11.0"
|
||||
chacha20poly1305 = { version = "0.11.0" }
|
||||
crc-fast = "1.10.0"
|
||||
hmac = { version = "0.13.0" }
|
||||
jsonwebtoken = { version = "11.1.0" }
|
||||
openidconnect = { default-features = false, version = "4.0" }
|
||||
openidconnect = { default-features = false, version = "4.1.0" }
|
||||
pbkdf2 = "0.13.0"
|
||||
p256 = { version = "0.14.0", features = ["ecdsa", "pkcs8"] }
|
||||
rsa = { version = "=0.10.0-rc.18" }
|
||||
@@ -234,7 +236,7 @@ proptest = "1"
|
||||
# Time and Date
|
||||
chrono = { version = "0.4.45" }
|
||||
humantime = "2.4.0"
|
||||
jiff = { version = "0.2.38" }
|
||||
jiff = { version = "0.2.39" }
|
||||
time = { version = "0.3.55" }
|
||||
|
||||
# Database
|
||||
@@ -253,9 +255,9 @@ atoi = "3.1.1"
|
||||
atomic_enum = "0.3.0"
|
||||
aws-config = { version = "1.13.0" }
|
||||
aws-credential-types = { version = "1.3.1" }
|
||||
aws-sdk-kms = { default-features = false, version = "1.128.0" }
|
||||
aws-sdk-s3 = { default-features = false, version = "1.156.0" }
|
||||
aws-sdk-sts = { default-features = false, version = "1.123.0" }
|
||||
aws-sdk-kms = { default-features = false, version = "1.129.0" }
|
||||
aws-sdk-s3 = { default-features = false, version = "1.157.0" }
|
||||
aws-sdk-sts = { default-features = false, version = "1.124.0" }
|
||||
aws-smithy-async = { version = "1.3.1" }
|
||||
aws-smithy-http-client = { default-features = false, version = "1.5.2" }
|
||||
aws-smithy-runtime-api = { version = "1.19.2" }
|
||||
@@ -294,6 +296,7 @@ local-ip-address = "0.6.13"
|
||||
log = "0.4"
|
||||
memmap2 = "0.9.12"
|
||||
lz4 = "1.28.1"
|
||||
lz4_flex = { version = "=0.14.0", default-features = false, features = ["std", "frame"] }
|
||||
matchit = "0.9.2"
|
||||
md-5 = "0.11.0"
|
||||
mime_guess = "2.0.5"
|
||||
@@ -319,7 +322,7 @@ rustfs-erasure-codec = { version = "9.0.0" }
|
||||
reed-solomon-simd = "3.1.0"
|
||||
regex = { version = "1.13.1" }
|
||||
rumqttc = { package = "rumqttc-next", version = "0.34.0" }
|
||||
redis = { version = "1.8.0" }
|
||||
redis = { version = "1.8.1" }
|
||||
rustify = { version = "0.7", default-features = false }
|
||||
rustix = { version = "1.1.6" }
|
||||
rust-embed = { version = "8.13.0" }
|
||||
@@ -334,10 +337,10 @@ compact_str = "0.10.0"
|
||||
snap = "1.1.2"
|
||||
starshard = { version = "2.4.0" }
|
||||
strum = { version = "0.28.0" }
|
||||
sysinfo = "0.40.0"
|
||||
sysinfo = "0.40.1"
|
||||
temp-env = "0.3.6"
|
||||
tempfile = "3.27.0"
|
||||
test-case = "3.4.0"
|
||||
test-case = "3.4.1"
|
||||
thiserror = "2.0.21"
|
||||
tracing = { version = "0.1.44" }
|
||||
tracing-appender = "0.2.5"
|
||||
@@ -356,13 +359,13 @@ winapi-util = "0.1.11"
|
||||
windows = { version = "0.62.2" }
|
||||
windows-sys = "0.61.2"
|
||||
xxhash-rust = { version = "0.8.19" }
|
||||
zip = "9.1.3"
|
||||
zip = "9.3.0"
|
||||
zstd = "0.14.3"
|
||||
|
||||
# Observability and Metrics
|
||||
metrics = "0.24.6"
|
||||
metrics-util = "0.20"
|
||||
dial9 = { version = "0.5.6", default-features = false }
|
||||
dial9 = { version = "0.5.7", default-features = false }
|
||||
opentelemetry = { version = "0.33.1" }
|
||||
opentelemetry-appender-tracing = { version = "0.33.1" }
|
||||
opentelemetry-otlp = { version = "0.33.1" }
|
||||
@@ -377,16 +380,16 @@ libunftp = { version = "0.24.0" }
|
||||
unftp-core = "0.1.0"
|
||||
suppaftp = { version = "12.2.1" }
|
||||
rcgen = { version = "0.14.10", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
|
||||
russh = { version = "0.65.0" }
|
||||
russh = { version = "0.65.1" }
|
||||
russh-sftp = "3.0.1"
|
||||
|
||||
# WebDAV
|
||||
dav-server = "0.11.0"
|
||||
|
||||
# Performance Analysis and Memory Profiling
|
||||
rustfs-mimalloc = { version = "0.7.2" }
|
||||
rustfs-mimalloc = { version = "0.7.3" }
|
||||
# Preserve Unicode focus filters until rustfs/backlog#2302 is resolved.
|
||||
hotpath = { version = "0.30.1", default-features = false }
|
||||
hotpath = { version = "0.30.3", default-features = false }
|
||||
# Snapshot testing for output format regression detection
|
||||
insta = { version = "1.49" }
|
||||
|
||||
@@ -427,7 +430,3 @@ strip = "none"
|
||||
# Pin upstream fixes until compatible crates.io releases are available.
|
||||
[patch.crates-io]
|
||||
zstd = { path = "crates/zstd-compat" }
|
||||
# Preserve RustFS OIDC flows while migrating the old elliptic-curve dependency chain.
|
||||
# Upstream: openidconnect-rs#244 and oauth2-rs#352.
|
||||
openidconnect = { git = "https://github.com/ramosbugs/openidconnect-rs", rev = "2f77ee73bc4554e1cb6279adb386005429fb4e41" }
|
||||
oauth2 = { git = "https://github.com/ramosbugs/oauth2-rs", rev = "89585f4a280afd7b285eaf1c9be0068874e03d22" }
|
||||
|
||||
+11
-3
@@ -16,6 +16,7 @@ FROM alpine:3.24.2 AS build
|
||||
|
||||
ARG TARGETARCH
|
||||
ARG RELEASE=latest
|
||||
ARG RC_RELEASE=latest
|
||||
|
||||
RUN apk add --no-cache ca-certificates curl jq unzip
|
||||
WORKDIR /build
|
||||
@@ -69,6 +70,9 @@ RUN set -eux; \
|
||||
chmod +x /build/rustfs; \
|
||||
rm -rf rustfs.zip /build/.tmp || true
|
||||
|
||||
COPY scripts/download_rc.sh /usr/local/bin/download_rc.sh
|
||||
RUN sh /usr/local/bin/download_rc.sh "$TARGETARCH" musl "$RC_RELEASE" /build/rc
|
||||
|
||||
|
||||
FROM alpine:3.24.2
|
||||
|
||||
@@ -95,14 +99,18 @@ RUN apk upgrade --no-cache && \
|
||||
ca-certificates \
|
||||
coreutils \
|
||||
curl \
|
||||
jq \
|
||||
tzdata \
|
||||
&& test "$(TZ=Asia/Kolkata date +%z)" = "+0530"
|
||||
|
||||
COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
|
||||
COPY --from=build /build/rustfs /usr/bin/rustfs
|
||||
COPY entrypoint.sh /entrypoint.sh
|
||||
COPY --from=build /build/rc /usr/bin/rc
|
||||
COPY entrypoint.sh /etc/rustfs/entrypoint.sh
|
||||
COPY bootstrap/ /etc/rustfs/bootstrap/
|
||||
|
||||
RUN chmod +x /usr/bin/rustfs /entrypoint.sh
|
||||
RUN chmod +x /usr/bin/rustfs /usr/bin/rc /etc/rustfs/entrypoint.sh && \
|
||||
ln -s /etc/rustfs/entrypoint.sh /entrypoint.sh
|
||||
|
||||
RUN addgroup -g 10001 -S rustfs && \
|
||||
adduser -u 10001 -G rustfs -S rustfs -D && \
|
||||
@@ -124,6 +132,6 @@ VOLUME ["/data"]
|
||||
|
||||
USER rustfs
|
||||
|
||||
ENTRYPOINT ["/entrypoint.sh"]
|
||||
ENTRYPOINT ["/etc/rustfs/entrypoint.sh"]
|
||||
|
||||
CMD ["rustfs"]
|
||||
|
||||
@@ -12,7 +12,10 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
FROM rust:1.97.1-trixie
|
||||
FROM rust:1.99.1-trixie
|
||||
|
||||
ARG TARGETARCH
|
||||
ARG RC_RELEASE=latest
|
||||
|
||||
RUN set -eux; \
|
||||
export DEBIAN_FRONTEND=noninteractive; \
|
||||
@@ -21,6 +24,7 @@ RUN set -eux; \
|
||||
build-essential \
|
||||
ca-certificates \
|
||||
curl \
|
||||
jq \
|
||||
git \
|
||||
pkg-config \
|
||||
libssl-dev \
|
||||
@@ -36,6 +40,7 @@ COPY . .
|
||||
RUN ./scripts/static.sh
|
||||
RUN cargo run --bin gproto
|
||||
RUN cargo build --release --locked --bin rustfs
|
||||
RUN sh /usr/src/rustfs/scripts/download_rc.sh "$TARGETARCH" musl "$RC_RELEASE" /usr/local/bin/rc
|
||||
|
||||
RUN set -eux; \
|
||||
groupadd -g 10001 rustfs; \
|
||||
@@ -50,8 +55,12 @@ RUN set -eux; \
|
||||
chown -R rustfs:rustfs /data /logs /app; \
|
||||
chmod 0750 /data /logs
|
||||
|
||||
COPY entrypoint.sh /entrypoint.sh
|
||||
RUN install -m 0755 /usr/src/rustfs/target/release/rustfs /usr/bin/rustfs && chmod +x /entrypoint.sh
|
||||
COPY entrypoint.sh /etc/rustfs/entrypoint.sh
|
||||
COPY bootstrap/ /etc/rustfs/bootstrap/
|
||||
RUN install -m 0755 /usr/src/rustfs/target/release/rustfs /usr/bin/rustfs && \
|
||||
install -m 0755 /usr/local/bin/rc /usr/bin/rc && \
|
||||
chmod +x /etc/rustfs/entrypoint.sh && \
|
||||
ln -s /etc/rustfs/entrypoint.sh /entrypoint.sh
|
||||
|
||||
ENV RUSTFS_VOLUMES="/data" \
|
||||
RUST_LOG="warn" \
|
||||
@@ -63,5 +72,5 @@ ENV RUSTFS_VOLUMES="/data" \
|
||||
|
||||
EXPOSE 9000 9001
|
||||
|
||||
ENTRYPOINT ["/entrypoint.sh"]
|
||||
ENTRYPOINT ["/etc/rustfs/entrypoint.sh"]
|
||||
CMD ["/usr/bin/rustfs"]
|
||||
|
||||
+11
-3
@@ -16,6 +16,7 @@ FROM ubuntu:26.04 AS build
|
||||
|
||||
ARG TARGETARCH
|
||||
ARG RELEASE=latest
|
||||
ARG RC_RELEASE=latest
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
ca-certificates \
|
||||
@@ -76,6 +77,9 @@ RUN set -eux; \
|
||||
chmod +x /build/rustfs; \
|
||||
rm -rf rustfs.zip /build/.tmp || true
|
||||
|
||||
COPY scripts/download_rc.sh /usr/local/bin/download_rc.sh
|
||||
RUN sh /usr/local/bin/download_rc.sh "$TARGETARCH" gnu "$RC_RELEASE" /build/rc
|
||||
|
||||
FROM ubuntu:26.04
|
||||
|
||||
ARG RELEASE=latest
|
||||
@@ -99,14 +103,18 @@ RUN apt-get update && apt-get upgrade -y \
|
||||
&& DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
|
||||
ca-certificates \
|
||||
curl \
|
||||
jq \
|
||||
tzdata \
|
||||
&& test "$(TZ=Asia/Kolkata date +%z)" = "+0530" \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
COPY --from=build /build/rustfs /usr/bin/rustfs
|
||||
COPY entrypoint.sh /entrypoint.sh
|
||||
COPY --from=build /build/rc /usr/bin/rc
|
||||
COPY entrypoint.sh /etc/rustfs/entrypoint.sh
|
||||
COPY bootstrap/ /etc/rustfs/bootstrap/
|
||||
|
||||
RUN chmod +x /usr/bin/rustfs /entrypoint.sh
|
||||
RUN chmod +x /usr/bin/rustfs /usr/bin/rc /etc/rustfs/entrypoint.sh && \
|
||||
ln -s /etc/rustfs/entrypoint.sh /entrypoint.sh
|
||||
|
||||
RUN groupadd -g 10001 rustfs && \
|
||||
useradd -u 10001 -g rustfs -m -s /sbin/nologin rustfs && \
|
||||
@@ -131,6 +139,6 @@ VOLUME ["/data"]
|
||||
|
||||
USER rustfs
|
||||
|
||||
ENTRYPOINT ["/entrypoint.sh"]
|
||||
ENTRYPOINT ["/etc/rustfs/entrypoint.sh"]
|
||||
|
||||
CMD ["rustfs"]
|
||||
|
||||
+19
-7
@@ -28,17 +28,19 @@ ARG TARGETPLATFORM
|
||||
ARG BUILDPLATFORM
|
||||
ARG TARGETARCH
|
||||
ARG RUSTFS_BUILD_FEATURES=""
|
||||
ARG RC_RELEASE=latest
|
||||
|
||||
# -----------------------------
|
||||
# Build stage
|
||||
# -----------------------------
|
||||
FROM rust:1.97.1-trixie AS builder
|
||||
FROM rust:1.99.1-trixie AS builder
|
||||
|
||||
# Re-declare args after FROM
|
||||
ARG TARGETPLATFORM
|
||||
ARG BUILDPLATFORM
|
||||
ARG TARGETARCH
|
||||
ARG RUSTFS_BUILD_FEATURES
|
||||
ARG RC_RELEASE
|
||||
|
||||
# Debug: print platforms
|
||||
RUN echo "Build info -> BUILDPLATFORM=${BUILDPLATFORM}, TARGETPLATFORM=${TARGETPLATFORM}"
|
||||
@@ -52,6 +54,7 @@ RUN set -eux; \
|
||||
build-essential \
|
||||
ca-certificates \
|
||||
curl \
|
||||
jq \
|
||||
git \
|
||||
pkg-config \
|
||||
libssl-dev \
|
||||
@@ -166,6 +169,8 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry \
|
||||
;; \
|
||||
esac
|
||||
|
||||
RUN sh /usr/src/rustfs/scripts/download_rc.sh "$TARGETARCH" musl "$RC_RELEASE" /usr/local/bin/rc
|
||||
|
||||
# -----------------------------
|
||||
# Development stage (keeps toolchain)
|
||||
# -----------------------------
|
||||
@@ -199,10 +204,13 @@ ENV RUSTFS_VOLUMES="/data" \
|
||||
# We rely on cargo run to build and run
|
||||
EXPOSE 9000 9001
|
||||
|
||||
COPY entrypoint.sh /entrypoint.sh
|
||||
RUN chmod +x /entrypoint.sh
|
||||
COPY entrypoint.sh /etc/rustfs/entrypoint.sh
|
||||
COPY bootstrap/ /etc/rustfs/bootstrap/
|
||||
RUN install -m 0755 /usr/local/bin/rc /usr/bin/rc && \
|
||||
chmod +x /etc/rustfs/entrypoint.sh && \
|
||||
ln -s /etc/rustfs/entrypoint.sh /entrypoint.sh
|
||||
|
||||
ENTRYPOINT ["/entrypoint.sh"]
|
||||
ENTRYPOINT ["/etc/rustfs/entrypoint.sh"]
|
||||
CMD ["cargo", "run", "--bin", "rustfs", "--"]
|
||||
|
||||
# -----------------------------
|
||||
@@ -227,6 +235,7 @@ RUN set -eux; \
|
||||
apt-get install -y --no-install-recommends \
|
||||
ca-certificates \
|
||||
curl \
|
||||
jq \
|
||||
tzdata \
|
||||
coreutils; \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
@@ -248,8 +257,11 @@ RUN set -eux; \
|
||||
|
||||
# Copy the freshly built binary and the entrypoint
|
||||
COPY --from=builder /usr/local/bin/rustfs /usr/bin/rustfs
|
||||
COPY entrypoint.sh /entrypoint.sh
|
||||
RUN chmod +x /usr/bin/rustfs /entrypoint.sh
|
||||
COPY --from=builder /usr/local/bin/rc /usr/bin/rc
|
||||
COPY entrypoint.sh /etc/rustfs/entrypoint.sh
|
||||
COPY bootstrap/ /etc/rustfs/bootstrap/
|
||||
RUN chmod +x /usr/bin/rustfs /usr/bin/rc /etc/rustfs/entrypoint.sh && \
|
||||
ln -s /etc/rustfs/entrypoint.sh /entrypoint.sh
|
||||
|
||||
# Default environment (override in docker run/compose as needed)
|
||||
ENV RUSTFS_VOLUMES="/data" \
|
||||
@@ -263,5 +275,5 @@ EXPOSE 9000
|
||||
VOLUME ["/data"]
|
||||
|
||||
# Keep root here; entrypoint will drop privileges using chroot --userspec
|
||||
ENTRYPOINT ["/entrypoint.sh"]
|
||||
ENTRYPOINT ["/etc/rustfs/entrypoint.sh"]
|
||||
CMD ["/usr/bin/rustfs"]
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
# RustFS image bootstrap
|
||||
|
||||
The official image bundles this directory at `/etc/rustfs/bootstrap`. The default `buckets.json` is an empty JSON array, so the image starts RustFS directly and does not require a bootstrap endpoint or bootstrap credentials.
|
||||
|
||||
To create buckets, replace `buckets.json` in a derived image with a JSON array of bucket names. The entrypoint creates each bucket if it does not exist and applies `public-access-block.json`, whose default blocks all four public access paths. Edit that file in the derived image only when public access is an explicit requirement.
|
||||
|
||||
Optional per-bucket files use the bucket name as the filename without an extension:
|
||||
|
||||
| Directory | File format | Applied operation |
|
||||
| --- | --- | --- |
|
||||
| `policies/` | S3 bucket policy JSON | `rc bucket anonymous set-json` |
|
||||
| `versioning/` | JSON boolean (`true` or `false`) | Enable or suspend bucket versioning |
|
||||
| `lifecycle/` | `rc bucket lifecycle rule import` JSON | Replace lifecycle rules |
|
||||
| `cors/` | JSON or XML accepted by `rc bucket cors set` | Replace CORS rules |
|
||||
| `notifications/` | JSON array of `{ "arn": "...", "events": ["..."] }` | Add or replace event rules by ARN (prefix/suffix filters are not currently accepted) |
|
||||
|
||||
Every per-bucket file must correspond to a bucket listed in `buckets.json`. Applying these files is repeatable: bucket creation ignores existing buckets, while each configuration operation replaces or reconciles that bucket's requested state.
|
||||
|
||||
When `buckets.json` is non-empty, set `RUSTFS_BOOTSTRAP_ENDPOINT` to the RustFS S3 endpoint reachable from the container, for example `http://rustfs:9000` for a Compose service or the internal service DNS name in Kubernetes. The entrypoint waits on that endpoint's `/health/ready`; it never assumes `localhost`. `RUSTFS_BOOTSTRAP_TIMEOUT` controls the readiness wait in seconds (default `120`, maximum `3600`). `RUSTFS_BOOTSTRAP_DIR` can point to a different in-image configuration directory; it defaults to `/etc/rustfs/bootstrap`. Set `RUSTFS_ACCESS_KEY` and `RUSTFS_SECRET_KEY`, or their `_FILE` variants, for bootstrap authentication. The endpoint and credentials are read at runtime and are not baked into the image.
|
||||
|
||||
RustFS built-in IAM policies remain unchanged; the default bootstrap installs no custom bucket policies, lifecycle rules, versioning, CORS rules, or notification rules. The bundled RustFS CLI is resolved from a published release and its release checksum is verified. Set the Docker build argument `RC_RELEASE` to a version to pin it. Build the RustFS image after a CLI release containing the `bucket public-access-block` commands is available. For Kubernetes, the configured service must route traffic to the starting pod before its readiness probe succeeds (for example, with `publishNotReadyAddresses: true`); otherwise the service endpoint cannot reach the pod while the entrypoint waits for readiness.
|
||||
@@ -0,0 +1 @@
|
||||
[]
|
||||
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"block_public_acls": true,
|
||||
"ignore_public_acls": true,
|
||||
"block_public_policy": true,
|
||||
"restrict_public_buckets": true
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
|
||||
@@ -10,6 +10,10 @@ use std::time::Duration;
|
||||
use tokio::sync::{Notify, Semaphore};
|
||||
use tokio::time::Instant;
|
||||
|
||||
pub mod permit_set;
|
||||
|
||||
const MAX_PERMIT_REFERENCES: usize = usize::MAX / 2;
|
||||
|
||||
/// A resource owner publishes a capacity; running units retain their permits
|
||||
/// after a reduction. Lease slots are reserved once, with no per-unit Arc
|
||||
/// allocation. Tokio wait queues may lazily initialize platform mutex storage
|
||||
@@ -36,6 +40,7 @@ pub struct AdaptiveAdmission {
|
||||
maximum: u32,
|
||||
slots: Box<[PermitSlots]>,
|
||||
next_chunk: AtomicUsize,
|
||||
completed: AtomicU64,
|
||||
}
|
||||
|
||||
struct PermitSlots {
|
||||
@@ -61,10 +66,12 @@ pub struct AdaptivePermit {
|
||||
|
||||
impl Clone for AdaptivePermit {
|
||||
fn clone(&self) -> Self {
|
||||
// Every slot reference owns a strong owner reference. Clone the Arc
|
||||
// first, so its overflow protection also bounds the slot refcount.
|
||||
// Subset carriers can retain slot references through a shared owner,
|
||||
// so the slot count needs its own bound in addition to Arc's guard.
|
||||
let owner = Arc::clone(&self.owner);
|
||||
owner.slots[self.slot / 64].references[self.slot % 64].fetch_add(1, Ordering::Relaxed);
|
||||
if owner.slots[self.slot / 64].references[self.slot % 64].fetch_add(1, Ordering::Relaxed) >= MAX_PERMIT_REFERENCES {
|
||||
std::process::abort();
|
||||
}
|
||||
Self { owner, slot: self.slot }
|
||||
}
|
||||
}
|
||||
@@ -114,11 +121,22 @@ impl NodeAdmission {
|
||||
|
||||
/// Publish feedback with this owner's actual completed-work counter.
|
||||
pub fn update(&self, controller: &mut FeedbackController, observation: Option<FeedbackObservation>) -> FeedbackDecision {
|
||||
self.update_with_probe(controller, observation, true)
|
||||
}
|
||||
|
||||
/// Coordinate growth with other resource controllers in the same window.
|
||||
/// Protective reductions never depend on permission to probe.
|
||||
pub fn update_with_probe(
|
||||
&self,
|
||||
controller: &mut FeedbackController,
|
||||
observation: Option<FeedbackObservation>,
|
||||
allow_probe: bool,
|
||||
) -> FeedbackDecision {
|
||||
let observation = observation.map(|mut observation| {
|
||||
observation.completed_units = self.completed();
|
||||
observation
|
||||
});
|
||||
controller.update(&self.capacity, observation)
|
||||
controller.update_with_probe(&self.capacity, observation, allow_probe)
|
||||
}
|
||||
|
||||
/// Current leased scheduling cap and held async units.
|
||||
@@ -249,16 +267,31 @@ impl FeedbackController {
|
||||
/// Consume one sample and publish a three-second lease to the same gate.
|
||||
/// If this function stops being called, admission expires independently.
|
||||
pub fn update(&mut self, gate: &AdaptiveAdmission, observation: Option<FeedbackObservation>) -> FeedbackDecision {
|
||||
self.update_with_probe(gate, observation, true)
|
||||
}
|
||||
|
||||
/// Allow at most the sampling owner's selected controller to probe upward.
|
||||
pub fn update_with_probe(
|
||||
&mut self,
|
||||
gate: &AdaptiveAdmission,
|
||||
observation: Option<FeedbackObservation>,
|
||||
allow_probe: bool,
|
||||
) -> FeedbackDecision {
|
||||
let observation = observation.map(|mut observation| {
|
||||
observation.max_concurrency = observation.max_concurrency.min(gate.maximum);
|
||||
observation
|
||||
});
|
||||
let decision = self.decide(Instant::now(), observation);
|
||||
let decision = self.decide_with_probe(Instant::now(), observation, allow_probe);
|
||||
gate.set_sampled_limit(decision.limit, Duration::from_secs(3));
|
||||
decision
|
||||
}
|
||||
|
||||
fn decide(&mut self, now: Instant, observation: Option<FeedbackObservation>) -> FeedbackDecision {
|
||||
fn decide_with_probe(
|
||||
&mut self,
|
||||
now: Instant,
|
||||
observation: Option<FeedbackObservation>,
|
||||
allow_probe: bool,
|
||||
) -> FeedbackDecision {
|
||||
let Some(observation) = observation else {
|
||||
self.previous = None;
|
||||
self.stable_since = None;
|
||||
@@ -303,7 +336,8 @@ impl FeedbackController {
|
||||
FeedbackReason::NewBaseline
|
||||
} else {
|
||||
let since = *self.stable_since.get_or_insert(now);
|
||||
if advanced
|
||||
if allow_probe
|
||||
&& advanced
|
||||
&& now.saturating_duration_since(since) >= Duration::from_secs(8)
|
||||
&& now.saturating_duration_since(self.last_change) >= Duration::from_secs(8)
|
||||
&& self.limit < observation.max_concurrency
|
||||
@@ -325,6 +359,13 @@ impl FeedbackController {
|
||||
}
|
||||
|
||||
impl AdaptiveAdmission {
|
||||
/// Successful async operations using this physical owner. Native clones
|
||||
/// retain capacity but never count another completion.
|
||||
pub fn completed(&self) -> Option<u64> {
|
||||
let value = self.completed.load(Ordering::Acquire);
|
||||
(value != u64::MAX).then_some(value)
|
||||
}
|
||||
|
||||
/// Construct a bounded owner, with zero initial capacity allowed for unknown
|
||||
/// input. The immutable maximum reserves at most 65,535 lease slots once;
|
||||
/// acquiring or cloning a permit does not allocate a per-unit Arc.
|
||||
@@ -351,6 +392,7 @@ impl AdaptiveAdmission {
|
||||
maximum: maximum_u32,
|
||||
slots,
|
||||
next_chunk: AtomicUsize::new(0),
|
||||
completed: AtomicU64::new(0),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -466,6 +508,21 @@ impl AdaptiveAdmission {
|
||||
self.slots[slot / 64].free.fetch_or(1_u64 << (slot % 64), Ordering::Release);
|
||||
}
|
||||
|
||||
fn release_reference(&self, slot: usize) {
|
||||
self.release_reference_with(slot, || {});
|
||||
}
|
||||
|
||||
fn release_reference_with(&self, slot: usize, before_release: impl FnOnce()) {
|
||||
if self.slots[slot / 64].references[slot % 64].fetch_sub(1, Ordering::AcqRel) != 1 {
|
||||
return;
|
||||
}
|
||||
before_release();
|
||||
// Publish the free slot before exposing another admission capacity.
|
||||
self.release_slot(slot);
|
||||
self.state.fetch_sub(1, Ordering::AcqRel);
|
||||
self.changed.notify_one();
|
||||
}
|
||||
|
||||
/// Wait for capacity. The future owns no permit until it returns; dropping
|
||||
/// a pending future leaves capacity untouched. Wrap in the caller's existing
|
||||
/// cancellation/deadline select before acquiring any storage commit locks.
|
||||
@@ -493,13 +550,7 @@ impl AdaptiveAdmission {
|
||||
|
||||
impl Drop for AdaptivePermit {
|
||||
fn drop(&mut self) {
|
||||
if self.owner.slots[self.slot / 64].references[self.slot % 64].fetch_sub(1, Ordering::AcqRel) != 1 {
|
||||
return;
|
||||
}
|
||||
// Publish the free slot before exposing another admission capacity.
|
||||
self.owner.release_slot(self.slot);
|
||||
self.owner.state.fetch_sub(1, Ordering::AcqRel);
|
||||
self.owner.changed.notify_one();
|
||||
self.owner.release_reference(self.slot);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -861,6 +912,24 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn shared_probe_permission_blocks_growth_but_never_protective_cuts() {
|
||||
let gate = AdaptiveAdmission::new(NonZeroU16::new(8).expect("bound"), 0);
|
||||
let mut controller = FeedbackController::new();
|
||||
controller.update_with_probe(&gate, Some(healthy(0)), false);
|
||||
for completed in 1..=20 {
|
||||
tokio::time::advance(Duration::from_secs(1)).await;
|
||||
assert_eq!(controller.update_with_probe(&gate, Some(healthy(completed)), false).limit, 1);
|
||||
}
|
||||
tokio::time::advance(Duration::from_secs(1)).await;
|
||||
assert_eq!(controller.update_with_probe(&gate, Some(healthy(21)), true).reason, FeedbackReason::Probe);
|
||||
assert_eq!(gate.snapshot().limit, 2);
|
||||
let mut busy = healthy(22);
|
||||
busy.pressure = ResourcePressure::Busy;
|
||||
assert_eq!(controller.update_with_probe(&gate, Some(busy), false).limit, 1);
|
||||
assert_eq!(controller.update_with_probe(&gate, None, false).limit, 0);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn controller_grows_one_unit_per_cooldown_and_cuts_pressure_immediately() {
|
||||
let gate = AdaptiveAdmission::new(NonZeroU16::new(8).expect("bounded test owner"), 0);
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -121,6 +121,11 @@ when restoring `/rustfs/console` for a custom OEM build.
|
||||
- `RUSTFS_SCANNER_CYCLE_MAX_DURATION_SECS` (canonical)
|
||||
- `RUSTFS_SCANNER_CYCLE_MAX_OBJECTS` (canonical)
|
||||
- `RUSTFS_SCANNER_CYCLE_MAX_DIRECTORIES` (canonical)
|
||||
- `RUSTFS_SCANNER_BUCKET_QUANTUM_DURATION_SECS` (canonical)
|
||||
- `RUSTFS_SCANNER_BUCKET_QUANTUM_OBJECTS` (canonical)
|
||||
- `RUSTFS_SCANNER_BUCKET_QUANTUM_DIRECTORIES` (canonical)
|
||||
- `RUSTFS_SCANNER_BUCKET_QUANTUM_ENTRIES` (canonical)
|
||||
- `RUSTFS_SCANNER_BUCKET_QUANTUM_NAME_BYTES` (canonical)
|
||||
- `RUSTFS_SCANNER_MAX_CONCURRENT_SET_SCANS` (canonical)
|
||||
- `RUSTFS_SCANNER_MAX_CONCURRENT_DISK_SCANS` (canonical)
|
||||
- `RUSTFS_SCANNER_WALKERS_PER_DISK` (canonical)
|
||||
@@ -130,6 +135,12 @@ Scanner cycle budget controls:
|
||||
- When `RUSTFS_SCANNER_CYCLE_MAX_DURATION_SECS` is unset, the finite default is 1800 seconds (30 minutes), matching the scanner benchmark guidance.
|
||||
- An explicit `0` preserves the compatibility behavior of an unbounded runtime budget. Object and directory budgets likewise remain unbounded when explicitly set to `0`.
|
||||
|
||||
Scanner bucket quantum controls:
|
||||
|
||||
- `RUSTFS_SCANNER_BUCKET_QUANTUM_DURATION_SECS`, `RUSTFS_SCANNER_BUCKET_QUANTUM_OBJECTS`, `RUSTFS_SCANNER_BUCKET_QUANTUM_DIRECTORIES`, `RUSTFS_SCANNER_BUCKET_QUANTUM_ENTRIES`, and `RUSTFS_SCANNER_BUCKET_QUANTUM_NAME_BYTES` cap one bucket admission. Their persistent keys are `scanner.bucket_quantum_duration`, `scanner.bucket_quantum_objects`, `scanner.bucket_quantum_directories`, `scanner.bucket_quantum_entries`, and `scanner.bucket_quantum_name_bytes`; duration is in whole seconds and the remaining keys are work counts or bytes.
|
||||
- All five defaults are `0` for controlled rollout. A positive value enables that dimension; `0` disables that dimension, and all five zero values disable the bucket quantum. Parent cycle limits remain in force. Invalid unsigned integers and durations outside the platform timer range are rejected; environment overrides take precedence but cannot hide invalid persisted values. Admin scanner status reports each effective value and source under `runtime_config.bucket_quantum_*`, using `bucket_quantum_duration_seconds` for duration.
|
||||
- Disabled defaults are an initial rollout state, not completion of usage-convergence acceptance. Production defaults require matched-topology A/B measurements and the remote progress/fairness checks in [Scanner runtime controls](../../docs/operations/scanner-runtime-controls.md#bucket-quantum-rollout-and-acceptance). These controls do not certify partial usage or acknowledge pending dirty work.
|
||||
|
||||
Scanner recovery retry controls:
|
||||
|
||||
- `RUSTFS_SCANNER_RECOVERY_BACKOFF_CAP_SECS` caps the wait between automatic startup recovery retries. The persistent admin key is `scanner.recovery_backoff_cap`; the environment takes precedence over the persistent value, and the unset default remains 600 seconds (10 minutes). Values must be positive integer seconds within the platform timer range; `0`, malformed values, and overflow are rejected. A valid environment override does not hide an invalid persistent value from `admin config set` validation.
|
||||
|
||||
@@ -30,6 +30,16 @@ pub const COMMENT_KEY: &str = "comment";
|
||||
pub const ENV_AUDIT_ENABLE: &str = "RUSTFS_AUDIT_ENABLE";
|
||||
/// Global switch for enabling the notify module.
|
||||
pub const ENV_NOTIFY_ENABLE: &str = "RUSTFS_NOTIFY_ENABLE";
|
||||
/// Maximum audit entries per second for authorization denials of authenticated callers.
|
||||
/// `0` disables denial auditing for that class of caller.
|
||||
pub const ENV_AUDIT_DENIAL_RATE: &str = "RUSTFS_AUDIT_DENIAL_RATE";
|
||||
/// Maximum audit entries per second for authorization denials of anonymous callers.
|
||||
/// `0` disables denial auditing for that class of caller.
|
||||
pub const ENV_AUDIT_DENIAL_ANONYMOUS_RATE: &str = "RUSTFS_AUDIT_DENIAL_ANONYMOUS_RATE";
|
||||
/// Default denial audit rate for authenticated callers (entries per second).
|
||||
pub const DEFAULT_AUDIT_DENIAL_RATE: u64 = 50;
|
||||
/// Default denial audit rate for anonymous callers (entries per second).
|
||||
pub const DEFAULT_AUDIT_DENIAL_ANONYMOUS_RATE: u64 = 5;
|
||||
/// Default global audit switch (disabled by default).
|
||||
pub const DEFAULT_AUDIT_ENABLE: bool = false;
|
||||
/// Default global notify switch (disabled by default).
|
||||
@@ -328,5 +338,7 @@ mod tests {
|
||||
fn test_global_audit_notify_switch_constants() {
|
||||
assert_eq!(ENV_AUDIT_ENABLE, "RUSTFS_AUDIT_ENABLE");
|
||||
assert_eq!(ENV_NOTIFY_ENABLE, "RUSTFS_NOTIFY_ENABLE");
|
||||
assert_eq!(ENV_AUDIT_DENIAL_RATE, "RUSTFS_AUDIT_DENIAL_RATE");
|
||||
assert_eq!(ENV_AUDIT_DENIAL_ANONYMOUS_RATE, "RUSTFS_AUDIT_DENIAL_ANONYMOUS_RATE");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -184,15 +184,27 @@ const _: () = assert!(!DEFAULT_INTERNODE_RPC_REPLAY_SCOPE_STRICT);
|
||||
/// Capacity (distinct nonces) of the process-local internode RPC replay cache that enforces
|
||||
/// one-time consumption of authenticated RPC signatures.
|
||||
///
|
||||
/// The cache retains each nonce for the ~10-minute signature freshness envelope. Once peers use
|
||||
/// replay-scoped v3 authentication, every authenticated RPC consumes one entry, so the steady
|
||||
/// state holds roughly `authenticated RPC RPS x 601s` entries. This default is the minimum floor:
|
||||
/// explicit operator values and resource-aware auto sizing both clamp upward to at least this
|
||||
/// value. Overflow fails closed — legitimate signed traffic is the only thing that can fill the
|
||||
/// cache (replays are rejected before insertion, and an attacker cannot mint valid nonces without
|
||||
/// the shared secret) — and increments
|
||||
/// `rustfs_system_network_internode_replay_cache_overflow_total`, so a sustained non-zero overflow
|
||||
/// counter means this capacity is undersized for the node's peak authenticated RPC rate.
|
||||
/// Nonces remain protected until their signed timestamp is outside the observed wall-clock
|
||||
/// validity window, including after clock rollback. Current-time signatures live about five
|
||||
/// minutes; the maximum accepted forward clock skew can extend that to ten minutes. A body-bound
|
||||
/// v3 request consumes both its v3 and compatibility-v2 nonces, so size for receiver nonce intake,
|
||||
/// not S3 request rate or CPU count.
|
||||
///
|
||||
/// Automatic admission budgets 13% of effective process memory for signed-time
|
||||
/// hash tables, allocator overhead and simultaneous old/new tables during growth.
|
||||
/// The reported count is an absolute UUID payload ceiling, not guaranteed throughput:
|
||||
/// actual byte admission can reject earlier depending on timestamp distribution.
|
||||
/// Tables grow with demand and are released together when their signed time expires.
|
||||
/// CPU count, a historical minimum, and a fixed 32M ceiling do not constrain this budget.
|
||||
/// If memory detection fails, the historical default remains a bounded count fallback.
|
||||
///
|
||||
/// Explicit values remain count ceilings with the historical minimum of 1,048,576
|
||||
/// for configuration compatibility, and bypass automatic byte admission. That
|
||||
/// compatibility floor does not apply to the automatically detected memory budget.
|
||||
/// Zero never disables replay protection.
|
||||
/// Overflow rejects the request without evicting a still-valid identity, increments
|
||||
/// `rustfs_system_network_internode_replay_cache_overflow_total`, and returns resource
|
||||
/// exhaustion rather than invalid credentials. Measure headroom before overriding.
|
||||
pub const ENV_INTERNODE_RPC_REPLAY_CACHE_CAPACITY: &str = "RUSTFS_INTERNODE_RPC_REPLAY_CACHE_CAPACITY";
|
||||
pub const DEFAULT_INTERNODE_RPC_REPLAY_CACHE_CAPACITY: usize = 1_048_576;
|
||||
|
||||
|
||||
@@ -32,8 +32,9 @@ pub const ENV_OBJECT_MEDIUM_CONCURRENCY_THRESHOLD: &str = "RUSTFS_OBJECT_MEDIUM_
|
||||
|
||||
/// Environment variable name for maximum concurrent disk reads for object operations.
|
||||
/// - Purpose: Limit the number of concurrent disk read operations for object reads to prevent I/O saturation.
|
||||
/// - Unit: request count (usize).
|
||||
/// - Semantics: Throttling disk reads helps maintain overall system responsiveness under load.
|
||||
/// - Unit: request count (usize) for GET; estimated drive fanout for native LIST.
|
||||
/// - Semantics: GET bodies and native LIST walks use separate per-instance budgets of this size.
|
||||
/// A LIST consumes up to one budget, including its disk-producer cleanup. `0` disables both caps.
|
||||
/// - Example: `export RUSTFS_OBJECT_MAX_CONCURRENT_DISK_READS=16`
|
||||
/// - Note: This setting may interact with OS-level I/O scheduling and should be tuned based on hardware capabilities.
|
||||
pub const ENV_OBJECT_MAX_CONCURRENT_DISK_READS: &str = "RUSTFS_OBJECT_MAX_CONCURRENT_DISK_READS";
|
||||
|
||||
@@ -30,10 +30,10 @@ pub const ENV_TRUSTED_PROXY_VALIDATION_MODE: &str = "RUSTFS_TRUSTED_PROXY_VALIDA
|
||||
/// Default validation mode is "hop_by_hop".
|
||||
pub const DEFAULT_TRUSTED_PROXY_VALIDATION_MODE: &str = "hop_by_hop";
|
||||
|
||||
/// Environment variable to enable RFC 7239 "Forwarded" header support.
|
||||
/// Select RFC 7239 "Forwarded" instead of "X-Forwarded-For" in legacy mode.
|
||||
pub const ENV_TRUSTED_PROXY_ENABLE_RFC7239: &str = "RUSTFS_TRUSTED_PROXY_ENABLE_RFC7239";
|
||||
/// RFC 7239 support is enabled by default.
|
||||
pub const DEFAULT_TRUSTED_PROXY_ENABLE_RFC7239: bool = true;
|
||||
/// Legacy mode uses X-Forwarded-For unless RFC 7239 is explicitly selected.
|
||||
pub const DEFAULT_TRUSTED_PROXY_ENABLE_RFC7239: bool = false;
|
||||
|
||||
/// Environment variable for the maximum allowed proxy hops.
|
||||
pub const ENV_TRUSTED_PROXY_MAX_HOPS: &str = "RUSTFS_TRUSTED_PROXY_MAX_HOPS";
|
||||
|
||||
@@ -44,6 +44,21 @@ pub const SCANNER_CYCLE_MAX_OBJECTS: &str = "cycle_max_objects";
|
||||
/// Scanner config key capping directories entered by one cycle.
|
||||
pub const SCANNER_CYCLE_MAX_DIRECTORIES: &str = "cycle_max_directories";
|
||||
|
||||
/// Per-bucket admission limit in seconds; zero disables this dimension.
|
||||
pub const SCANNER_BUCKET_QUANTUM_DURATION: &str = "bucket_quantum_duration";
|
||||
|
||||
/// Per-bucket admission limit in objects; zero disables this dimension.
|
||||
pub const SCANNER_BUCKET_QUANTUM_OBJECTS: &str = "bucket_quantum_objects";
|
||||
|
||||
/// Per-bucket admission limit in directories; zero disables this dimension.
|
||||
pub const SCANNER_BUCKET_QUANTUM_DIRECTORIES: &str = "bucket_quantum_directories";
|
||||
|
||||
/// Per-bucket admission limit in raw entries; zero disables this dimension.
|
||||
pub const SCANNER_BUCKET_QUANTUM_ENTRIES: &str = "bucket_quantum_entries";
|
||||
|
||||
/// Per-bucket admission limit in raw entry-name bytes; zero disables this dimension.
|
||||
pub const SCANNER_BUCKET_QUANTUM_NAME_BYTES: &str = "bucket_quantum_name_bytes";
|
||||
|
||||
/// Scanner config key setting the periodic bitrot scan cycle in seconds.
|
||||
pub const SCANNER_BITROT_CYCLE: &str = "bitrot_cycle";
|
||||
|
||||
@@ -93,6 +108,11 @@ pub const SCANNER_KEYS: &[&str] = &[
|
||||
SCANNER_CYCLE_MAX_DURATION,
|
||||
SCANNER_CYCLE_MAX_OBJECTS,
|
||||
SCANNER_CYCLE_MAX_DIRECTORIES,
|
||||
SCANNER_BUCKET_QUANTUM_DURATION,
|
||||
SCANNER_BUCKET_QUANTUM_OBJECTS,
|
||||
SCANNER_BUCKET_QUANTUM_DIRECTORIES,
|
||||
SCANNER_BUCKET_QUANTUM_ENTRIES,
|
||||
SCANNER_BUCKET_QUANTUM_NAME_BYTES,
|
||||
SCANNER_BITROT_CYCLE,
|
||||
SCANNER_IDLE_MODE,
|
||||
SCANNER_CACHE_SAVE_TIMEOUT,
|
||||
@@ -141,6 +161,36 @@ pub const ENV_SCANNER_CYCLE_MAX_OBJECTS: &str = "RUSTFS_SCANNER_CYCLE_MAX_OBJECT
|
||||
/// - Example: `export RUSTFS_SCANNER_CYCLE_MAX_DIRECTORIES=100000`
|
||||
pub const ENV_SCANNER_CYCLE_MAX_DIRECTORIES: &str = "RUSTFS_SCANNER_CYCLE_MAX_DIRECTORIES";
|
||||
|
||||
/// Per-bucket admission limit in seconds; zero disables this dimension.
|
||||
pub const ENV_SCANNER_BUCKET_QUANTUM_DURATION_SECS: &str = "RUSTFS_SCANNER_BUCKET_QUANTUM_DURATION_SECS";
|
||||
|
||||
/// Disabled until controlled-fleet acceptance selects production limits.
|
||||
pub const DEFAULT_SCANNER_BUCKET_QUANTUM_DURATION_SECS: u64 = 0;
|
||||
|
||||
/// Per-bucket admission limit in objects; zero disables this dimension.
|
||||
pub const ENV_SCANNER_BUCKET_QUANTUM_OBJECTS: &str = "RUSTFS_SCANNER_BUCKET_QUANTUM_OBJECTS";
|
||||
|
||||
/// Disabled until controlled-fleet acceptance selects production limits.
|
||||
pub const DEFAULT_SCANNER_BUCKET_QUANTUM_OBJECTS: u64 = 0;
|
||||
|
||||
/// Per-bucket admission limit in directories; zero disables this dimension.
|
||||
pub const ENV_SCANNER_BUCKET_QUANTUM_DIRECTORIES: &str = "RUSTFS_SCANNER_BUCKET_QUANTUM_DIRECTORIES";
|
||||
|
||||
/// Disabled until controlled-fleet acceptance selects production limits.
|
||||
pub const DEFAULT_SCANNER_BUCKET_QUANTUM_DIRECTORIES: u64 = 0;
|
||||
|
||||
/// Per-bucket admission limit in raw entries; zero disables this dimension.
|
||||
pub const ENV_SCANNER_BUCKET_QUANTUM_ENTRIES: &str = "RUSTFS_SCANNER_BUCKET_QUANTUM_ENTRIES";
|
||||
|
||||
/// Disabled until controlled-fleet acceptance selects production limits.
|
||||
pub const DEFAULT_SCANNER_BUCKET_QUANTUM_ENTRIES: u64 = 0;
|
||||
|
||||
/// Per-bucket admission limit in raw entry-name bytes; zero disables this dimension.
|
||||
pub const ENV_SCANNER_BUCKET_QUANTUM_NAME_BYTES: &str = "RUSTFS_SCANNER_BUCKET_QUANTUM_NAME_BYTES";
|
||||
|
||||
/// Disabled until controlled-fleet acceptance selects production limits.
|
||||
pub const DEFAULT_SCANNER_BUCKET_QUANTUM_NAME_BYTES: u64 = 0;
|
||||
|
||||
/// Environment variable that selects the scanner speed preset.
|
||||
/// Valid values: `fastest`, `fast`, `default`, `slow`, `slowest`.
|
||||
/// Controls the sleep factor, maximum sleep duration, and cycle interval.
|
||||
|
||||
@@ -146,3 +146,9 @@ pub const ENV_TARGET_STORE_COMPRESS: &str = "RUSTFS_TARGET_STORE_COMPRESS";
|
||||
|
||||
/// Queue-store compression is enabled by default to reduce disk footprint.
|
||||
pub const DEFAULT_TARGET_STORE_COMPRESS: bool = true;
|
||||
|
||||
/// Maximum on-disk bytes in each target queue, including failed entries and pending writes.
|
||||
pub const ENV_TARGET_STORE_MAX_BYTES: &str = "RUSTFS_TARGET_STORE_MAX_BYTES";
|
||||
|
||||
/// Queue writes fail at one GiB unless the operator sets another positive byte budget.
|
||||
pub const DEFAULT_TARGET_STORE_MAX_BYTES: u64 = 1024 * 1024 * 1024;
|
||||
|
||||
@@ -175,12 +175,17 @@ pub const ENV_HTTP_SOCKET_RECV_BUFFER_BYTES: &str = "RUSTFS_HTTP_SOCKET_RECV_BUF
|
||||
pub const DEFAULT_HTTP_SOCKET_RECV_BUFFER_BYTES: usize = 0;
|
||||
|
||||
/// Environment variable for the S3 request-body inter-chunk read timeout
|
||||
/// (seconds). Default: 300. Set to 0 to disable.
|
||||
/// (seconds). Default: 300. Set to 0 to disable the inactivity timeout.
|
||||
///
|
||||
/// This bounds how long a `PutObject`/upload may wait for the *next* body chunk
|
||||
/// while more bytes are still expected. It resets on every chunk, so it does not
|
||||
/// penalize slow-but-progressing uploads — it only fires when a peer sends a
|
||||
/// partial body and then goes silent *without* closing the connection (no EOF).
|
||||
/// Independently of this knob, `PutObject` and `UploadPart` require at least
|
||||
/// 64 KiB of body progress per 60 seconds of server wait (the minimum-progress
|
||||
/// floor, logged as `upload_body_progress_stalled`); it bounds trickle uploads
|
||||
/// that would otherwise pin upload admission indefinitely, and it stays in
|
||||
/// force when this timeout is set to 0.
|
||||
/// A reverse proxy or CDN that forwards a truncated body this way would
|
||||
/// otherwise make RustFS wait forever for bytes that never arrive; with this
|
||||
/// timeout the stalled read is aborted and logged with the received/expected
|
||||
|
||||
@@ -39,5 +39,8 @@ serde = { workspace = true, features = ["derive"] }
|
||||
rmp-serde = { workspace = true }
|
||||
rustfs-filemeta = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
serde_json.workspace = true
|
||||
|
||||
[lib]
|
||||
doctest = false
|
||||
|
||||
@@ -24,6 +24,14 @@ use std::{
|
||||
time::{Duration, SystemTime},
|
||||
};
|
||||
|
||||
/// Copy an existing cache-model value without including surrounding work.
|
||||
#[inline]
|
||||
pub fn clone_cache_value<T: ToOwned + ?Sized>(source: &T) -> T::Owned {
|
||||
#[cfg(feature = "hotpath-alloc")]
|
||||
let _copy_scope = hotpath::MeasurementGuardSync::new("cache_model.owned_copy", false, false);
|
||||
source.to_owned()
|
||||
}
|
||||
|
||||
/// Maximum amount a persisted `last_update` may lead the local wall clock before the
|
||||
/// persisted timestamp is treated as untrustworthy.
|
||||
///
|
||||
@@ -342,7 +350,7 @@ impl UnknownTierStats {
|
||||
self.diagnostics_dropped = checked_saturating_add(1, self.diagnostics_dropped, &mut self.counter_overflowed);
|
||||
continue;
|
||||
}
|
||||
self.diagnostics.push(diagnostic.clone());
|
||||
self.diagnostics.push(clone_cache_value(diagnostic));
|
||||
}
|
||||
}
|
||||
if !self.diagnostics.is_empty() {
|
||||
@@ -367,7 +375,7 @@ impl UnknownTierStats {
|
||||
if !self.fits_add(other) {
|
||||
return None;
|
||||
}
|
||||
let mut merged = self.clone();
|
||||
let mut merged = clone_cache_value(self);
|
||||
merged.merge(other);
|
||||
Some(merged)
|
||||
}
|
||||
@@ -407,7 +415,7 @@ impl AllTierStats {
|
||||
if st.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let entry = self.tiers.entry(tier.clone()).or_default();
|
||||
let entry = self.tiers.entry(clone_cache_value(tier)).or_default();
|
||||
*entry = entry.add(st);
|
||||
}
|
||||
}
|
||||
@@ -437,7 +445,7 @@ impl AllTierStats {
|
||||
.tiers
|
||||
.keys()
|
||||
.filter(|tier| tier.as_str() != "STANDARD" && tier.as_str() != "REDUCED_REDUNDANCY" && !known.contains(tier.as_str()))
|
||||
.cloned()
|
||||
.map(clone_cache_value)
|
||||
.collect();
|
||||
for tier in retired_tiers {
|
||||
if let Some(stats) = self.tiers.remove(&tier) {
|
||||
@@ -466,6 +474,12 @@ pub struct BucketTargetUsageInfo {
|
||||
#[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub struct BucketUsageInfo {
|
||||
pub size: u64,
|
||||
/// Provenance of this value. Missing and unknown states remain unproven.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub usage_value_state: Option<crate::BucketUsageValueState>,
|
||||
/// Certification of this exact value, independent of aggregation time.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub usage_certification: Option<crate::BucketUsageCertification>,
|
||||
/// Bucket identity certified by every set contributing to this snapshot.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub bucket_incarnation: Option<String>,
|
||||
@@ -572,6 +586,18 @@ pub struct DataUsageInfo {
|
||||
/// which last-known-good generation contributed to the view.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub usage_snapshot_set_states: Vec<DataUsageSnapshotSetState>,
|
||||
/// Complete mixed observations for admin display only. Legacy bucket maps
|
||||
/// and quota consumers continue to use the canonical values above.
|
||||
#[serde(
|
||||
default,
|
||||
skip_serializing_if = "HashMap::is_empty",
|
||||
deserialize_with = "crate::usage_observation::deserialize_bucket_observations"
|
||||
)]
|
||||
pub usage_snapshot_bucket_observations: HashMap<String, crate::DataUsageBucketObservation>,
|
||||
/// Response-only marker: a projected admin view must never be persisted or
|
||||
/// admitted as a quota/cache baseline, including after serialization.
|
||||
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
|
||||
pub usage_snapshot_admin_projection: bool,
|
||||
/// Buckets whose dirty-usage generation moved while this complete
|
||||
/// snapshot was being walked. Their `buckets_usage` values are the walk
|
||||
/// result, which the cycle could not certify as current; the next cycle
|
||||
@@ -667,6 +693,10 @@ impl DataUsageInfo {
|
||||
/// cycle. The timestamp fallback preserves ordering for legacy snapshots that
|
||||
/// predate those fields.
|
||||
pub fn data_usage_snapshot_is_newer(candidate: &DataUsageInfo, baseline: &DataUsageInfo) -> bool {
|
||||
data_usage_identity_is_newer(candidate.snapshot_identity(), baseline.snapshot_identity())
|
||||
}
|
||||
|
||||
fn data_usage_identity_is_newer(candidate: DataUsageSnapshotIdentity, baseline: DataUsageSnapshotIdentity) -> bool {
|
||||
match (
|
||||
candidate.scanner_epoch.zip(candidate.scanner_cycle),
|
||||
baseline.scanner_epoch.zip(baseline.scanner_cycle),
|
||||
@@ -689,10 +719,19 @@ pub fn data_usage_snapshot_is_newer(candidate: &DataUsageInfo, baseline: &DataUs
|
||||
/// already advance the authoritative timestamp when deleting a bucket, so a
|
||||
/// rollback delete/recreate fences the previous bucket incarnation too.
|
||||
pub fn observed_data_usage_is_newer(observed: &DataUsageInfo, authoritative: &DataUsageInfo) -> bool {
|
||||
observed.usage_snapshot_converged == Some(false)
|
||||
observed_data_usage_is_newer_than_identity(observed, authoritative.snapshot_identity())
|
||||
}
|
||||
|
||||
/// Return whether a nonconverged observation may supersede `authoritative`.
|
||||
///
|
||||
/// This applies the structural, exact-baseline, and generation checks of
|
||||
/// [`observed_data_usage_is_newer`] without retaining the baseline's bucket maps.
|
||||
pub fn observed_data_usage_is_newer_than_identity(observed: &DataUsageInfo, authoritative: DataUsageSnapshotIdentity) -> bool {
|
||||
!observed.usage_snapshot_admin_projection
|
||||
&& observed.usage_snapshot_converged == Some(false)
|
||||
&& (observed.is_complete_bucket_usage_snapshot() || observed.is_valid_partial_snapshot())
|
||||
&& observed.usage_snapshot_authoritative_baseline.as_ref() == Some(&authoritative.snapshot_identity())
|
||||
&& data_usage_snapshot_is_newer(observed, authoritative)
|
||||
&& observed.usage_snapshot_authoritative_baseline == Some(authoritative)
|
||||
&& data_usage_identity_is_newer(observed.snapshot_identity(), authoritative)
|
||||
}
|
||||
|
||||
/// Metadata describing the status of a disk-level data usage snapshot.
|
||||
@@ -1237,10 +1276,10 @@ impl DataUsageEntry {
|
||||
}
|
||||
|
||||
pub fn add_child(&mut self, hash: &DataUsageHash) {
|
||||
if self.children.contains(&hash.key()) {
|
||||
if self.children.contains(&clone_cache_value(&hash.0)) {
|
||||
return;
|
||||
}
|
||||
self.children.insert(hash.key());
|
||||
self.children.insert(clone_cache_value(&hash.0));
|
||||
}
|
||||
|
||||
pub fn merge(&mut self, other: &DataUsageEntry) {
|
||||
@@ -1257,7 +1296,7 @@ impl DataUsageEntry {
|
||||
s_rep.replica_size += o_rep.replica_size;
|
||||
s_rep.replica_count += o_rep.replica_count;
|
||||
for (arn, stat) in o_rep.targets.iter() {
|
||||
let st = s_rep.targets.entry(arn.clone()).or_default();
|
||||
let st = s_rep.targets.entry(clone_cache_value(arn)).or_default();
|
||||
st.pending_size += stat.pending_size;
|
||||
st.replicated_size += stat.replicated_size;
|
||||
st.failed_size += stat.failed_size;
|
||||
@@ -1451,7 +1490,7 @@ pub fn prefix_usage_in_cache(
|
||||
} else {
|
||||
format!("{bucket}/{prefix}")
|
||||
};
|
||||
let entry = cache.get(&hash_path(&root).key())?.clone();
|
||||
let entry = clone_cache_value(cache.get(&clone_cache_value(&hash_path(&root).0))?);
|
||||
|
||||
let usage = PrefixUsageSummary::from_entry(&flatten_entry(cache, &entry, 0)?);
|
||||
|
||||
@@ -1594,38 +1633,38 @@ pub struct DataUsageCache {
|
||||
impl DataUsageCache {
|
||||
pub fn replace(&mut self, path: &str, parent: &str, e: DataUsageEntry) {
|
||||
let hash = hash_path(path);
|
||||
self.cache.insert(hash.key(), e);
|
||||
self.cache.insert(clone_cache_value(&hash.0), e);
|
||||
if !parent.is_empty() {
|
||||
let phash = hash_path(parent);
|
||||
let p = {
|
||||
let p = self.cache.entry(phash.key()).or_default();
|
||||
let p = self.cache.entry(clone_cache_value(&phash.0)).or_default();
|
||||
p.add_child(&hash);
|
||||
p.clone()
|
||||
clone_cache_value(p)
|
||||
};
|
||||
self.cache.insert(phash.key(), p);
|
||||
self.cache.insert(clone_cache_value(&phash.0), p);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn replace_hashed(&mut self, hash: &DataUsageHash, parent: &Option<DataUsageHash>, e: &DataUsageEntry) {
|
||||
self.cache.insert(hash.key(), e.clone());
|
||||
self.cache.insert(clone_cache_value(&hash.0), clone_cache_value(e));
|
||||
if let Some(parent) = parent {
|
||||
self.cache.entry(parent.key()).or_default().add_child(hash);
|
||||
self.cache.entry(clone_cache_value(&parent.0)).or_default().add_child(hash);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn find(&self, path: &str) -> Option<DataUsageEntry> {
|
||||
self.cache.get(&hash_path(path).key()).cloned()
|
||||
self.cache.get(&clone_cache_value(&hash_path(path).0)).map(clone_cache_value)
|
||||
}
|
||||
|
||||
pub fn find_children_copy(&mut self, h: DataUsageHash) -> DataUsageHashMap {
|
||||
self.cache.entry(h.string()).or_default().children.clone()
|
||||
clone_cache_value(&self.cache.entry(clone_cache_value(&h.0)).or_default().children)
|
||||
}
|
||||
|
||||
pub fn flatten(&self, root: &DataUsageEntry) -> DataUsageEntry {
|
||||
let mut root = root.clone();
|
||||
for id in root.children.clone().iter() {
|
||||
let mut root = clone_cache_value(root);
|
||||
for id in clone_cache_value(&root.children).iter() {
|
||||
if let Some(e) = self.cache.get(id) {
|
||||
let mut e = e.clone();
|
||||
let mut e = clone_cache_value(e);
|
||||
if !e.children.is_empty() {
|
||||
e = self.flatten(&e);
|
||||
}
|
||||
@@ -1637,16 +1676,16 @@ impl DataUsageCache {
|
||||
}
|
||||
|
||||
pub fn copy_with_children(&mut self, src: &DataUsageCache, hash: &DataUsageHash, parent: &Option<DataUsageHash>) {
|
||||
if let Some(e) = src.cache.get(&hash.string()) {
|
||||
self.cache.insert(hash.key(), e.clone());
|
||||
if let Some(e) = src.cache.get(&clone_cache_value(&hash.0)) {
|
||||
self.cache.insert(clone_cache_value(&hash.0), clone_cache_value(e));
|
||||
for ch in e.children.iter() {
|
||||
if *ch == hash.key() {
|
||||
if *ch == clone_cache_value(&hash.0) {
|
||||
return;
|
||||
}
|
||||
self.copy_with_children(src, &DataUsageHash(ch.to_string()), &Some(hash.clone()));
|
||||
self.copy_with_children(src, &DataUsageHash(clone_cache_value(ch)), &Some(clone_cache_value(hash)));
|
||||
}
|
||||
if let Some(parent) = parent {
|
||||
let p = self.cache.entry(parent.key()).or_default();
|
||||
let p = self.cache.entry(clone_cache_value(&parent.0)).or_default();
|
||||
p.add_child(hash);
|
||||
}
|
||||
}
|
||||
@@ -1654,14 +1693,14 @@ impl DataUsageCache {
|
||||
|
||||
pub fn delete_recursive(&mut self, hash: &DataUsageHash) {
|
||||
let mut need_remove = Vec::new();
|
||||
if let Some(v) = self.cache.get(&hash.string()) {
|
||||
if let Some(v) = self.cache.get(&clone_cache_value(&hash.0)) {
|
||||
for child in v.children.iter() {
|
||||
need_remove.push(child.clone());
|
||||
need_remove.push(clone_cache_value(child));
|
||||
}
|
||||
}
|
||||
self.cache.remove(&hash.string());
|
||||
self.cache.remove(&clone_cache_value(&hash.0));
|
||||
need_remove.iter().for_each(|child| {
|
||||
self.delete_recursive(&DataUsageHash(child.to_string()));
|
||||
self.delete_recursive(&DataUsageHash(clone_cache_value(child)));
|
||||
});
|
||||
}
|
||||
|
||||
@@ -1682,7 +1721,7 @@ impl DataUsageCache {
|
||||
}
|
||||
|
||||
pub fn search_parent(&self, hash: &DataUsageHash) -> Option<DataUsageHash> {
|
||||
let want = hash.key();
|
||||
let want = clone_cache_value(&hash.0);
|
||||
if let Some(last_index) = want.rfind('/')
|
||||
&& let Some(v) = self.find(&want[0..last_index])
|
||||
&& v.children.contains(&want)
|
||||
@@ -1693,7 +1732,7 @@ impl DataUsageCache {
|
||||
|
||||
for (k, v) in self.cache.iter() {
|
||||
if v.children.contains(&want) {
|
||||
let found = DataUsageHash(k.clone());
|
||||
let found = DataUsageHash(clone_cache_value(k));
|
||||
return Some(found);
|
||||
}
|
||||
}
|
||||
@@ -1701,7 +1740,7 @@ impl DataUsageCache {
|
||||
}
|
||||
|
||||
pub fn is_compacted(&self, hash: &DataUsageHash) -> bool {
|
||||
match self.cache.get(&hash.key()) {
|
||||
match self.cache.get(&clone_cache_value(&hash.0)) {
|
||||
Some(due) => due.compacted,
|
||||
None => false,
|
||||
}
|
||||
@@ -1726,7 +1765,7 @@ impl DataUsageCache {
|
||||
if self.cache.len() < limit {
|
||||
return;
|
||||
}
|
||||
let top = hash_path(&self.info.name).key();
|
||||
let top = clone_cache_value(&hash_path(&self.info.name).0);
|
||||
let top_e = match self.find(&top) {
|
||||
Some(e) => e,
|
||||
None => return,
|
||||
@@ -1744,16 +1783,11 @@ impl DataUsageCache {
|
||||
let mut found = HashSet::new();
|
||||
found.insert(top);
|
||||
mark(self, &top_e, &mut found);
|
||||
self.cache.retain(|k, _| {
|
||||
if !found.contains(k) {
|
||||
return false;
|
||||
}
|
||||
true
|
||||
});
|
||||
self.cache.retain(|k, _| found.contains(k));
|
||||
}
|
||||
|
||||
pub fn reduce_children_of(&mut self, path: &DataUsageHash, limit: usize, compact_self: bool) {
|
||||
let e = match self.cache.get(&path.key()) {
|
||||
let e = match self.cache.get(&clone_cache_value(&path.0)) {
|
||||
Some(e) => e,
|
||||
None => return,
|
||||
};
|
||||
@@ -1763,13 +1797,13 @@ impl DataUsageCache {
|
||||
}
|
||||
|
||||
if e.children.len() > limit && compact_self {
|
||||
let mut flat = self.size_recursive(&path.key()).unwrap_or_default();
|
||||
let mut flat = self.size_recursive(&clone_cache_value(&path.0)).unwrap_or_default();
|
||||
flat.compacted = true;
|
||||
self.delete_recursive(path);
|
||||
self.replace_hashed(path, &None, &flat);
|
||||
return;
|
||||
}
|
||||
let total = self.total_children_rec(&path.key());
|
||||
let total = self.total_children_rec(&clone_cache_value(&path.0));
|
||||
if total < limit {
|
||||
return;
|
||||
}
|
||||
@@ -1782,12 +1816,12 @@ impl DataUsageCache {
|
||||
let mut candidate_index = 0;
|
||||
while remove > 0 && candidate_index < candidates.len() {
|
||||
let e = &candidates[candidate_index];
|
||||
let candidate = e.path.clone();
|
||||
let candidate = clone_cache_value(&e.path);
|
||||
if candidate == *path && !compact_self {
|
||||
break;
|
||||
}
|
||||
let removing = self.total_children_rec(&candidate.key());
|
||||
let mut flat = match self.size_recursive(&candidate.key()) {
|
||||
let removing = self.total_children_rec(&clone_cache_value(&candidate.0));
|
||||
let mut flat = match self.size_recursive(&clone_cache_value(&candidate.0)) {
|
||||
Some(flat) => flat,
|
||||
None => {
|
||||
candidate_index += 1;
|
||||
@@ -1824,7 +1858,7 @@ impl DataUsageCache {
|
||||
if o.root().is_none() {
|
||||
return;
|
||||
}
|
||||
*self = o.clone();
|
||||
*self = clone_cache_value(o);
|
||||
return;
|
||||
};
|
||||
|
||||
@@ -1837,7 +1871,8 @@ impl DataUsageCache {
|
||||
}
|
||||
|
||||
existing_root.merge(&other_root);
|
||||
self.cache.insert(hash_path(&self.info.name).key(), existing_root);
|
||||
self.cache
|
||||
.insert(clone_cache_value(&hash_path(&self.info.name).0), existing_root);
|
||||
|
||||
let root_hash = self.root_hash();
|
||||
for key in other_root.children.iter() {
|
||||
@@ -1848,7 +1883,7 @@ impl DataUsageCache {
|
||||
if let Some(existing) = self.cache.get_mut(key) {
|
||||
existing.merge(&flat);
|
||||
} else {
|
||||
self.replace_hashed(&DataUsageHash(key.clone()), &Some(root_hash.clone()), &flat);
|
||||
self.replace_hashed(&DataUsageHash(clone_cache_value(key)), &Some(clone_cache_value(&root_hash)), &flat);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1892,7 +1927,7 @@ impl DataUsageCache {
|
||||
|
||||
for (arn, stat) in rs.targets.iter() {
|
||||
bui.replication_info.insert(
|
||||
arn.clone(),
|
||||
clone_cache_value(arn),
|
||||
BucketTargetUsageInfo {
|
||||
replication_pending_size: stat.pending_size,
|
||||
replicated_size: stat.replicated_size,
|
||||
@@ -1905,7 +1940,7 @@ impl DataUsageCache {
|
||||
);
|
||||
}
|
||||
}
|
||||
buckets_usage.insert(bucket_name.clone(), bui);
|
||||
buckets_usage.insert(clone_cache_value(bucket_name), bui);
|
||||
}
|
||||
|
||||
DataUsageInfo {
|
||||
@@ -1937,20 +1972,20 @@ struct Inner {
|
||||
}
|
||||
|
||||
fn add(data_usage_cache: &DataUsageCache, path: &DataUsageHash, candidates: &mut Vec<Inner>) -> usize {
|
||||
let e = match data_usage_cache.cache.get(&path.key()) {
|
||||
let e = match data_usage_cache.cache.get(&clone_cache_value(&path.0)) {
|
||||
Some(e) => e,
|
||||
None => return 0,
|
||||
};
|
||||
let mut objects = e.objects;
|
||||
for ch in e.children.iter() {
|
||||
objects += add(data_usage_cache, &DataUsageHash(ch.clone()), candidates);
|
||||
objects += add(data_usage_cache, &DataUsageHash(clone_cache_value(ch)), candidates);
|
||||
}
|
||||
// Collect internal nodes (with children) as compaction candidates.
|
||||
// Leaf nodes have no children to remove, so compacting them is a no-op.
|
||||
if !e.children.is_empty() {
|
||||
candidates.push(Inner {
|
||||
objects,
|
||||
path: path.clone(),
|
||||
path: clone_cache_value(path),
|
||||
});
|
||||
}
|
||||
objects
|
||||
@@ -1958,7 +1993,7 @@ fn add(data_usage_cache: &DataUsageCache, path: &DataUsageHash, candidates: &mut
|
||||
|
||||
fn mark(duc: &DataUsageCache, entry: &DataUsageEntry, found: &mut HashSet<String>) {
|
||||
for k in entry.children.iter() {
|
||||
found.insert(k.to_string());
|
||||
found.insert(clone_cache_value(k));
|
||||
if let Some(ch) = duc.cache.get(k) {
|
||||
mark(duc, ch, found);
|
||||
}
|
||||
@@ -2008,7 +2043,8 @@ impl DataUsageInfo {
|
||||
|
||||
/// Whether this snapshot authoritatively covers every reported bucket.
|
||||
pub fn is_complete_bucket_usage_snapshot(&self) -> bool {
|
||||
!self.usage_snapshot_bootstrap_pending
|
||||
!self.usage_snapshot_admin_projection
|
||||
&& !self.usage_snapshot_bootstrap_pending
|
||||
&& self.usage_snapshot_complete
|
||||
&& self.last_update.is_some()
|
||||
&& u64::try_from(self.buckets_usage.len()).ok() == Some(self.buckets_count)
|
||||
@@ -2018,7 +2054,8 @@ impl DataUsageInfo {
|
||||
/// admin display. Partial data is accepted only with unique set states,
|
||||
/// a plan digest for every state, and at least one usable generation.
|
||||
pub fn is_valid_partial_snapshot(&self) -> bool {
|
||||
if self.usage_snapshot_bootstrap_pending
|
||||
if self.usage_snapshot_admin_projection
|
||||
|| self.usage_snapshot_bootstrap_pending
|
||||
|| !self.usage_snapshot_partial
|
||||
|| self.usage_snapshot_converged != Some(false)
|
||||
|| self.last_update.is_none()
|
||||
@@ -2056,9 +2093,11 @@ impl DataUsageInfo {
|
||||
Ok(name) => name,
|
||||
Err(_) => return,
|
||||
};
|
||||
self.usage_snapshot_bucket_observations.remove(&bucket_name);
|
||||
|
||||
// Update bucket statistics
|
||||
if let Some(bucket_usage) = self.buckets_usage.get_mut(&bucket_name) {
|
||||
bucket_usage.mark_observed();
|
||||
bucket_usage.size += meta_object.size as u64;
|
||||
bucket_usage.objects_count += 1;
|
||||
bucket_usage.versions_count += 1; // Simplified: assume 1 version per object
|
||||
@@ -2111,6 +2150,7 @@ impl DataUsageInfo {
|
||||
Ok(name) => name,
|
||||
Err(_) => return,
|
||||
};
|
||||
self.usage_snapshot_bucket_observations.remove(&bucket_name);
|
||||
|
||||
// Calculate accurate statistics from all versions
|
||||
let mut total_size = 0u64;
|
||||
@@ -2139,6 +2179,7 @@ impl DataUsageInfo {
|
||||
|
||||
// Update bucket statistics
|
||||
if let Some(bucket_usage) = self.buckets_usage.get_mut(&bucket_name) {
|
||||
bucket_usage.mark_observed();
|
||||
bucket_usage.size += total_size;
|
||||
bucket_usage.objects_count += 1;
|
||||
bucket_usage.versions_count += versions_count;
|
||||
@@ -2319,6 +2360,9 @@ impl BucketUsageInfo {
|
||||
/// Add size summary to this bucket usage
|
||||
/// Merge another BucketUsageInfo into this one
|
||||
pub fn merge(&mut self, other: &BucketUsageInfo) {
|
||||
// Adding contributions produces a new value, not either input's
|
||||
// whole-bucket certificate. The scanner may certify the final sum.
|
||||
self.mark_observed();
|
||||
if self.bucket_incarnation != other.bucket_incarnation {
|
||||
self.bucket_incarnation = None;
|
||||
}
|
||||
@@ -2408,11 +2452,11 @@ impl SizeSummary {
|
||||
}
|
||||
|
||||
for entry in &other.size_reconciliation {
|
||||
self.record_size_reconciliation(entry.clone());
|
||||
self.record_size_reconciliation(clone_cache_value(entry));
|
||||
}
|
||||
self.size_reconciliation_truncated |= other.size_reconciliation_truncated;
|
||||
for scope in &other.reconciliation_scopes {
|
||||
self.record_reconciliation_scope(&scope.bucket, &scope.object);
|
||||
self.record_reconciliation_scope_with(&scope.bucket, &scope.object, || clone_cache_value(scope));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2437,6 +2481,13 @@ impl SizeSummary {
|
||||
/// Mark one object scope as refreshed. Duplicate scopes are suppressed so
|
||||
/// merging summaries remains bounded and deterministic.
|
||||
pub fn record_reconciliation_scope(&mut self, bucket: &str, object: &str) {
|
||||
self.record_reconciliation_scope_with(bucket, object, || SizeReconciliationScope {
|
||||
bucket: bucket.to_string(),
|
||||
object: object.to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
fn record_reconciliation_scope_with(&mut self, bucket: &str, object: &str, build: impl FnOnce() -> SizeReconciliationScope) {
|
||||
if !self
|
||||
.reconciliation_scopes
|
||||
.iter()
|
||||
@@ -2446,10 +2497,7 @@ impl SizeSummary {
|
||||
self.size_reconciliation_truncated = true;
|
||||
return;
|
||||
}
|
||||
self.reconciliation_scopes.push(SizeReconciliationScope {
|
||||
bucket: bucket.to_string(),
|
||||
object: object.to_string(),
|
||||
});
|
||||
self.reconciliation_scopes.push(build());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -3127,12 +3175,17 @@ mod tests {
|
||||
};
|
||||
|
||||
assert!(observed_data_usage_is_newer(&observed, &authoritative));
|
||||
assert!(observed_data_usage_is_newer_than_identity(&observed, authoritative.snapshot_identity()));
|
||||
|
||||
authoritative.last_update = Some(SystemTime::UNIX_EPOCH + Duration::from_secs(601));
|
||||
assert!(
|
||||
!observed_data_usage_is_newer(&observed, &authoritative),
|
||||
"an old-binary namespace mutation must fence the prior bucket incarnation regardless of clock skew"
|
||||
);
|
||||
assert!(
|
||||
!observed_data_usage_is_newer_than_identity(&observed, authoritative.snapshot_identity()),
|
||||
"a captured identity must preserve the exact namespace fence"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -3159,6 +3212,18 @@ mod tests {
|
||||
assert!(!observed_data_usage_is_newer(&candidate(2, 9, Some(false), true), &authoritative));
|
||||
assert!(!observed_data_usage_is_newer(&candidate(2, 11, Some(true), true), &authoritative));
|
||||
assert!(!observed_data_usage_is_newer(&candidate(2, 11, Some(false), false), &authoritative));
|
||||
for (observed, expected) in [
|
||||
(candidate(2, 11, Some(false), true), true),
|
||||
(candidate(2, 9, Some(false), true), false),
|
||||
(candidate(2, 11, Some(true), true), false),
|
||||
(candidate(2, 11, Some(false), false), false),
|
||||
] {
|
||||
assert_eq!(
|
||||
observed_data_usage_is_newer_than_identity(&observed, authoritative.snapshot_identity()),
|
||||
expected,
|
||||
"observation identity selection must preserve convergence, completeness, and generation gates"
|
||||
);
|
||||
}
|
||||
|
||||
let mut partial = candidate(2, 11, Some(false), false);
|
||||
partial.usage_snapshot_partial = true;
|
||||
@@ -3173,6 +3238,30 @@ mod tests {
|
||||
segment_invalidation_proof: None,
|
||||
}];
|
||||
assert!(observed_data_usage_is_newer(&partial, &authoritative));
|
||||
assert!(observed_data_usage_is_newer_than_identity(&partial, authoritative.snapshot_identity()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn observation_identity_selection_keeps_legacy_timestamp_ordering() {
|
||||
let baseline = DataUsageSnapshotIdentity {
|
||||
last_update: Some(SystemTime::UNIX_EPOCH + Duration::from_secs(10)),
|
||||
// An epoch without a cycle is not a coordinated position.
|
||||
scanner_epoch: Some(2),
|
||||
scanner_cycle: None,
|
||||
};
|
||||
let mut observed = DataUsageInfo {
|
||||
last_update: Some(SystemTime::UNIX_EPOCH + Duration::from_secs(11)),
|
||||
scanner_epoch: Some(1),
|
||||
usage_snapshot_complete: true,
|
||||
usage_snapshot_converged: Some(false),
|
||||
usage_snapshot_authoritative_baseline: Some(baseline),
|
||||
..Default::default()
|
||||
};
|
||||
assert!(observed_data_usage_is_newer_than_identity(&observed, baseline));
|
||||
observed.last_update = baseline.last_update;
|
||||
assert!(!observed_data_usage_is_newer_than_identity(&observed, baseline));
|
||||
observed.last_update = None;
|
||||
assert!(!observed_data_usage_is_newer_than_identity(&observed, baseline));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -3404,6 +3493,42 @@ mod tests {
|
||||
assert_eq!(summary1.versions, 15);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_reconciliation_scope_builds_only_after_admission() {
|
||||
let mut summary = SizeSummary::new();
|
||||
let builds = std::cell::Cell::new(0);
|
||||
let build = || {
|
||||
builds.set(builds.get() + 1);
|
||||
SizeReconciliationScope {
|
||||
bucket: "bucket".to_string(),
|
||||
object: "object".to_string(),
|
||||
}
|
||||
};
|
||||
|
||||
summary.record_reconciliation_scope_with("bucket", "object", build);
|
||||
assert_eq!(builds.get(), 1, "an admitted scope is copied once");
|
||||
assert_eq!(summary.reconciliation_scopes.len(), 1);
|
||||
assert_eq!(summary.reconciliation_scopes[0].bucket, "bucket");
|
||||
assert_eq!(summary.reconciliation_scopes[0].object, "object");
|
||||
|
||||
summary.record_reconciliation_scope_with("bucket", "object", build);
|
||||
assert_eq!(builds.get(), 1, "duplicate scopes must not be copied");
|
||||
assert_eq!(summary.reconciliation_scopes.len(), 1);
|
||||
assert!(!summary.size_reconciliation_truncated);
|
||||
|
||||
for index in 1..1024 {
|
||||
summary.record_reconciliation_scope("bucket", &index.to_string());
|
||||
}
|
||||
summary.record_reconciliation_scope_with("bucket", "object", build);
|
||||
assert_eq!(builds.get(), 1, "a duplicate at capacity must not be copied");
|
||||
assert!(!summary.size_reconciliation_truncated);
|
||||
|
||||
summary.record_reconciliation_scope_with("bucket", "overflow", build);
|
||||
assert_eq!(builds.get(), 1, "a rejected scope must not be copied");
|
||||
assert_eq!(summary.reconciliation_scopes.len(), 1024);
|
||||
assert!(summary.size_reconciliation_truncated);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_size_histogram_compat_rollup_sums_all_sub_buckets() {
|
||||
let mut hist = SizeHistogram::default();
|
||||
|
||||
@@ -13,5 +13,7 @@
|
||||
// limitations under the License.
|
||||
|
||||
pub mod data_usage;
|
||||
pub mod usage_observation;
|
||||
|
||||
pub use data_usage::*;
|
||||
pub use usage_observation::*;
|
||||
|
||||
@@ -0,0 +1,265 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
//! Diagnostic provenance and response-only projections of complete bucket observations.
|
||||
|
||||
use crate::{BucketUsageInfo, DataUsageInfo, DataUsageSnapshotIdentity};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::{collections::HashMap, time::SystemTime};
|
||||
|
||||
pub const DATA_USAGE_BUCKET_OBSERVATION_VERSION: u16 = 1;
|
||||
|
||||
/// Provenance of the returned bucket value, independent of quota authority.
|
||||
#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, Eq)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum BucketUsageValueState {
|
||||
Certified,
|
||||
Carried,
|
||||
Observed,
|
||||
#[serde(other)]
|
||||
Unknown,
|
||||
}
|
||||
|
||||
/// A certification of this exact bucket value, never its publication timestamp.
|
||||
#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, Eq)]
|
||||
pub struct BucketUsageCertification {
|
||||
pub epoch: u64,
|
||||
pub cycle: u64,
|
||||
pub certified_at: SystemTime,
|
||||
}
|
||||
|
||||
/// Immutable provenance of one complete traversal contribution.
|
||||
///
|
||||
/// The scanner validates its storage-owned sweep identity before constructing
|
||||
/// this public projection. Equal start/request plans are allowed when a fresh
|
||||
/// set contributes alongside another set's mixed traversal.
|
||||
#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
|
||||
pub struct DataUsageBucketObservationSource {
|
||||
pub pool_index: u64,
|
||||
pub set_index: u64,
|
||||
pub bucket_incarnation: String,
|
||||
pub scanner_epoch: u64,
|
||||
pub scanner_cycle: u64,
|
||||
pub scan_plan_digest: [u8; 32],
|
||||
pub started_plan: [u8; 32],
|
||||
pub requested_plan: [u8; 32],
|
||||
pub completed_at: SystemTime,
|
||||
}
|
||||
|
||||
/// An additive companion; legacy bucket maps never contain this mixed value.
|
||||
#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
|
||||
pub struct DataUsageBucketObservation {
|
||||
pub version: u16,
|
||||
pub usage: BucketUsageInfo,
|
||||
/// Exact enclosing candidate identity. Rewrapping preserves source times.
|
||||
pub snapshot_identity: DataUsageSnapshotIdentity,
|
||||
/// Oldest contribution time, so a newer set cannot hide an older value.
|
||||
pub completed_at: SystemTime,
|
||||
/// Strictly ordered, complete coverage of the enclosing live set inventory.
|
||||
pub sources: Vec<DataUsageBucketObservationSource>,
|
||||
}
|
||||
|
||||
/// Companion evolution must not make an otherwise usable canonical snapshot
|
||||
/// unreadable. Unsupported or malformed diagnostic entries never grant proof.
|
||||
pub(crate) fn deserialize_bucket_observations<'de, D>(
|
||||
deserializer: D,
|
||||
) -> Result<HashMap<String, DataUsageBucketObservation>, D::Error>
|
||||
where
|
||||
D: serde::Deserializer<'de>,
|
||||
{
|
||||
#[derive(Deserialize)]
|
||||
#[serde(untagged)]
|
||||
enum Entry {
|
||||
Observation(Box<DataUsageBucketObservation>),
|
||||
Ignored(serde::de::IgnoredAny),
|
||||
}
|
||||
Ok(HashMap::<String, Entry>::deserialize(deserializer)?
|
||||
.into_iter()
|
||||
.filter_map(|(bucket, entry)| match entry {
|
||||
Entry::Observation(observation) if observation.version == DATA_USAGE_BUCKET_OBSERVATION_VERSION => {
|
||||
Some((bucket, *observation))
|
||||
}
|
||||
Entry::Observation(_) | Entry::Ignored(_) => None,
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
impl BucketUsageInfo {
|
||||
/// Attach a certificate only after the caller proves complete bucket coverage.
|
||||
pub fn certify(&mut self, certification: BucketUsageCertification) {
|
||||
self.usage_value_state = Some(BucketUsageValueState::Certified);
|
||||
self.usage_certification = Some(certification);
|
||||
}
|
||||
|
||||
/// Preserve the certificate when carrying the exact prior bucket value.
|
||||
pub fn mark_carried(&mut self) {
|
||||
self.normalize_usage_provenance();
|
||||
self.usage_value_state = Some(BucketUsageValueState::Carried);
|
||||
}
|
||||
|
||||
/// The value changed without a complete bucket certificate.
|
||||
pub fn mark_observed(&mut self) {
|
||||
self.usage_value_state = Some(BucketUsageValueState::Observed);
|
||||
self.usage_certification = None;
|
||||
}
|
||||
|
||||
/// Future or legacy provenance cannot authorize a retained certificate.
|
||||
pub fn normalize_usage_provenance(&mut self) {
|
||||
if !matches!(
|
||||
self.usage_value_state,
|
||||
Some(BucketUsageValueState::Certified | BucketUsageValueState::Carried)
|
||||
) {
|
||||
self.usage_certification = None;
|
||||
}
|
||||
if self.usage_value_state == Some(BucketUsageValueState::Certified) && self.usage_certification.is_none() {
|
||||
self.usage_value_state = Some(BucketUsageValueState::Unknown);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl DataUsageBucketObservation {
|
||||
/// Verify the public provenance against the enclosing candidate and current
|
||||
/// storage-owned incarnation. This never grants publication or ACK authority.
|
||||
pub fn matches_snapshot(&self, snapshot: &DataUsageInfo, incarnation: &str) -> bool {
|
||||
let Some(root_generation) = snapshot.scanner_epoch.zip(snapshot.scanner_cycle) else {
|
||||
return false;
|
||||
};
|
||||
if self.version != DATA_USAGE_BUCKET_OBSERVATION_VERSION
|
||||
|| snapshot.usage_snapshot_admin_projection
|
||||
|| self.snapshot_identity != snapshot.snapshot_identity()
|
||||
|| incarnation.is_empty()
|
||||
|| self.usage.bucket_incarnation.as_deref() != Some(incarnation)
|
||||
|| self.sources.is_empty()
|
||||
|| self.usage.usage_certification.is_some()
|
||||
|| self.usage.usage_value_state != Some(BucketUsageValueState::Observed)
|
||||
|| self.sources.iter().map(|source| source.completed_at).min() != Some(self.completed_at)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
let mut previous = None;
|
||||
let mut plan = None;
|
||||
for state in &snapshot.usage_snapshot_set_states {
|
||||
let key = (state.pool_index, state.set_index);
|
||||
if previous.is_some_and(|previous| previous >= key)
|
||||
|| state.scan_plan_digest.is_none()
|
||||
|| plan.is_some_and(|plan| Some(plan) != state.scan_plan_digest)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
previous = Some(key);
|
||||
plan = state.scan_plan_digest;
|
||||
}
|
||||
let mut sources = self.sources.iter();
|
||||
for state in snapshot.usage_snapshot_set_states.iter().filter(|state| !state.tombstone) {
|
||||
let Some(source) = sources.next() else {
|
||||
return false;
|
||||
};
|
||||
let Some(set_generation) = state.scanner_epoch.zip(state.scanner_cycle) else {
|
||||
return false;
|
||||
};
|
||||
if (source.pool_index, source.set_index) != (state.pool_index, state.set_index)
|
||||
|| source.bucket_incarnation != incarnation
|
||||
|| Some(source.scan_plan_digest) != state.scan_plan_digest
|
||||
|| (source.scanner_epoch, source.scanner_cycle) > set_generation
|
||||
|| set_generation > root_generation
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
sources.next().is_none()
|
||||
}
|
||||
}
|
||||
|
||||
impl DataUsageInfo {
|
||||
/// Overlay validated companions on an owned admin response only.
|
||||
///
|
||||
/// `incarnations` must come from current storage metadata, not the companion.
|
||||
/// The projection is marked explicitly so persistence and quota boundaries
|
||||
/// can reject accidental reuse. Invalid companions leave canonical values
|
||||
/// unchanged. Counter overflow rejects the whole projection.
|
||||
pub fn project_bucket_observations_for_admin(&mut self, incarnations: &HashMap<String, String>) -> usize {
|
||||
if self.usage_snapshot_admin_projection
|
||||
|| (!self.is_complete_bucket_usage_snapshot() && !self.is_valid_partial_snapshot())
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
let observations =
|
||||
self.usage_snapshot_bucket_observations
|
||||
.iter()
|
||||
.filter(|(bucket, observation)| {
|
||||
if self.usage_snapshot_complete
|
||||
&& (!self.usage_snapshot_pending_buckets.iter().any(|pending| pending == *bucket)
|
||||
|| u64::try_from(self.usage_snapshot_pending_buckets.len()).ok()
|
||||
!= Some(self.usage_snapshot_pending_bucket_count))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
if self
|
||||
.buckets_usage
|
||||
.get(*bucket)
|
||||
.and_then(|usage| usage.usage_certification)
|
||||
.is_some_and(|certificate| {
|
||||
observation.sources.iter().all(|source| {
|
||||
(source.scanner_epoch, source.scanner_cycle) <= (certificate.epoch, certificate.cycle)
|
||||
})
|
||||
})
|
||||
{
|
||||
return false;
|
||||
}
|
||||
incarnations.get(*bucket).is_some_and(|incarnation| {
|
||||
self.buckets_usage
|
||||
.get(*bucket)
|
||||
.and_then(|usage| usage.bucket_incarnation.as_ref())
|
||||
.is_none_or(|canonical| canonical == incarnation)
|
||||
&& observation.matches_snapshot(self, incarnation)
|
||||
})
|
||||
})
|
||||
.collect::<HashMap<_, _>>();
|
||||
if observations.is_empty() {
|
||||
return 0;
|
||||
}
|
||||
let mut values = self
|
||||
.buckets_usage
|
||||
.iter()
|
||||
.filter(|(bucket, _)| !observations.contains_key(bucket))
|
||||
.map(|(_, usage)| usage)
|
||||
.chain(observations.values().map(|observation| &observation.usage));
|
||||
let totals = values.try_fold([0_u64; 4], |total, usage| {
|
||||
Some([
|
||||
total[0].checked_add(usage.size)?,
|
||||
total[1].checked_add(usage.objects_count)?,
|
||||
total[2].checked_add(usage.versions_count)?,
|
||||
total[3].checked_add(usage.delete_markers_count)?,
|
||||
])
|
||||
});
|
||||
let Some([size, objects, versions, markers]) = totals else {
|
||||
return 0;
|
||||
};
|
||||
let projected = observations.len();
|
||||
for (bucket, observation) in observations {
|
||||
self.buckets_usage.insert(bucket.clone(), observation.usage.clone());
|
||||
}
|
||||
self.bucket_sizes = self
|
||||
.buckets_usage
|
||||
.iter()
|
||||
.map(|(bucket, usage)| (bucket.clone(), usage.size))
|
||||
.collect();
|
||||
self.buckets_count = u64::try_from(self.buckets_usage.len()).unwrap_or(u64::MAX);
|
||||
self.objects_total_size = size;
|
||||
self.objects_total_count = objects;
|
||||
self.versions_total_count = versions;
|
||||
self.delete_markers_total_count = markers;
|
||||
self.replication_info.clear();
|
||||
self.tier_stats = None;
|
||||
self.unknown_tier_stats = None;
|
||||
self.usage_snapshot_complete = false;
|
||||
self.usage_snapshot_partial = true;
|
||||
self.usage_snapshot_converged = Some(false);
|
||||
self.usage_snapshot_observational = true;
|
||||
self.usage_snapshot_admin_projection = true;
|
||||
projected
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
@@ -0,0 +1,323 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
use super::*;
|
||||
use crate::DataUsageSnapshotSetState;
|
||||
use std::time::Duration;
|
||||
|
||||
const INCARNATION: &str = "11111111-1111-4111-8111-111111111111";
|
||||
|
||||
fn time(seconds: u64) -> SystemTime {
|
||||
SystemTime::UNIX_EPOCH + Duration::from_secs(seconds)
|
||||
}
|
||||
|
||||
fn usage(size: u64) -> BucketUsageInfo {
|
||||
BucketUsageInfo {
|
||||
size,
|
||||
objects_count: size,
|
||||
versions_count: size,
|
||||
bucket_incarnation: Some(INCARNATION.to_string()),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn snapshot() -> DataUsageInfo {
|
||||
let mut main = usage(10);
|
||||
main.certify(BucketUsageCertification {
|
||||
epoch: 2,
|
||||
cycle: 3,
|
||||
certified_at: time(30),
|
||||
});
|
||||
main.mark_carried();
|
||||
let mut snapshot = DataUsageInfo {
|
||||
last_update: Some(time(90)),
|
||||
scanner_epoch: Some(2),
|
||||
scanner_cycle: Some(9),
|
||||
buckets_count: 1,
|
||||
buckets_usage: HashMap::from([("bucket".to_string(), main)]),
|
||||
bucket_sizes: HashMap::from([("bucket".to_string(), 10)]),
|
||||
usage_snapshot_complete: true,
|
||||
usage_snapshot_converged: Some(false),
|
||||
usage_snapshot_pending_buckets: vec!["bucket".to_string()],
|
||||
usage_snapshot_pending_bucket_count: 1,
|
||||
usage_snapshot_set_states: (0..2)
|
||||
.map(|set_index| DataUsageSnapshotSetState {
|
||||
pool_index: 0,
|
||||
set_index,
|
||||
scanner_epoch: Some(2),
|
||||
scanner_cycle: Some(9),
|
||||
scan_plan_digest: Some([1; 32]),
|
||||
complete: true,
|
||||
..Default::default()
|
||||
})
|
||||
.collect(),
|
||||
..Default::default()
|
||||
};
|
||||
snapshot.calculate_totals();
|
||||
let mut observed = usage(25);
|
||||
observed.mark_observed();
|
||||
let observation = DataUsageBucketObservation {
|
||||
version: DATA_USAGE_BUCKET_OBSERVATION_VERSION,
|
||||
usage: observed,
|
||||
snapshot_identity: snapshot.snapshot_identity(),
|
||||
completed_at: time(40),
|
||||
sources: (0..2)
|
||||
.map(|set_index| DataUsageBucketObservationSource {
|
||||
pool_index: 0,
|
||||
set_index,
|
||||
bucket_incarnation: INCARNATION.to_string(),
|
||||
scanner_epoch: 2,
|
||||
scanner_cycle: 4 + set_index,
|
||||
scan_plan_digest: [1; 32],
|
||||
started_plan: [2; 32],
|
||||
requested_plan: [3; 32],
|
||||
completed_at: time(40 + set_index),
|
||||
})
|
||||
.collect(),
|
||||
};
|
||||
snapshot
|
||||
.usage_snapshot_bucket_observations
|
||||
.insert("bucket".to_string(), observation);
|
||||
snapshot
|
||||
}
|
||||
|
||||
fn incarnations() -> HashMap<String, String> {
|
||||
HashMap::from([("bucket".to_string(), INCARNATION.to_string())])
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn provenance_is_additive_and_unknown_states_do_not_keep_certificates() {
|
||||
let legacy = usage(10);
|
||||
let bytes = serde_json::to_vec(&legacy).expect("encode a legacy bucket without provenance");
|
||||
let text = std::str::from_utf8(&bytes).expect("JSON is UTF-8");
|
||||
assert!(!text.contains("usage_value_state"));
|
||||
assert!(!text.contains("usage_certification"));
|
||||
let decoded: BucketUsageInfo = serde_json::from_slice(&bytes).expect("read legacy bucket");
|
||||
assert_eq!(decoded.usage_value_state, None);
|
||||
assert_eq!(decoded.usage_certification, None);
|
||||
|
||||
let mut wire = serde_json::to_value(legacy).expect("encode future bucket fixture");
|
||||
wire["usage_value_state"] = serde_json::json!("future_state");
|
||||
wire["usage_certification"] =
|
||||
serde_json::json!({"epoch":2,"cycle":3,"certified_at":{"secs_since_epoch":30,"nanos_since_epoch":0}});
|
||||
let mut future: BucketUsageInfo = serde_json::from_value(wire).expect("unknown diagnostic enum remains readable");
|
||||
future.normalize_usage_provenance();
|
||||
assert_eq!(future.usage_value_state, Some(BucketUsageValueState::Unknown));
|
||||
assert_eq!(future.usage_certification, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn carrying_preserves_time_but_adding_a_source_invalidates_the_certificate() {
|
||||
let mut bucket = snapshot().buckets_usage.remove("bucket").expect("canonical bucket");
|
||||
bucket.mark_carried();
|
||||
assert_eq!(bucket.usage_certification.map(|certification| certification.certified_at), Some(time(30)));
|
||||
bucket.merge(&usage(1));
|
||||
assert_eq!(bucket.size, 11);
|
||||
assert_eq!(bucket.usage_value_state, Some(BucketUsageValueState::Observed));
|
||||
assert_eq!(bucket.usage_certification, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn direct_object_accumulation_does_not_reuse_the_old_certificate() {
|
||||
let mut simple = snapshot();
|
||||
simple.add_object("bucket/object", &rustfs_filemeta::MetaObject::default());
|
||||
assert_eq!(simple.buckets_usage["bucket"].usage_value_state, Some(BucketUsageValueState::Observed));
|
||||
assert_eq!(simple.buckets_usage["bucket"].usage_certification, None);
|
||||
assert!(simple.usage_snapshot_bucket_observations.is_empty());
|
||||
|
||||
let mut versioned = snapshot();
|
||||
versioned.add_object_from_file_meta("bucket/object", &rustfs_filemeta::FileMeta::default());
|
||||
assert_eq!(versioned.buckets_usage["bucket"].usage_value_state, Some(BucketUsageValueState::Observed));
|
||||
assert_eq!(versioned.buckets_usage["bucket"].usage_certification, None);
|
||||
assert!(versioned.usage_snapshot_bucket_observations.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn future_or_malformed_companions_do_not_break_the_canonical_reader() {
|
||||
let canonical = snapshot();
|
||||
for entry in [
|
||||
serde_json::json!({"version": 2, "future_payload": {"new": true}}),
|
||||
serde_json::json!({"version": 1, "usage": "broken"}),
|
||||
] {
|
||||
let mut wire = serde_json::to_value(&canonical).expect("canonical JSON");
|
||||
wire["usage_snapshot_bucket_observations"]["bucket"] = entry;
|
||||
let decoded: DataUsageInfo = serde_json::from_value(wire).expect("diagnostics cannot break canonical data");
|
||||
assert_eq!(decoded.buckets_usage, canonical.buckets_usage);
|
||||
assert!(decoded.usage_snapshot_bucket_observations.is_empty());
|
||||
assert!(decoded.is_complete_bucket_usage_snapshot());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn complete_companion_projects_only_the_admin_copy_and_keeps_measurement_times() {
|
||||
let canonical = snapshot();
|
||||
let canonical_bytes = serde_json::to_vec(&canonical).expect("encode canonical candidate");
|
||||
let mut response = canonical.clone();
|
||||
assert_eq!(response.project_bucket_observations_for_admin(&incarnations()), 1);
|
||||
assert_eq!(response.buckets_usage["bucket"].size, 25);
|
||||
assert_eq!(response.buckets_usage["bucket"].usage_value_state, Some(BucketUsageValueState::Observed));
|
||||
assert_eq!(response.buckets_usage["bucket"].usage_certification, None);
|
||||
assert_eq!(response.objects_total_size, 25);
|
||||
assert_eq!(response.bucket_sizes["bucket"], 25);
|
||||
assert_eq!(response.last_update, canonical.last_update);
|
||||
assert_eq!(response.usage_snapshot_bucket_observations["bucket"].completed_at, time(40));
|
||||
assert!(response.usage_snapshot_admin_projection);
|
||||
assert!(response.usage_snapshot_partial);
|
||||
assert!(response.usage_snapshot_observational);
|
||||
assert_eq!(response.usage_snapshot_converged, Some(false));
|
||||
assert!(!response.is_complete_bucket_usage_snapshot());
|
||||
assert!(
|
||||
!response.is_valid_partial_snapshot(),
|
||||
"response-only values cannot be a storage candidate"
|
||||
);
|
||||
assert_eq!(serde_json::to_vec(&canonical).expect("reencode original"), canonical_bytes);
|
||||
assert_eq!(canonical.buckets_usage["bucket"].size, 10);
|
||||
assert_eq!(
|
||||
canonical.buckets_usage["bucket"]
|
||||
.usage_certification
|
||||
.map(|certificate| certificate.certified_at),
|
||||
Some(time(30))
|
||||
);
|
||||
assert_eq!(response.project_bucket_observations_for_admin(&incarnations()), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_companions_preserve_the_canonical_bucket() {
|
||||
for invalid in 0..11 {
|
||||
let mut candidate = snapshot();
|
||||
let observation = candidate
|
||||
.usage_snapshot_bucket_observations
|
||||
.get_mut("bucket")
|
||||
.expect("companion");
|
||||
match invalid {
|
||||
0 => observation.version += 1,
|
||||
1 => observation.snapshot_identity.scanner_cycle = Some(8),
|
||||
2 => {
|
||||
observation.sources.pop();
|
||||
}
|
||||
3 => observation.sources[1].set_index = 0,
|
||||
4 => observation.sources[1].scanner_cycle = 10,
|
||||
5 => observation.sources[1].scan_plan_digest = [9; 32],
|
||||
6 => observation.sources[1].bucket_incarnation = "other".to_string(),
|
||||
7 => observation.completed_at = time(41),
|
||||
8 => observation.usage.certify(BucketUsageCertification {
|
||||
epoch: 2,
|
||||
cycle: 9,
|
||||
certified_at: time(90),
|
||||
}),
|
||||
9 => candidate.usage_snapshot_set_states[1].scanner_cycle = None,
|
||||
10 => {
|
||||
candidate
|
||||
.buckets_usage
|
||||
.get_mut("bucket")
|
||||
.expect("canonical")
|
||||
.bucket_incarnation = Some("recreated".to_string())
|
||||
}
|
||||
_ => unreachable!("bounded fixture variants"),
|
||||
}
|
||||
let before = candidate.clone();
|
||||
assert_eq!(
|
||||
candidate.project_bucket_observations_for_admin(&incarnations()),
|
||||
0,
|
||||
"invalid case {invalid}"
|
||||
);
|
||||
assert_eq!(candidate, before, "invalid case {invalid} changed canonical data");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_fresh_set_can_contribute_without_refreshing_an_older_mixed_set() {
|
||||
let mut candidate = snapshot();
|
||||
let observation = candidate
|
||||
.usage_snapshot_bucket_observations
|
||||
.get_mut("bucket")
|
||||
.expect("companion");
|
||||
observation.sources[1].started_plan = observation.sources[1].requested_plan;
|
||||
observation.sources[1].completed_at = time(90);
|
||||
assert_eq!(candidate.project_bucket_observations_for_admin(&incarnations()), 1);
|
||||
assert_eq!(candidate.usage_snapshot_bucket_observations["bucket"].completed_at, time(40));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn companion_only_partial_keeps_legacy_maps_empty_until_admin_projection() {
|
||||
let mut candidate = snapshot();
|
||||
candidate.buckets_usage.clear();
|
||||
candidate.bucket_sizes.clear();
|
||||
candidate.buckets_count = 0;
|
||||
candidate.calculate_totals();
|
||||
candidate.usage_snapshot_complete = false;
|
||||
candidate.usage_snapshot_partial = true;
|
||||
candidate.usage_snapshot_observational = true;
|
||||
for state in &mut candidate.usage_snapshot_set_states {
|
||||
state.complete = false;
|
||||
}
|
||||
assert!(candidate.is_valid_partial_snapshot());
|
||||
let wire = serde_json::to_value(&candidate).expect("encode companion-only observation");
|
||||
assert_eq!(wire["buckets_usage"], serde_json::json!({}));
|
||||
assert_eq!(wire["bucket_sizes"], serde_json::json!({}));
|
||||
assert_eq!(wire["objects_total_size"], 0);
|
||||
assert_eq!(candidate.project_bucket_observations_for_admin(&incarnations()), 1);
|
||||
assert_eq!(candidate.objects_total_size, 25);
|
||||
assert!(candidate.usage_snapshot_partial);
|
||||
assert!(!candidate.usage_snapshot_complete);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn projection_rejects_counter_overflow_without_partial_mutation() {
|
||||
let mut candidate = snapshot();
|
||||
candidate.buckets_usage.insert("cold".to_string(), usage(u64::MAX));
|
||||
candidate.buckets_count = 2;
|
||||
let before = candidate.clone();
|
||||
assert_eq!(candidate.project_bucket_observations_for_admin(&incarnations()), 0);
|
||||
assert_eq!(candidate, before);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_newer_canonical_certificate_and_missing_pending_prevent_old_overlay() {
|
||||
for newer_certificate in [false, true] {
|
||||
let mut candidate = snapshot();
|
||||
if newer_certificate {
|
||||
candidate
|
||||
.buckets_usage
|
||||
.get_mut("bucket")
|
||||
.expect("canonical")
|
||||
.certify(BucketUsageCertification {
|
||||
epoch: 2,
|
||||
cycle: 9,
|
||||
certified_at: time(90),
|
||||
});
|
||||
} else {
|
||||
candidate.usage_snapshot_pending_buckets.clear();
|
||||
candidate.usage_snapshot_pending_bucket_count = 0;
|
||||
}
|
||||
assert_eq!(candidate.project_bucket_observations_for_admin(&incarnations()), 0);
|
||||
assert_eq!(candidate.buckets_usage["bucket"].size, 10);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn provenance_and_companions_change_candidate_bytes_and_round_trip_as_maps() {
|
||||
let canonical = snapshot();
|
||||
let original = serde_json::to_vec(&canonical).expect("encode candidate");
|
||||
let mut changed = canonical.clone();
|
||||
changed
|
||||
.buckets_usage
|
||||
.get_mut("bucket")
|
||||
.expect("bucket")
|
||||
.usage_certification
|
||||
.as_mut()
|
||||
.expect("certificate")
|
||||
.certified_at = time(31);
|
||||
assert_ne!(serde_json::to_vec(&changed).expect("encode changed certificate"), original);
|
||||
let mut changed = canonical.clone();
|
||||
changed
|
||||
.usage_snapshot_bucket_observations
|
||||
.get_mut("bucket")
|
||||
.expect("observation")
|
||||
.sources[0]
|
||||
.started_plan = [8; 32];
|
||||
assert_ne!(serde_json::to_vec(&changed).expect("encode changed source"), original);
|
||||
let named = rmp_serde::to_vec_named(&canonical).expect("encode named map");
|
||||
let restored: DataUsageInfo = rmp_serde::from_slice(&named).expect("read named map with additive fields");
|
||||
assert_eq!(restored, canonical);
|
||||
}
|
||||
@@ -25,6 +25,7 @@ workspace = true
|
||||
|
||||
[features]
|
||||
default = []
|
||||
scanner-peer-mixed-version = []
|
||||
hotpath = [
|
||||
"hotpath/hotpath",
|
||||
"hotpath/tokio",
|
||||
|
||||
@@ -26,7 +26,7 @@ Registered in [`src/lib.rs`](src/lib.rs). Grouped by concern:
|
||||
| **protocols** | [`src/protocols/`](src/protocols) | FTPS, WebDAV, SFTP compliance. Fixed ports, own guide: [`src/protocols/README.md`](src/protocols/README.md) |
|
||||
| **reliant** | [`src/reliant/`](src/reliant) | Tests that reuse an **externally started** server (SQL/select, conditional writes, lifecycle, deleted-object reads, node-interact). Run via [`scripts/run_e2e_tests.sh`](../../scripts/run_e2e_tests.sh); see [`src/reliant/README.md`](src/reliant/README.md) |
|
||||
| **cluster** | `cluster_concurrency_test`, `stale_multipart_cleanup_cluster_test`, `namespace_lock_quorum_test`, `admin_timeout_regression_test`, `object_lambda_test`, `replication_extension_test`, `tier_stats_cluster_test` | Multi-node scenarios via `RustFSTestClusterEnvironment` |
|
||||
| **distributed 4×4** | [`src/distributed/`](src/distributed) | Storage-sensitive PR and nightly `e2e-distributed` lane: S3, object lock/WORM, versioning, bucket/site replication, quota, expand/decommission/rebalance, concurrency, chaos, 4-node upgrade of historical data and IAM AK/SK. Map: [`docs/testing/distributed-e2e.md`](../../docs/testing/distributed-e2e.md) |
|
||||
| **distributed 4×4** | [`src/distributed/`](src/distributed) | Nightly `e2e-distributed` lane: S3, object lock/WORM, versioning, bucket/site replication, quota, expand/decommission/rebalance, concurrency, chaos, 4-node upgrade of historical data and IAM AK/SK. Map: [`docs/testing/distributed-e2e.md`](../../docs/testing/distributed-e2e.md) |
|
||||
| **chaos / reliability** | [`src/chaos.rs`](src/chaos.rs), `reliability_disk_fault_test`, `heal_erasure_disk_rebuild_test`, `server_startup_failfast_test` | Disk offline/replace/corrupt, EC rebuild, heal, fail-fast startup |
|
||||
| **upgrade compatibility** | `upgrade_compatibility_test` | Pinned previous-release writes followed by current-build reads on the same data directory |
|
||||
|
||||
@@ -64,6 +64,25 @@ Each build/run holds an exclusive `rustfs.e2e.lock` marker beside the binary; co
|
||||
|
||||
The protocols suite has its own fixed-port and single-worker contract in [`src/protocols/README.md`](src/protocols/README.md). Use its command under [Troubleshooting](#troubleshooting).
|
||||
|
||||
### Scanner mixed-version gate
|
||||
|
||||
The Linux N/O scanner gate needs a separately qualified `33482bb` server, which supports scoped ACKs. Set `RUSTFS_SCANNER_COMPAT_OLD_BINARY` to that executable and `RUSTFS_SCANNER_COMPAT_OLD_SHA256` to its independently verified digest. The release binary used by ordinary upgrade CI is a different baseline. Missing or mismatched inputs fail the test.
|
||||
|
||||
Enable the harness-only `scanner-peer-mixed-version` feature explicitly; keep the server wrapper features identical between build and run:
|
||||
|
||||
```bash
|
||||
python3 scripts/e2e_binary.py build --features e2e-test-hooks
|
||||
python3 scripts/e2e_binary.py run --features e2e-test-hooks -- \
|
||||
cargo nextest run --profile e2e-distributed -p e2e_test \
|
||||
--features scanner-peer-mixed-version --ignore-default-filter \
|
||||
-E 'test(=distributed::scanner_peer_bucket_test::scanner_peer_bucket_new_coordinator_certifies_old_peers_and_scopes_next_cycle)' \
|
||||
--test-threads 1 --retries 0 --no-tests=fail
|
||||
```
|
||||
|
||||
The separate `scanner_peer_bucket_new_coordinator_certifies_old_worker_peers_and_scopes_next_cycle` case uses the same feature and qualified old binary inputs. Select its exact name in the command above. It keeps the original N process enabled and starts the three official O replacements with `RUSTFS_SCANNER_ENABLED=false`: their autonomous scanner loops are disabled, while real S3 dirty tracking, authenticated snapshots, publication leases, scoped ACKs, and N-dispatched RemoteScanner workers remain active. It records each node's actual scanner status and retains the same identity, durability, root, ACK, data-conservation, and physical-walk requirements. This fixed-role case does not validate autonomous O scanning or all-enabled election behavior. The original all-enabled N/O test remains separate and has not passed acceptance; a fixed-role result cannot satisfy that gate.
|
||||
|
||||
For this fixed-role case, the `e2e-test-hooks` server also records the N client's complete bounded worker inventory. Requests are observed before opening the normal stream and paired only after authenticated source/plan validation and terminal EOF. Closed manifests and final health receipts must bind both cycles to their actual candidate/root; the next cycle needs exactly four sets of hot-only execution and at least one positive-progress worker mapped to a measured official O process. A zero-progress Current response, missing/extra request, unhealthy recorder, or missing physical coverage fails the gate. The recorder does not add O hooks or change any protocol or timeout. Final health reads check immutable closed-cycle evidence, not current N liveness.
|
||||
|
||||
### `#[ignore]` semantics
|
||||
|
||||
Ignored tests are excluded from the default `cargo nextest run` pass because
|
||||
@@ -175,9 +194,9 @@ the same profile for membership and execution with one nightly worker.
|
||||
| `s3s-e2e` black-box | `e2e-tests` + `e2e-tests-rio-v2` jobs | **Active** (external conformance tool) |
|
||||
| ILM / lifecycle (ignored) | `test-ilm-integration-serial` lane, `-j1` | **Active** (backlog#1148 ilm-1) |
|
||||
| KMS suite | `e2e-full` job, merge queue + main | **Active** |
|
||||
| Direct and mixed-version rolling upgrades from pinned previous release | `e2e-upgrade.yml`, storage-sensitive PRs + release tags + weekly | **Active** |
|
||||
| Direct and mixed-version rolling upgrades from pinned previous release | `e2e-upgrade.yml`, release tags + weekly schedule + manual dispatch | **Active** |
|
||||
| Cluster faults (`e2e-nightly` profile) | consolidated nightly workflow | **Active** (backlog#1149 ci-7) |
|
||||
| Distributed 4-node 4-disk (`e2e-distributed` profile) | `.github/workflows/e2e-distributed.yml` | **Active** (storage-sensitive PR / nightly / dispatch) |
|
||||
| Distributed 4-node 4-disk (`e2e-distributed` profile) | `.github/workflows/e2e-distributed.yml` | **Active** (nightly / dispatch) |
|
||||
| Protocols (FTPS/WebDAV/SFTP) | consolidated nightly workflow, serial | **Active** (backlog#1149 ci-7) |
|
||||
| Replication (fast subset) | `e2e-smoke` profile, `e2e-tests` job, every PR | **Active** (backlog#1147 repl-1) |
|
||||
| Replication (slow + multi-node) | `e2e-repl-nightly` profile, consolidated nightly workflow | **Active** (backlog#1147 repl-1) |
|
||||
|
||||
@@ -260,7 +260,7 @@ async fn iam_policy_requires_if_match(env: &RustFSTestEnvironment, admin: &Clien
|
||||
let key = "guarded.txt";
|
||||
|
||||
admin_create_user(env, &user, user_secret).await?;
|
||||
let policy = serde_json::json!({
|
||||
let mut policy = serde_json::json!({
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{
|
||||
"Effect": "Allow",
|
||||
@@ -268,9 +268,8 @@ async fn iam_policy_requires_if_match(env: &RustFSTestEnvironment, admin: &Clien
|
||||
"Resource": [format!("arn:aws:s3:::{bucket}/*")],
|
||||
"Condition": { "Null": { "s3:if-match": "false" } }
|
||||
}]
|
||||
})
|
||||
.to_string();
|
||||
admin_add_and_attach_policy(env, &policy_name, &policy, &user).await?;
|
||||
});
|
||||
admin_add_and_attach_policy(env, &policy_name, &policy.to_string(), &user).await?;
|
||||
|
||||
admin.create_bucket().bucket(&bucket).send().await?;
|
||||
let original = admin
|
||||
@@ -293,6 +292,46 @@ async fn iam_policy_requires_if_match(env: &RustFSTestEnvironment, admin: &Clien
|
||||
.expect_err("PutObject without If-Match must be denied");
|
||||
assert_error(&denied, "AccessDenied", 403, "unconditional PutObject");
|
||||
|
||||
let denied = client
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.if_match(&etag)
|
||||
.body(ByteStream::from_static(b"missing-read-permission"))
|
||||
.send()
|
||||
.await
|
||||
.expect_err("If-Match without GetObject must be denied");
|
||||
assert_error(&denied, "AccessDenied", 403, "conditional PutObject without GetObject");
|
||||
|
||||
// S3 requires GetObject as well as PutObject for an ETag-based write.
|
||||
policy["Statement"]
|
||||
.as_array_mut()
|
||||
.expect("policy statements")
|
||||
.push(serde_json::json!({
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:GetObject"],
|
||||
"Resource": [format!("arn:aws:s3:::{bucket}/{key}")]
|
||||
}));
|
||||
admin_add_canned_policy_via(
|
||||
AdminTransport::Signed,
|
||||
&env.url,
|
||||
&env.access_key,
|
||||
&env.secret_key,
|
||||
&policy_name,
|
||||
&policy.to_string(),
|
||||
)
|
||||
.await?;
|
||||
|
||||
let denied = client
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.body(ByteStream::from_static(b"blind-overwrite"))
|
||||
.send()
|
||||
.await
|
||||
.expect_err("GetObject permission must not allow an unconditional PutObject");
|
||||
assert_error(&denied, "AccessDenied", 403, "unconditional PutObject with GetObject");
|
||||
|
||||
client
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
@@ -301,6 +340,8 @@ async fn iam_policy_requires_if_match(env: &RustFSTestEnvironment, admin: &Clien
|
||||
.body(ByteStream::from_static(b"v2"))
|
||||
.send()
|
||||
.await?;
|
||||
let written = client.get_object().bucket(&bucket).key(key).send().await?;
|
||||
assert_eq!(written.body.collect().await?.into_bytes().as_ref(), b"v2");
|
||||
|
||||
let _ = admin.delete_object().bucket(&bucket).key(key).send().await;
|
||||
let _ = admin.delete_bucket().bucket(&bucket).send().await;
|
||||
|
||||
@@ -502,10 +502,163 @@ mod tests {
|
||||
assert!(response.contains(if expected == reqwest::StatusCode::FORBIDDEN {
|
||||
"<Code>AccessDenied</Code>"
|
||||
} else {
|
||||
"<Code>BucketAlreadyExists</Code>"
|
||||
"<Code>BucketAlreadyOwnedByYou</Code>"
|
||||
}));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// A deployment owns a single bucket namespace, so an IAM user that recreates
|
||||
/// an existing bucket is the owner, not another account. In the default region
|
||||
/// every such request returns the legacy idempotent `200 OK` (AWS CreateBucket
|
||||
/// semantics), including buckets created by root and by another IAM user.
|
||||
#[tokio::test]
|
||||
async fn test_create_bucket_duplicate_in_default_region_is_idempotent_for_iam_user()
|
||||
-> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec!["--region", "us-east-1"]).await?;
|
||||
let root = env.create_s3_client();
|
||||
|
||||
admin_create_user(&env, "duplicate-iam", "duplicate-iam-secret").await?;
|
||||
admin_attach_user_policy_via(
|
||||
AdminTransport::Signed,
|
||||
&env.url,
|
||||
&env.access_key,
|
||||
&env.secret_key,
|
||||
"readwrite",
|
||||
"duplicate-iam",
|
||||
)
|
||||
.await?;
|
||||
let iam = Client::from_conf(build_test_s3_config(
|
||||
&env.url,
|
||||
"duplicate-iam",
|
||||
"duplicate-iam-secret",
|
||||
None,
|
||||
"duplicate-create",
|
||||
));
|
||||
|
||||
root.create_bucket()
|
||||
.bucket("duplicate-root-owned")
|
||||
.send()
|
||||
.await
|
||||
.expect("root creates the first bucket");
|
||||
iam.create_bucket()
|
||||
.bucket("duplicate-root-owned")
|
||||
.send()
|
||||
.await
|
||||
.expect("an IAM user recreating a root-owned bucket in us-east-1 gets 200 OK");
|
||||
|
||||
iam.create_bucket()
|
||||
.bucket("duplicate-iam-owned")
|
||||
.send()
|
||||
.await
|
||||
.expect("the IAM user creates its own bucket");
|
||||
iam.create_bucket()
|
||||
.bucket("duplicate-iam-owned")
|
||||
.send()
|
||||
.await
|
||||
.expect("the same IAM user recreating its bucket gets 200 OK");
|
||||
root.create_bucket()
|
||||
.bucket("duplicate-iam-owned")
|
||||
.send()
|
||||
.await
|
||||
.expect("root recreating an IAM-created bucket gets 200 OK");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Outside the default region the same ownership answer becomes
|
||||
/// `409 BucketAlreadyOwnedByYou` for every authorized requester. Requests that
|
||||
/// lack `s3:CreateBucket` still fail authorization before the duplicate check.
|
||||
#[tokio::test]
|
||||
async fn test_create_bucket_duplicate_outside_default_region_reports_owned_by_you() -> Result<(), Box<dyn Error + Send + Sync>>
|
||||
{
|
||||
init_logging();
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec!["--region", "eu-west-1"]).await?;
|
||||
let region = "eu-west-1";
|
||||
let root = Client::from_conf(
|
||||
env.create_s3_client()
|
||||
.config()
|
||||
.to_builder()
|
||||
.region(Region::new(region))
|
||||
.build(),
|
||||
);
|
||||
|
||||
admin_create_user(&env, "duplicate-eu-iam", "duplicate-eu-iam-secret").await?;
|
||||
admin_attach_user_policy_via(
|
||||
AdminTransport::Signed,
|
||||
&env.url,
|
||||
&env.access_key,
|
||||
&env.secret_key,
|
||||
"readwrite",
|
||||
"duplicate-eu-iam",
|
||||
)
|
||||
.await?;
|
||||
let iam = Client::from_conf(
|
||||
build_test_s3_config(&env.url, "duplicate-eu-iam", "duplicate-eu-iam-secret", None, "duplicate-create")
|
||||
.to_builder()
|
||||
.region(Region::new(region))
|
||||
.build(),
|
||||
);
|
||||
|
||||
admin_create_user(&env, "duplicate-eu-readonly", "duplicate-eu-readonly-secret").await?;
|
||||
admin_attach_user_policy_via(
|
||||
AdminTransport::Signed,
|
||||
&env.url,
|
||||
&env.access_key,
|
||||
&env.secret_key,
|
||||
"readonly",
|
||||
"duplicate-eu-readonly",
|
||||
)
|
||||
.await?;
|
||||
let readonly = Client::from_conf(
|
||||
build_test_s3_config(
|
||||
&env.url,
|
||||
"duplicate-eu-readonly",
|
||||
"duplicate-eu-readonly-secret",
|
||||
None,
|
||||
"duplicate-create",
|
||||
)
|
||||
.to_builder()
|
||||
.region(Region::new(region))
|
||||
.build(),
|
||||
);
|
||||
|
||||
for (client, bucket) in [(&root, "duplicate-eu-root"), (&iam, "duplicate-eu-iam")] {
|
||||
client
|
||||
.create_bucket()
|
||||
.bucket(bucket)
|
||||
.send()
|
||||
.await
|
||||
.unwrap_or_else(|error| panic!("{bucket} must be created: {error:?}"));
|
||||
// Both the creator and the other authenticated identity own the namespace.
|
||||
for creator in [&root, &iam] {
|
||||
let error = creator
|
||||
.create_bucket()
|
||||
.bucket(bucket)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("a duplicate create outside us-east-1 must report ownership");
|
||||
assert_eq!(
|
||||
error.as_service_error().and_then(ProvideErrorMetadata::code),
|
||||
Some("BucketAlreadyOwnedByYou"),
|
||||
"{bucket}"
|
||||
);
|
||||
assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(409), "{bucket}");
|
||||
}
|
||||
}
|
||||
|
||||
let denied = readonly
|
||||
.create_bucket()
|
||||
.bucket("duplicate-eu-root")
|
||||
.send()
|
||||
.await
|
||||
.expect_err("a requester without s3:CreateBucket must be denied before the duplicate check");
|
||||
assert_eq!(denied.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,8 +16,193 @@
|
||||
mod tests {
|
||||
use crate::common::{RustFSTestEnvironment, init_logging};
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::config::http::HttpResponse;
|
||||
use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError};
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
|
||||
use aws_sdk_s3::types::{BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, VersioningConfiguration};
|
||||
|
||||
fn assert_not_found<E: ProvideErrorMetadata + std::fmt::Debug>(error: &SdkError<E, HttpResponse>) {
|
||||
assert_eq!(error.raw_response().map(|r| r.status().as_u16()), Some(404), "{error:?}");
|
||||
assert_eq!(
|
||||
error.as_service_error().and_then(ProvideErrorMetadata::code),
|
||||
Some("NoSuchKey"),
|
||||
"{error:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn conditional_writes_cannot_replace_delete_marker_with_if_match() {
|
||||
init_logging();
|
||||
let mut env = RustFSTestEnvironment::new().await.expect("create test environment");
|
||||
env.start_rustfs_server(vec![]).await.expect("start RustFS");
|
||||
let client = env.create_s3_client();
|
||||
let bucket = "conditional-delete-marker-proof";
|
||||
create_versioned_bucket(&client, bucket).await;
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key("source")
|
||||
.body(ByteStream::from_static(b"source"))
|
||||
.send()
|
||||
.await
|
||||
.expect("write copy source");
|
||||
|
||||
for operation in ["put", "copy", "multipart"] {
|
||||
let old = client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(operation)
|
||||
.body(ByteStream::from_static(b"old"))
|
||||
.send()
|
||||
.await
|
||||
.expect("write historical version");
|
||||
let old_etag = old.e_tag().expect("historical ETag");
|
||||
let marker = client
|
||||
.delete_object()
|
||||
.bucket(bucket)
|
||||
.key(operation)
|
||||
.send()
|
||||
.await
|
||||
.expect("create delete marker");
|
||||
let marker_id = marker.version_id().expect("delete marker version");
|
||||
|
||||
let (upload_id, completed) = if operation == "multipart" {
|
||||
let upload = client
|
||||
.create_multipart_upload()
|
||||
.bucket(bucket)
|
||||
.key(operation)
|
||||
.send()
|
||||
.await
|
||||
.expect("initiate multipart upload");
|
||||
let upload_id = upload.upload_id().expect("upload ID").to_owned();
|
||||
let part = client
|
||||
.upload_part()
|
||||
.bucket(bucket)
|
||||
.key(operation)
|
||||
.upload_id(&upload_id)
|
||||
.part_number(1)
|
||||
.body(ByteStream::from_static(b"new"))
|
||||
.send()
|
||||
.await
|
||||
.expect("upload part");
|
||||
let completed = CompletedMultipartUpload::builder()
|
||||
.parts(
|
||||
CompletedPart::builder()
|
||||
.part_number(1)
|
||||
.set_e_tag(part.e_tag().map(str::to_owned))
|
||||
.build(),
|
||||
)
|
||||
.build();
|
||||
(Some(upload_id), Some(completed))
|
||||
} else {
|
||||
(None, None)
|
||||
};
|
||||
|
||||
for etag in [old_etag, "\"00000000000000000000000000000000\""] {
|
||||
match operation {
|
||||
"put" => {
|
||||
let err = client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(operation)
|
||||
.if_match(etag)
|
||||
.body(ByteStream::from_static(b"new"))
|
||||
.send()
|
||||
.await
|
||||
.expect_err("If-Match must reject marker");
|
||||
assert_not_found(&err);
|
||||
}
|
||||
"copy" => {
|
||||
let err = client
|
||||
.copy_object()
|
||||
.bucket(bucket)
|
||||
.key(operation)
|
||||
.copy_source(format!("{bucket}/source"))
|
||||
.if_match(etag)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("If-Match copy must reject marker");
|
||||
assert_not_found(&err);
|
||||
}
|
||||
_ => {
|
||||
let err = client
|
||||
.complete_multipart_upload()
|
||||
.bucket(bucket)
|
||||
.key(operation)
|
||||
.upload_id(upload_id.as_deref().expect("upload ID"))
|
||||
.multipart_upload(completed.clone().expect("completed parts"))
|
||||
.if_match(etag)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("If-Match completion must reject marker");
|
||||
assert_not_found(&err);
|
||||
}
|
||||
}
|
||||
let versions = client
|
||||
.list_object_versions()
|
||||
.bucket(bucket)
|
||||
.prefix(operation)
|
||||
.send()
|
||||
.await
|
||||
.expect("list versions after rejected write");
|
||||
assert_eq!(versions.versions().len(), 1, "{operation}: failed write created a version");
|
||||
assert_eq!(versions.delete_markers().len(), 1);
|
||||
assert_eq!(versions.delete_markers()[0].version_id(), Some(marker_id));
|
||||
assert_eq!(versions.delete_markers()[0].is_latest(), Some(true));
|
||||
}
|
||||
|
||||
match operation {
|
||||
"put" => {
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(operation)
|
||||
.if_none_match("*")
|
||||
.body(ByteStream::from_static(b"new"))
|
||||
.send()
|
||||
.await
|
||||
.expect("If-None-Match PUT over marker");
|
||||
}
|
||||
"copy" => {
|
||||
client
|
||||
.copy_object()
|
||||
.bucket(bucket)
|
||||
.key(operation)
|
||||
.copy_source(format!("{bucket}/source"))
|
||||
.if_none_match("*")
|
||||
.send()
|
||||
.await
|
||||
.expect("If-None-Match copy over marker");
|
||||
}
|
||||
_ => {
|
||||
client
|
||||
.complete_multipart_upload()
|
||||
.bucket(bucket)
|
||||
.key(operation)
|
||||
.upload_id(upload_id.as_deref().expect("upload ID"))
|
||||
.multipart_upload(completed.expect("completed parts"))
|
||||
.if_none_match("*")
|
||||
.send()
|
||||
.await
|
||||
.expect("If-None-Match completion over marker");
|
||||
}
|
||||
}
|
||||
let current = client
|
||||
.get_object()
|
||||
.bucket(bucket)
|
||||
.key(operation)
|
||||
.send()
|
||||
.await
|
||||
.expect("read successful conditional write")
|
||||
.body
|
||||
.collect()
|
||||
.await
|
||||
.expect("read object body")
|
||||
.into_bytes();
|
||||
let expected: &[u8] = if operation == "copy" { b"source" } else { b"new" };
|
||||
assert_eq!(current.as_ref(), expected, "{operation}: unexpected current object");
|
||||
}
|
||||
}
|
||||
|
||||
async fn create_versioned_bucket(client: &Client, bucket: &str) {
|
||||
client
|
||||
|
||||
@@ -12,11 +12,249 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::harness::{DistCluster, DistLayout, TestResult, assert_object_bytes, payload_for, put_object, unique_bucket};
|
||||
use super::harness::{DistCluster, DistLayout, TestResult, payload_for, s3_probe_client, sha256_hex, unique_bucket};
|
||||
use crate::common::init_logging;
|
||||
use aws_sdk_s3::{
|
||||
Client,
|
||||
error::{ProvideErrorMetadata, SdkError},
|
||||
primitives::ByteStream,
|
||||
};
|
||||
use std::any::TypeId;
|
||||
use std::collections::BTreeSet;
|
||||
use std::fmt::Debug;
|
||||
use std::future::Future;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use tokio::sync::Barrier;
|
||||
use tokio::time::Instant;
|
||||
|
||||
const LOCK_ADMISSION_BUDGET: Duration = Duration::from_secs(45);
|
||||
|
||||
async fn after_lock_contention<T, E, F, Fut>(operation: &str, mut call: F) -> TestResult<Result<T, SdkError<E>>>
|
||||
where
|
||||
E: Debug + ProvideErrorMetadata + 'static,
|
||||
F: FnMut() -> Fut,
|
||||
Fut: Future<Output = Result<T, SdkError<E>>>,
|
||||
{
|
||||
let started = Instant::now();
|
||||
let mut attempts = 0;
|
||||
// The workload uses unique unversioned keys. Repeat only explicitly
|
||||
// advertised lock backpressure; every step still needs its real result.
|
||||
let outcome = tokio::time::timeout(LOCK_ADMISSION_BUDGET, async {
|
||||
loop {
|
||||
attempts += 1;
|
||||
let result = call().await;
|
||||
let retry = result.as_ref().err().is_some_and(|error| {
|
||||
let Some(response) = error.raw_response() else {
|
||||
return false;
|
||||
};
|
||||
let Some(service_error) = error.as_service_error() else {
|
||||
return false;
|
||||
};
|
||||
// HEAD has no XML error body, so its SDK service error can
|
||||
// lack a code even when the server supplies retry headers.
|
||||
let namespace_code = service_error.code() == Some("ServiceUnavailable")
|
||||
|| (service_error.code().is_none()
|
||||
&& TypeId::of::<E>() == TypeId::of::<aws_sdk_s3::operation::head_object::HeadObjectError>());
|
||||
response.status().as_u16() == 503
|
||||
&& namespace_code
|
||||
&& response.headers().get("x-rustfs-error-cause") == Some("namespace_lock_timeout")
|
||||
&& response.headers().get("retry-after") == Some("1")
|
||||
});
|
||||
if !retry {
|
||||
return result;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_secs(1)).await;
|
||||
}
|
||||
})
|
||||
.await;
|
||||
if attempts > 1 || !matches!(&outcome, Ok(Ok(_))) {
|
||||
eprintln!(
|
||||
"mixed workload request: operation={operation}, attempts={attempts}, elapsed={:?}",
|
||||
started.elapsed()
|
||||
);
|
||||
}
|
||||
outcome.map_err(|_| {
|
||||
format!("{operation} exceeded its {LOCK_ADMISSION_BUDGET:?} request budget after {attempts} attempts").into()
|
||||
})
|
||||
}
|
||||
|
||||
async fn assert_workload_bytes(client: &Client, bucket: &str, key: &str, expected: &[u8]) -> TestResult {
|
||||
let output = after_lock_contention(&format!("GET {key}"), || client.get_object().bucket(bucket).key(key).send()).await??;
|
||||
let actual = output.body.collect().await?.into_bytes();
|
||||
if actual.as_ref() != expected {
|
||||
return Err(format!(
|
||||
"object {key} bytes mismatch: expected {} bytes sha256={} got {} bytes sha256={}",
|
||||
expected.len(),
|
||||
sha256_hex(expected),
|
||||
actual.len(),
|
||||
sha256_hex(&actual)
|
||||
)
|
||||
.into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use aws_smithy_http_client::test_util::infallible_client_fn;
|
||||
use aws_smithy_types::body::SdkBody;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
||||
type Reply = (u16, &'static str, Option<&'static str>, Option<&'static str>, &'static str);
|
||||
|
||||
fn client(respond: impl Fn(usize) -> Reply + Send + Sync + 'static) -> (Client, Arc<AtomicUsize>) {
|
||||
let requests = Arc::new(AtomicUsize::new(0));
|
||||
let seen = Arc::clone(&requests);
|
||||
let transport = infallible_client_fn(move |_| {
|
||||
let (status, code, cause, retry_after, body) = respond(seen.fetch_add(1, Ordering::SeqCst));
|
||||
let body = if code.is_empty() {
|
||||
body.to_string()
|
||||
} else {
|
||||
format!("<Error><Code>{code}</Code></Error>")
|
||||
};
|
||||
let mut response = http::Response::builder()
|
||||
.status(status)
|
||||
.header("content-type", "application/xml")
|
||||
.header("content-length", body.len());
|
||||
if let Some(cause) = cause {
|
||||
response = response.header("x-rustfs-error-cause", cause);
|
||||
}
|
||||
if let Some(retry_after) = retry_after {
|
||||
response = response.header("retry-after", retry_after);
|
||||
}
|
||||
response.body(SdkBody::from(body)).expect("scripted S3 response")
|
||||
});
|
||||
let config = crate::common::build_test_s3_config(
|
||||
"http://contention.example.invalid",
|
||||
"synthetic-access",
|
||||
"synthetic-secret",
|
||||
None,
|
||||
"contention-test",
|
||||
);
|
||||
let client = Client::from_conf(config.to_builder().http_client(transport).build());
|
||||
(s3_probe_client(&client), requests)
|
||||
}
|
||||
|
||||
fn contention() -> Reply {
|
||||
(503, "ServiceUnavailable", Some("namespace_lock_timeout"), Some("1"), "")
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn advertised_contention_recovers_after_standard_sdk_attempts_are_exhausted() -> TestResult {
|
||||
let (client, requests) = client(|attempt| {
|
||||
if attempt < 3 {
|
||||
contention()
|
||||
} else {
|
||||
(200, "", None, None, "")
|
||||
}
|
||||
});
|
||||
let standard = Client::from_conf(
|
||||
client
|
||||
.config()
|
||||
.to_builder()
|
||||
.retry_config(aws_sdk_s3::config::retry::RetryConfig::standard().with_max_attempts(3))
|
||||
.build(),
|
||||
);
|
||||
standard
|
||||
.put_object()
|
||||
.bucket("bucket")
|
||||
.key("key")
|
||||
.body(ByteStream::from_static(b"body"))
|
||||
.send()
|
||||
.await
|
||||
.expect_err("three advertised lock responses exhaust the standard SDK policy");
|
||||
assert_eq!(requests.load(Ordering::SeqCst), 3);
|
||||
requests.store(0, Ordering::SeqCst);
|
||||
let started = Instant::now();
|
||||
after_lock_contention("PUT recovery", || {
|
||||
client
|
||||
.put_object()
|
||||
.bucket("bucket")
|
||||
.key("key")
|
||||
.body(ByteStream::from_static(b"body"))
|
||||
.send()
|
||||
})
|
||||
.await??;
|
||||
assert_eq!(requests.load(Ordering::SeqCst), 4);
|
||||
assert!(started.elapsed() >= Duration::from_secs(3), "honor each Retry-After response");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn bodyless_head_contention_uses_explicit_retry_headers() -> TestResult {
|
||||
let (client, requests) = client(|attempt| {
|
||||
if attempt == 0 {
|
||||
(503, "", Some("namespace_lock_timeout"), Some("1"), "")
|
||||
} else {
|
||||
(200, "", None, None, "")
|
||||
}
|
||||
});
|
||||
after_lock_contention("HEAD recovery", || client.head_object().bucket("bucket").key("key").send()).await??;
|
||||
assert_eq!(requests.load(Ordering::SeqCst), 2);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn unrelated_or_malformed_responses_are_not_retried() -> TestResult {
|
||||
for reply in [
|
||||
(500, "InternalError", Some("namespace_lock_timeout"), Some("1"), ""),
|
||||
(403, "AccessDenied", None, None, ""),
|
||||
(404, "NoSuchKey", None, None, ""),
|
||||
(503, "ServiceUnavailable", Some("storage_write_quorum"), None, ""),
|
||||
(503, "ServiceUnavailable", Some("namespace_lock_unavailable"), Some("1"), ""),
|
||||
(503, "InternalError", Some("namespace_lock_timeout"), Some("1"), ""),
|
||||
(503, "", Some("namespace_lock_timeout"), Some("1"), ""),
|
||||
(503, "ServiceUnavailable", None, Some("1"), ""),
|
||||
(503, "ServiceUnavailable", Some("namespace_lock_timeout"), None, ""),
|
||||
(503, "ServiceUnavailable", Some("namespace_lock_timeout"), Some("2"), ""),
|
||||
] {
|
||||
let (client, requests) = client(move |_| reply);
|
||||
let result = after_lock_contention("PUT fatal response", || {
|
||||
client
|
||||
.put_object()
|
||||
.bucket("bucket")
|
||||
.key("key")
|
||||
.body(ByteStream::from_static(b"body"))
|
||||
.send()
|
||||
})
|
||||
.await?;
|
||||
assert!(result.is_err(), "fatal response cannot become success: {reply:?}");
|
||||
assert_eq!(requests.load(Ordering::SeqCst), 1, "fatal response must remain visible: {reply:?}");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn persistent_lock_contention_exhausts_the_request_budget() {
|
||||
let (client, requests) = client(|_| contention());
|
||||
let started = Instant::now();
|
||||
let result = after_lock_contention("PUT unavailable", || {
|
||||
client
|
||||
.put_object()
|
||||
.bucket("bucket")
|
||||
.key("key")
|
||||
.body(ByteStream::from_static(b"body"))
|
||||
.send()
|
||||
})
|
||||
.await;
|
||||
assert!(result.is_err(), "persistent contention cannot acknowledge an operation");
|
||||
assert!(started.elapsed() >= LOCK_ADMISSION_BUDGET);
|
||||
assert!(requests.load(Ordering::SeqCst) > 1);
|
||||
assert!(requests.load(Ordering::SeqCst) <= LOCK_ADMISSION_BUDGET.as_secs() as usize + 1);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn acknowledged_wrong_bytes_fail_without_retry() {
|
||||
let (client, requests) = client(|_| (200, "", None, None, "wrong bytes"));
|
||||
let error = assert_workload_bytes(&client, "bucket", "key", b"expected bytes")
|
||||
.await
|
||||
.expect_err("a successful but incorrect read is a consistency failure");
|
||||
assert!(error.to_string().contains("bytes mismatch"));
|
||||
assert_eq!(requests.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node() -> TestResult {
|
||||
@@ -24,7 +262,9 @@ async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node()
|
||||
let dist = DistCluster::start(DistLayout::FourByFour).await?;
|
||||
let bucket = unique_bucket("conc");
|
||||
dist.create_bucket(&bucket).await?;
|
||||
let clients = Arc::new(dist.clients()?);
|
||||
// This consistency probe owns its admission policy; unrelated responses
|
||||
// must not be hidden by the SDK's general-purpose retries.
|
||||
let clients = Arc::new(dist.clients()?.iter().map(s3_probe_client).collect::<Vec<_>>());
|
||||
let worker_count = 24;
|
||||
let rounds = 4;
|
||||
let barrier = Arc::new(Barrier::new(worker_count));
|
||||
@@ -44,31 +284,46 @@ async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node()
|
||||
let key = format!("source/worker-{idx:02}-round-{round}.bin");
|
||||
let copy_key = format!("retained/worker-{idx:02}-round-{round}.bin");
|
||||
let body = payload_for(&key, 64 * 1024);
|
||||
put_object(writer, &bucket, &key, body.clone()).await?;
|
||||
let context = format!("worker={idx} round={round}");
|
||||
after_lock_contention(&format!("{context} PUT source"), || {
|
||||
writer
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key(&key)
|
||||
.body(ByteStream::from(body.clone()))
|
||||
.send()
|
||||
})
|
||||
.await??;
|
||||
|
||||
let head = reader.head_object().bucket(&bucket).key(&key).send().await?;
|
||||
let head = after_lock_contention(&format!("{context} HEAD source"), || {
|
||||
reader.head_object().bucket(&bucket).key(&key).send()
|
||||
})
|
||||
.await??;
|
||||
if head.content_length() != Some(body.len() as i64) {
|
||||
return Err(format!("HEAD returned the wrong size for {key}: {head:?}").into());
|
||||
}
|
||||
assert_object_bytes(reader, &bucket, &key, &body).await?;
|
||||
assert_workload_bytes(reader, &bucket, &key, &body).await?;
|
||||
|
||||
copier
|
||||
.copy_object()
|
||||
.bucket(&bucket)
|
||||
.key(©_key)
|
||||
.copy_source(format!("{bucket}/{key}"))
|
||||
.send()
|
||||
.await?;
|
||||
assert_object_bytes(writer, &bucket, ©_key, &body).await?;
|
||||
after_lock_contention(&format!("{context} COPY retained"), || {
|
||||
copier
|
||||
.copy_object()
|
||||
.bucket(&bucket)
|
||||
.key(©_key)
|
||||
.copy_source(format!("{bucket}/{key}"))
|
||||
.send()
|
||||
})
|
||||
.await??;
|
||||
assert_workload_bytes(writer, &bucket, ©_key, &body).await?;
|
||||
|
||||
writer.delete_object().bucket(&bucket).key(&key).send().await?;
|
||||
let missing = reader
|
||||
.head_object()
|
||||
.bucket(&bucket)
|
||||
.key(&key)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("deleted source key must not remain visible");
|
||||
after_lock_contention(&format!("{context} DELETE source"), || {
|
||||
writer.delete_object().bucket(&bucket).key(&key).send()
|
||||
})
|
||||
.await??;
|
||||
let missing = after_lock_contention(&format!("{context} HEAD deleted source"), || {
|
||||
reader.head_object().bucket(&bucket).key(&key).send()
|
||||
})
|
||||
.await?
|
||||
.expect_err("deleted source key must not remain visible");
|
||||
if missing.raw_response().map(|response| response.status().as_u16()) != Some(404) {
|
||||
return Err(format!("deleted source {key} returned an unexpected result: {missing:?}").into());
|
||||
}
|
||||
@@ -82,14 +337,22 @@ async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node()
|
||||
for handle in handles {
|
||||
inventory.extend(handle.await??);
|
||||
}
|
||||
assert_eq!(
|
||||
inventory.len(),
|
||||
worker_count * rounds,
|
||||
"every worker round must retain its acknowledged copy"
|
||||
);
|
||||
|
||||
let expected_keys: BTreeSet<_> = inventory.iter().map(|(key, _)| key.as_str()).collect();
|
||||
for (node_idx, client) in clients.iter().enumerate() {
|
||||
let listed = client.list_objects_v2().bucket(&bucket).prefix("retained/").send().await?;
|
||||
let listed = after_lock_contention(&format!("node={node_idx} LIST retained"), || {
|
||||
client.list_objects_v2().bucket(&bucket).prefix("retained/").send()
|
||||
})
|
||||
.await??;
|
||||
let listed_keys: BTreeSet<_> = listed.contents().iter().filter_map(|object| object.key()).collect();
|
||||
assert_eq!(listed_keys, expected_keys, "node {node_idx} returned a divergent retained-key listing");
|
||||
for (key, body) in &inventory {
|
||||
assert_object_bytes(client, &bucket, key, body)
|
||||
assert_workload_bytes(client, &bucket, key, body)
|
||||
.await
|
||||
.map_err(|error| format!("node {node_idx} failed to read {key}: {error}"))?;
|
||||
}
|
||||
|
||||
@@ -39,6 +39,8 @@ mod s3_during_data_movement_test;
|
||||
mod scanner_heal_topology_fixture;
|
||||
mod scanner_heal_topology_test;
|
||||
mod scanner_heal_topology_tier_test;
|
||||
mod scanner_peer_bucket_fixture;
|
||||
mod scanner_peer_bucket_test;
|
||||
mod site_replication_test;
|
||||
mod table_catalog_test;
|
||||
mod upgrade_test;
|
||||
|
||||
@@ -18,7 +18,7 @@ use super::harness::{
|
||||
};
|
||||
use crate::common::{ClusterTopology, RustFSTestClusterEnvironment};
|
||||
use crate::rpc_fault_proxy::{RpcFaultProxy, RpcPause};
|
||||
use crate::storage_api::multipart_fence_acceptance::RUSTFS_META_MULTIPART_BUCKET;
|
||||
use crate::storage_api::multipart_fence_acceptance::{LOCK_DIAGNOSTICS_FILTER, RUSTFS_META_MULTIPART_BUCKET};
|
||||
use aws_sdk_s3::{
|
||||
Client,
|
||||
config::retry::RetryConfig,
|
||||
@@ -38,14 +38,16 @@ const DEADLINE: Duration = Duration::from_secs(45);
|
||||
async fn fixture(
|
||||
drives_per_node: usize,
|
||||
storage_class: &str,
|
||||
lock_diagnostics: bool,
|
||||
) -> TestResult<(RustFSTestClusterEnvironment, Vec<RpcFaultProxy>, Vec<Client>, String)> {
|
||||
fixture_from_binary(drives_per_node, storage_class, None).await
|
||||
fixture_from_binary(drives_per_node, storage_class, None, lock_diagnostics).await
|
||||
}
|
||||
|
||||
async fn fixture_from_binary(
|
||||
drives_per_node: usize,
|
||||
storage_class: &str,
|
||||
binary: Option<&std::path::Path>,
|
||||
lock_diagnostics: bool,
|
||||
) -> TestResult<(RustFSTestClusterEnvironment, Vec<RpcFaultProxy>, Vec<Client>, String)> {
|
||||
let mut cluster =
|
||||
RustFSTestClusterEnvironment::with_topology(ClusterTopology::single_pool_multidrive(4, drives_per_node)).await?;
|
||||
@@ -90,6 +92,10 @@ async fn fixture_from_binary(
|
||||
] {
|
||||
cluster.set_env(name, value);
|
||||
}
|
||||
if lock_diagnostics {
|
||||
cluster.set_env("RUST_LOG", LOCK_DIAGNOSTICS_FILTER);
|
||||
cluster.set_env("RUSTFS_OBJECT_LOCK_DIAG_ENABLE", "true");
|
||||
}
|
||||
let mut proxies = Vec::new();
|
||||
if binary.is_none() {
|
||||
for node in 0..4 {
|
||||
@@ -363,26 +369,120 @@ fn upload_path_scope(bucket: &str, key: &str, upload_id: &str) -> TestResult<Str
|
||||
))
|
||||
}
|
||||
|
||||
async fn with_paused_peers<T: Send + 'static>(
|
||||
async fn with_paused_peers<T, E>(
|
||||
proxies: &[RpcFaultProxy],
|
||||
methods: &[&'static str],
|
||||
limits: &[usize],
|
||||
scope: &str,
|
||||
request: impl std::future::Future<Output = T> + Send + 'static,
|
||||
) -> TestResult<T> {
|
||||
request: impl std::future::Future<Output = Result<T, SdkError<E>>> + Send + 'static,
|
||||
) -> TestResult<Result<T, SdkError<E>>>
|
||||
where
|
||||
T: Send + 'static,
|
||||
E: std::error::Error + ProvideErrorMetadata + Send + Sync + 'static,
|
||||
{
|
||||
let deadline = tokio::time::Instant::now() + DEADLINE;
|
||||
let pauses: Vec<_> = proxies[1..=limits.len()]
|
||||
.iter()
|
||||
.zip(limits)
|
||||
.map(|(proxy, limit)| proxy.pause(methods, *limit, Some(scope)))
|
||||
.collect::<Result<_, _>>()?;
|
||||
let task = tokio::spawn(request);
|
||||
wait_for_faults(&pauses, methods).await?;
|
||||
let result = tokio::time::timeout(DEADLINE, task).await??;
|
||||
let mut task = tokio::spawn(request);
|
||||
let mut joined = None;
|
||||
let mut deadline_expired = false;
|
||||
let mut barrier_result = {
|
||||
let faults = wait_for_faults(&pauses, methods);
|
||||
tokio::pin!(faults);
|
||||
match tokio::time::timeout_at(deadline, async {
|
||||
tokio::select! {
|
||||
result = &mut faults => result,
|
||||
result = &mut task => {
|
||||
joined = Some(result);
|
||||
// A ready request is not evidence that every selected fault
|
||||
// was reached. Poll the original barrier once for that proof.
|
||||
match futures::poll!(faults.as_mut()) {
|
||||
std::task::Poll::Ready(result) => result,
|
||||
std::task::Poll::Pending => Err("SDK request completed before the selected RPC fault barrier".into()),
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
{
|
||||
Ok(result) => result,
|
||||
Err(error) => {
|
||||
deadline_expired = true;
|
||||
Err(std::io::Error::new(std::io::ErrorKind::TimedOut, error).into())
|
||||
}
|
||||
}
|
||||
};
|
||||
// Tokio's timeout polls an already-ready inner future before its timer.
|
||||
// A late observation cannot certify the original absolute deadline.
|
||||
deadline_expired |= tokio::time::Instant::now() >= deadline;
|
||||
if deadline_expired {
|
||||
barrier_result = Err(std::io::Error::new(
|
||||
std::io::ErrorKind::TimedOut,
|
||||
"selected RPC fault proof observed after the absolute MPU deadline",
|
||||
)
|
||||
.into());
|
||||
}
|
||||
if barrier_result.is_err() {
|
||||
for pause in &pauses {
|
||||
pause.discard_pending();
|
||||
}
|
||||
}
|
||||
let mut client_task_aborted = false;
|
||||
let request_result = match joined {
|
||||
Some(result) => result,
|
||||
None => match tokio::time::timeout_at(deadline, &mut task).await {
|
||||
Ok(result) => result,
|
||||
Err(error) => {
|
||||
deadline_expired = true;
|
||||
for pause in &pauses {
|
||||
pause.discard_pending();
|
||||
}
|
||||
barrier_result = Err(std::io::Error::new(std::io::ErrorKind::TimedOut, error).into());
|
||||
// This owns only the test's SDK request, not server/native work.
|
||||
// Retire the client task without granting another operation budget.
|
||||
client_task_aborted = true;
|
||||
task.abort();
|
||||
task.await
|
||||
}
|
||||
},
|
||||
};
|
||||
deadline_expired |= tokio::time::Instant::now() >= deadline;
|
||||
if deadline_expired {
|
||||
barrier_result = Err(std::io::Error::new(
|
||||
std::io::ErrorKind::TimedOut,
|
||||
"SDK request result observed after the absolute MPU deadline",
|
||||
)
|
||||
.into());
|
||||
}
|
||||
let hits: Vec<_> = pauses.iter().map(RpcPause::hits).collect();
|
||||
for pause in &pauses {
|
||||
pause.discard_pending();
|
||||
}
|
||||
drop(pauses);
|
||||
Ok(result)
|
||||
if barrier_result.is_err() {
|
||||
let sdk_error = request_result.as_ref().ok().and_then(|result| result.as_ref().err());
|
||||
let response = sdk_error.and_then(SdkError::raw_response);
|
||||
eprintln!(
|
||||
"MPU fault barrier failed: methods={methods:?}, hits={hits:?}, client_task_aborted={client_task_aborted}, deadline_expired={deadline_expired}, sdk_code={:?}, cause={:?}, request_id={:?}",
|
||||
sdk_error
|
||||
.and_then(|error| error.as_service_error())
|
||||
.and_then(|error| error.meta().code()),
|
||||
response.and_then(|response| response.headers().get("x-rustfs-error-cause")),
|
||||
response.and_then(|response| response.headers().get("x-amz-request-id")),
|
||||
);
|
||||
}
|
||||
match (barrier_result, request_result) {
|
||||
// Preserve the actual SDK source instead of replacing an early service
|
||||
// failure with an unrelated fault-wait timeout. The outer Err stays red.
|
||||
(Err(_), Ok(Err(error))) => Err(Box::new(error)),
|
||||
(Err(_), Err(error)) if !client_task_aborted => Err(Box::new(error)),
|
||||
(Err(error), _) => Err(error),
|
||||
(Ok(()), Ok(result)) => Ok(result),
|
||||
(Ok(()), Err(error)) => Err(Box::new(error)),
|
||||
}
|
||||
}
|
||||
|
||||
fn quorum_outcome<T: std::fmt::Debug, E: std::fmt::Debug + ProvideErrorMetadata>(
|
||||
@@ -1318,7 +1418,7 @@ async fn concurrent_capped_parts_respect_limit_on_distinct_nodes(clients: &[Clie
|
||||
|
||||
#[tokio::test]
|
||||
async fn metadata_delay_preserves_write_quorum_and_rejects_quorum_minus_one() -> TestResult {
|
||||
let (cluster, proxies, clients, bucket) = fixture(1, "EC:2").await?;
|
||||
let (cluster, proxies, clients, bucket) = fixture(1, "EC:2", true).await?;
|
||||
list_parts_disk_loss_after_metadata_read(&cluster, &proxies, &clients, &bucket).await?;
|
||||
concurrent_capped_parts_respect_limit_on_distinct_nodes(&clients, &bucket).await?;
|
||||
phase_quorum_matrix(&cluster, &proxies, &clients, &bucket, false, 1, 3).await
|
||||
@@ -1326,13 +1426,13 @@ async fn metadata_delay_preserves_write_quorum_and_rejects_quorum_minus_one() ->
|
||||
|
||||
#[tokio::test]
|
||||
async fn lock_delay_preserves_admission_quorum_and_reports_timeout() -> TestResult {
|
||||
let (cluster, proxies, clients, bucket) = fixture(1, "EC:2").await?;
|
||||
let (cluster, proxies, clients, bucket) = fixture(1, "EC:2", false).await?;
|
||||
phase_quorum_matrix(&cluster, &proxies, &clients, &bucket, true, 1, 3).await
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn sixteen_shard_multipart_metadata_requires_twelve_votes() -> TestResult {
|
||||
let (cluster, proxies, clients, bucket) = fixture(4, "EC:4").await?;
|
||||
let (cluster, proxies, clients, bucket) = fixture(4, "EC:4", false).await?;
|
||||
let storage = cluster_admin_ok(&cluster, http::Method::GET, "/rustfs/admin/v3/storageinfo", None).await?;
|
||||
let storage: serde_json::Value = serde_json::from_str(&storage)?;
|
||||
let storage: rustfs_madmin::StorageInfo = serde_json::from_value(storage["info"].clone())?;
|
||||
@@ -1346,7 +1446,7 @@ async fn sixteen_shard_multipart_metadata_requires_twelve_votes() -> TestResult
|
||||
#[tokio::test]
|
||||
async fn mixed_version_failed_complete_requires_current_coordinator_for_abort() -> TestResult {
|
||||
let previous = super::upgrade_test::source_binary()?;
|
||||
let (mut cluster, mut proxies, clients, bucket) = fixture_from_binary(4, "EC:4", Some(&previous)).await?;
|
||||
let (mut cluster, mut proxies, clients, bucket) = fixture_from_binary(4, "EC:4", Some(&previous), false).await?;
|
||||
assert!(
|
||||
proxies.is_empty(),
|
||||
"the genuine previous fleet boots and seeds through its normal direct transport"
|
||||
@@ -1549,7 +1649,7 @@ async fn native_certified_partial_abort_retry_reclaims_survivors_below_metadata_
|
||||
capability_sha256: Option<[u8; 32]>,
|
||||
abort: Option<AbortRef>,
|
||||
}
|
||||
let (mut cluster, proxies, clients, bucket) = fixture(4, "EC:4").await?;
|
||||
let (mut cluster, proxies, clients, bucket) = fixture(4, "EC:4", false).await?;
|
||||
let key = "partial-abort";
|
||||
let committed = payload_for("committed-before-abort", 4096);
|
||||
clients[0]
|
||||
@@ -1765,7 +1865,7 @@ async fn native_certified_partial_abort_retry_reclaims_survivors_below_metadata_
|
||||
#[tokio::test]
|
||||
async fn late_complete_packets_cannot_publish_after_abort_and_bucket_recreation() -> TestResult {
|
||||
for size in [0, 8 * 1024 * 1024] {
|
||||
let (cluster, proxies, clients, bucket) = fixture(1, "EC:2").await?;
|
||||
let (cluster, proxies, clients, bucket) = fixture(1, "EC:2", false).await?;
|
||||
let key = "late-complete";
|
||||
let upload = clients[0].create_multipart_upload().bucket(&bucket).key(key).send().await?;
|
||||
let upload_id = upload.upload_id().ok_or("missing upload ID")?;
|
||||
@@ -1892,7 +1992,7 @@ async fn late_complete_packets_cannot_publish_after_abort_and_bucket_recreation(
|
||||
async fn late_part_packets_preserve_retired_and_fresh_uploads() -> TestResult {
|
||||
for (method, settles) in [("PreparePartTransaction", false), ("SettlePartTransaction", true)] {
|
||||
for recreate_bucket in [false, true] {
|
||||
let (cluster, proxies, clients, bucket) = fixture(1, "EC:2").await?;
|
||||
let (cluster, proxies, clients, bucket) = fixture(1, "EC:2", false).await?;
|
||||
let key = "late-part-transaction";
|
||||
let previous = payload_for("committed-before-late-part", 4096);
|
||||
clients[0]
|
||||
@@ -2075,7 +2175,7 @@ async fn late_part_packets_preserve_retired_and_fresh_uploads() -> TestResult {
|
||||
|
||||
#[tokio::test]
|
||||
async fn cancelled_metadata_wait_releases_fence_for_bucket_recreation() -> TestResult {
|
||||
let (_cluster, proxies, clients, bucket) = fixture(1, "EC:2").await?;
|
||||
let (_cluster, proxies, clients, bucket) = fixture(1, "EC:2", false).await?;
|
||||
let pauses: Vec<_> = proxies[1..]
|
||||
.iter()
|
||||
.map(|proxy| proxy.pause(&["WriteMetadata"], usize::MAX, Some(&bucket)))
|
||||
@@ -2098,7 +2198,7 @@ async fn cancelled_metadata_wait_releases_fence_for_bucket_recreation() -> TestR
|
||||
async fn part_resend_owner_serializes_complete_and_abort_after_client_cancellation() -> TestResult {
|
||||
use md5::{Digest as _, Md5};
|
||||
|
||||
let (cluster, proxies, clients, bucket) = fixture(1, "EC:2").await?;
|
||||
let (cluster, proxies, clients, bucket) = fixture(1, "EC:2", false).await?;
|
||||
for cancel_waiter in [false, true] {
|
||||
for complete in [false, true] {
|
||||
let key = format!("resend-{cancel_waiter}-{complete}");
|
||||
@@ -2586,7 +2686,7 @@ async fn resume_at_rpc<T: Send + 'static>(
|
||||
|
||||
#[tokio::test]
|
||||
async fn each_multipart_phase_resumes_after_metadata_and_lock_delays() -> TestResult {
|
||||
let (_cluster, proxies, clients, bucket) = fixture(1, "EC:2").await?;
|
||||
let (_cluster, proxies, clients, bucket) = fixture(1, "EC:2", false).await?;
|
||||
for locks in [false, true] {
|
||||
let key = format!("phase-{locks}");
|
||||
let client = clients[0].clone();
|
||||
|
||||
@@ -766,9 +766,10 @@ impl Run {
|
||||
|| async {
|
||||
let mut ready = true;
|
||||
let mut observations = Vec::with_capacity(4);
|
||||
for node in 0..4 {
|
||||
assert!(self.dist.cluster.nodes.len() >= 4);
|
||||
for (node, cluster_node) in self.dist.cluster.nodes.iter().enumerate().take(4) {
|
||||
match http
|
||||
.get(format!("{}/health/ready", self.dist.cluster.nodes[node].url))
|
||||
.get(format!("{}/health/ready", cluster_node.url))
|
||||
.timeout(Duration::from_secs(3))
|
||||
.send()
|
||||
.await
|
||||
|
||||
@@ -25,7 +25,8 @@
|
||||
|
||||
use super::harness::{
|
||||
DistCluster, DistLayout, TestResult, cluster_admin, cluster_admin_ok, decommission_status_json, get_object_bytes,
|
||||
payload_for, put_object, s3_probe_client, s3_probe_error_code, sha256_hex, unique_bucket, wait_until,
|
||||
payload_for, put_object, rebalance_complete, rebalance_status_json, s3_probe_client, s3_probe_error_code, sha256_hex,
|
||||
unique_bucket, wait_until,
|
||||
};
|
||||
use crate::chaos::{VersionShardCensus, census_object_version_on_disk};
|
||||
use crate::common::{FAST_DATA_USAGE_SCANNER_ENV, RustFSTestClusterEnvironment};
|
||||
@@ -47,6 +48,7 @@ use tokio::time::{Instant, sleep};
|
||||
use uuid::Uuid;
|
||||
|
||||
pub(crate) const MOVEMENT_BARRIER_DIR_ENV: &str = "RUSTFS_E2E_DATA_MOVEMENT_BARRIER_DIR";
|
||||
pub(crate) const STAGE_AFTER_ENTRY_ADMISSION: &str = "after_entry_admission";
|
||||
pub(crate) const STAGE_BEFORE_TARGET_COMMIT: &str = "before_target_commit";
|
||||
pub(crate) const STAGE_BEFORE_SOURCE_CLEANUP: &str = "before_source_cleanup";
|
||||
pub(crate) const STAGE_AFTER_RESPONSIBILITY_PERSISTED: &str = "after_responsibility_persisted";
|
||||
@@ -1166,8 +1168,21 @@ pub(crate) async fn clear_decommission(cluster: &RustFSTestClusterEnvironment, p
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn stop_rebalance(cluster: &RustFSTestClusterEnvironment) -> TestResult<String> {
|
||||
cluster_admin_ok(cluster, Method::POST, "/rustfs/admin/v3/rebalance/stop", None).await
|
||||
pub(crate) async fn stop_rebalance(cluster: &RustFSTestClusterEnvironment, expected_id: &str) -> TestResult<String> {
|
||||
let path = "/rustfs/admin/v3/rebalance/stop";
|
||||
let (status, response) = cluster_admin(cluster, Method::POST, path, None).await?;
|
||||
if status.is_success() {
|
||||
return Ok(response);
|
||||
}
|
||||
// The run can finish between the active observation and the stop request.
|
||||
// Only completion of that same run makes a not-started response acceptable.
|
||||
if status == StatusCode::NOT_FOUND
|
||||
&& response.contains("<Code>NoSuchResource</Code>")
|
||||
&& rebalance_complete(&rebalance_status_json(cluster).await?, expected_id)?
|
||||
{
|
||||
return Ok(response);
|
||||
}
|
||||
Err(format!("POST {path} failed: {status} {response}").into())
|
||||
}
|
||||
|
||||
pub(crate) fn decommission_pool_entry(status: &Value, pool_id: usize) -> TestResult<Value> {
|
||||
@@ -1585,6 +1600,129 @@ pub(crate) async fn assert_every_node_serves(dist: &DistCluster, bucket: &str, l
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
const REBALANCE_NOT_STARTED: &str =
|
||||
"<Error><Code>NoSuchResource</Code><Message>pool rebalance is not started</Message></Error>";
|
||||
|
||||
async fn mock_rebalance_stop(
|
||||
stop_status: StatusCode,
|
||||
stop_body: &str,
|
||||
run_status: Value,
|
||||
) -> TestResult<(TestResult<String>, Vec<String>)> {
|
||||
use bytes::Bytes;
|
||||
use http_body_util::Full;
|
||||
use hyper::service::service_fn;
|
||||
use hyper_util::rt::TokioIo;
|
||||
use tokio::net::TcpListener;
|
||||
use tokio::sync::mpsc;
|
||||
use tokio::task::JoinSet;
|
||||
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await?;
|
||||
let mut cluster = RustFSTestClusterEnvironment::new(1).await?;
|
||||
cluster.nodes[0].url = format!("http://{}", listener.local_addr()?);
|
||||
let stop_body = stop_body.to_string();
|
||||
let (requests_tx, mut requests_rx) = mpsc::unbounded_channel();
|
||||
let mut server = JoinSet::new();
|
||||
server.spawn(async move {
|
||||
loop {
|
||||
let (stream, _) = match listener.accept().await {
|
||||
Ok(connection) => connection,
|
||||
Err(error) => return Err::<(), Box<dyn std::error::Error + Send + Sync>>(error.into()),
|
||||
};
|
||||
let stop_body = stop_body.clone();
|
||||
let run_status = run_status.clone();
|
||||
let requests_tx = requests_tx.clone();
|
||||
let handler = service_fn(move |request: hyper::Request<hyper::body::Incoming>| {
|
||||
let route = format!("{} {}", request.method(), request.uri().path());
|
||||
requests_tx.send(route.clone()).expect("record mock admin request");
|
||||
let (status, body) = match route.as_str() {
|
||||
"POST /rustfs/admin/v3/rebalance/stop" => (stop_status, stop_body.clone()),
|
||||
"GET /rustfs/admin/v3/rebalance/status" => (StatusCode::OK, run_status.to_string()),
|
||||
_ => (StatusCode::BAD_REQUEST, "unexpected admin request".to_string()),
|
||||
};
|
||||
async move { http::Response::builder().status(status).body(Full::new(Bytes::from(body))) }
|
||||
});
|
||||
hyper::server::conn::http1::Builder::new()
|
||||
.keep_alive(false)
|
||||
.serve_connection(TokioIo::new(stream), handler)
|
||||
.await?;
|
||||
}
|
||||
});
|
||||
let outcome = tokio::time::timeout(Duration::from_secs(5), stop_rebalance(&cluster, "run-1")).await?;
|
||||
server.abort_all();
|
||||
let requests = std::iter::from_fn(|| requests_rx.try_recv().ok()).collect();
|
||||
Ok((outcome, requests))
|
||||
}
|
||||
|
||||
fn completed_rebalance() -> Value {
|
||||
serde_json::json!({
|
||||
"id": "run-1",
|
||||
"pools": [
|
||||
{ "id": 0, "status": "Completed", "progress": { "versions": 113 } },
|
||||
{ "id": 1, "status": "None" }
|
||||
]
|
||||
})
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rebalance_stop_success_does_not_query_completion() -> TestResult {
|
||||
let (outcome, requests) = mock_rebalance_stop(StatusCode::OK, "", Value::Null).await?;
|
||||
assert_eq!(outcome?, "");
|
||||
assert_eq!(requests, ["POST /rustfs/admin/v3/rebalance/stop"]);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rebalance_stop_accepts_completed_run_after_not_started_response() -> TestResult {
|
||||
let (outcome, requests) =
|
||||
mock_rebalance_stop(StatusCode::NOT_FOUND, REBALANCE_NOT_STARTED, completed_rebalance()).await?;
|
||||
outcome?;
|
||||
assert_eq!(
|
||||
requests,
|
||||
[
|
||||
"POST /rustfs/admin/v3/rebalance/stop",
|
||||
"GET /rustfs/admin/v3/rebalance/status"
|
||||
]
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rebalance_stop_rejects_unproven_completion_and_other_errors() -> TestResult {
|
||||
let complete = completed_rebalance();
|
||||
let mut invalid = vec![Value::Null];
|
||||
for (pointer, value) in [
|
||||
("/id", serde_json::json!("run-2")),
|
||||
("/pools/0/status", serde_json::json!("Started")),
|
||||
("/pools/0/status", serde_json::json!("Failed")),
|
||||
("/pools/0/status", serde_json::json!("Stopped")),
|
||||
("/pools/0/progress/versions", serde_json::json!(0)),
|
||||
] {
|
||||
let mut status = complete.clone();
|
||||
*status.pointer_mut(pointer).expect("completion fixture field") = value;
|
||||
invalid.push(status);
|
||||
}
|
||||
let mut pending = complete.clone();
|
||||
pending["stopPropagation"] = serde_json::json!({ "pendingTerminalReload": true });
|
||||
invalid.push(pending);
|
||||
let mut failed = complete.clone();
|
||||
failed["pools"][0]["lastError"] = serde_json::json!("migration failed");
|
||||
invalid.push(failed);
|
||||
for status in invalid {
|
||||
let (outcome, _) = mock_rebalance_stop(StatusCode::NOT_FOUND, REBALANCE_NOT_STARTED, status.clone()).await?;
|
||||
assert!(outcome.is_err(), "unproven completion accepted: {status}");
|
||||
}
|
||||
for (code, body) in [
|
||||
(StatusCode::FORBIDDEN, REBALANCE_NOT_STARTED),
|
||||
(StatusCode::INTERNAL_SERVER_ERROR, REBALANCE_NOT_STARTED),
|
||||
(StatusCode::NOT_FOUND, "<Error><Code>NoSuchBucket</Code></Error>"),
|
||||
] {
|
||||
let (outcome, requests) = mock_rebalance_stop(code, body, complete.clone()).await?;
|
||||
assert!(outcome.is_err(), "unrelated error accepted: {code} {body}");
|
||||
assert_eq!(requests, ["POST /rustfs/admin/v3/rebalance/stop"]);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn version_ids_normalize_null_spellings() {
|
||||
for spelling in [None, Some(""), Some("null"), Some("00000000-0000-0000-0000-000000000000")] {
|
||||
|
||||
@@ -28,7 +28,7 @@ use super::harness::{
|
||||
wait_for_rebalance_active, wait_for_rebalance_complete,
|
||||
};
|
||||
use super::scanner_heal_topology_fixture::{
|
||||
LiveWorkload, MOVEMENT_ACTIVE_TIMEOUT, MOVEMENT_COMPLETE_TIMEOUT, MovementOperation, SEED_POOL,
|
||||
LiveWorkload, MOVEMENT_ACTIVE_TIMEOUT, MOVEMENT_COMPLETE_TIMEOUT, MovementOperation, SEED_POOL, STAGE_AFTER_ENTRY_ADMISSION,
|
||||
STAGE_AFTER_RESPONSIBILITY_PERSISTED, STAGE_BEFORE_SOURCE_CLEANUP, STAGE_BEFORE_TARGET_COMMIT, TopologyManifest, TopologyRun,
|
||||
assert_bucket_durability_strict, assert_every_node_serves, assert_probe_objects, assert_process_durability_strict,
|
||||
assert_seed_pool_census_unchanged, assert_single_pool_placement, cancel_decommission, capture_seed_pool_census,
|
||||
@@ -274,6 +274,7 @@ async fn wait_for_rebalance_stopped_or_completed(run: &TopologyRun, rebalance_id
|
||||
#[tokio::test]
|
||||
async fn scanner_heal_topology_rebalance_survives_live_writes_stop_and_restart() -> TestResult {
|
||||
init_logging();
|
||||
require_test_hooks();
|
||||
let mut run = TopologyRun::start("rebalance").await?;
|
||||
let manifest = seed_manifest(&run).await?;
|
||||
assert_bucket_durability_strict(&run.dist.cluster, &manifest.buckets()).await?;
|
||||
@@ -288,23 +289,52 @@ async fn scanner_heal_topology_rebalance_survives_live_writes_stop_and_restart()
|
||||
.map(|entry| (entry.key.clone(), entry.sha256.clone()))
|
||||
.collect();
|
||||
|
||||
let first_id = start_rebalance(&run.dist.cluster).await?;
|
||||
wait_for_rebalance_active(&run.dist.cluster, &first_id, MOVEMENT_ACTIVE_TIMEOUT).await?;
|
||||
let during_rebalance = put_probe_objects(&run.dist, &scratch, "during-rebalance", 4).await?;
|
||||
let workload = tokio::spawn(run_live_workload(LiveWorkload {
|
||||
clients: run.dist.clients()?,
|
||||
bucket: scratch.clone(),
|
||||
prefix: "live".to_string(),
|
||||
writers: LIVE_WRITERS,
|
||||
rounds: LIVE_ROUNDS,
|
||||
baseline_bucket: manifest.plain_bucket.clone(),
|
||||
baseline: baseline.clone(),
|
||||
pause: Duration::from_millis(100),
|
||||
}));
|
||||
sleep(Duration::from_secs(2)).await;
|
||||
stop_rebalance(&run.dist.cluster).await?;
|
||||
let (first_outcome, first_status) = wait_for_rebalance_stopped_or_completed(&run, &first_id).await?;
|
||||
let ops = workload.await??;
|
||||
// Keep an admitted entry on the source until stop cancels admission. Start
|
||||
// propagation may await that entry, so observe and stop before awaiting ACK.
|
||||
let first_key = manifest.plain_key("external", 0)?;
|
||||
run.movement_barrier
|
||||
.arm_movement("rebalance", &manifest.plain_bucket, &first_key, STAGE_AFTER_ENTRY_ADMISSION)?;
|
||||
let observe_first = async {
|
||||
let (observed_id, reached) = tokio::time::timeout(MOVEMENT_ACTIVE_TIMEOUT, async {
|
||||
let reached = run.movement_barrier.wait_reached(MOVEMENT_ACTIVE_TIMEOUT).await?;
|
||||
let status = rebalance_status_json(&run.dist.cluster).await?;
|
||||
let observed_id = status
|
||||
.get("id")
|
||||
.and_then(Value::as_str)
|
||||
.ok_or("rebalance status omitted id")?;
|
||||
wait_for_rebalance_active(&run.dist.cluster, observed_id, MOVEMENT_ACTIVE_TIMEOUT).await?;
|
||||
Ok::<_, Box<dyn std::error::Error + Send + Sync>>((observed_id.to_string(), reached))
|
||||
})
|
||||
.await??;
|
||||
let probes = put_probe_objects(&run.dist, &scratch, "during-rebalance", 4).await?;
|
||||
let workload = run_live_workload(LiveWorkload {
|
||||
clients: run.dist.clients()?,
|
||||
bucket: scratch.clone(),
|
||||
prefix: "live".to_string(),
|
||||
writers: LIVE_WRITERS,
|
||||
rounds: LIVE_ROUNDS,
|
||||
baseline_bucket: manifest.plain_bucket.clone(),
|
||||
baseline: baseline.clone(),
|
||||
pause: Duration::from_millis(100),
|
||||
});
|
||||
let stop = async {
|
||||
sleep(Duration::from_secs(2)).await;
|
||||
stop_rebalance(&run.dist.cluster, &observed_id).await?;
|
||||
let (outcome, status) = wait_for_rebalance_stopped_or_completed(&run, &observed_id).await?;
|
||||
if outcome != "stopped" {
|
||||
return Err("rebalance completed before stop; mid-run stop was not exercised".into());
|
||||
}
|
||||
Ok::<_, Box<dyn std::error::Error + Send + Sync>>((outcome, status))
|
||||
};
|
||||
let (ops, (outcome, status)) = tokio::try_join!(workload, stop)?;
|
||||
Ok::<_, Box<dyn std::error::Error + Send + Sync>>((observed_id, reached, probes, ops, outcome, status))
|
||||
};
|
||||
let (first_id, (observed_id, first_reached, during_rebalance, ops, first_outcome, first_status)) =
|
||||
tokio::try_join!(start_rebalance(&run.dist.cluster), observe_first)?;
|
||||
if first_id != observed_id {
|
||||
return Err(format!("rebalance start acknowledged {first_id}, but the stopped run was {observed_id}").into());
|
||||
}
|
||||
run.movement_barrier.disarm()?;
|
||||
let live_state = expected_live_state(&ops);
|
||||
|
||||
for node_idx in 0..run.dist.cluster.nodes.len() {
|
||||
@@ -333,8 +363,40 @@ async fn scanner_heal_topology_rebalance_survives_live_writes_stop_and_restart()
|
||||
verify_from_every_node(&run.dist, &manifest, "after restart following the stop").await?;
|
||||
verify_live_state(&run.dist.client(0)?, &scratch, &live_state, "after restart following the stop").await?;
|
||||
|
||||
let second_id = start_rebalance(&run.dist.cluster).await?;
|
||||
wait_for_rebalance_active(&run.dist.cluster, &second_id, MOVEMENT_ACTIVE_TIMEOUT).await?;
|
||||
// A short second run can finish before the start response reaches us.
|
||||
// Hold one remaining source entry until its active state is observed.
|
||||
let mut second_key = None;
|
||||
for entry in manifest.plain_entries() {
|
||||
if pools_holding(&run.dist, &manifest.plain_bucket, &entry.key, None)?.contains(&SEED_POOL) {
|
||||
second_key = Some(entry.key.clone());
|
||||
break;
|
||||
}
|
||||
}
|
||||
let second_key = second_key.ok_or("no seed-pool object remains for the second rebalance")?;
|
||||
let second_barrier = run.movement_barrier.next_phase();
|
||||
second_barrier.arm_movement("rebalance", &manifest.plain_bucket, &second_key, STAGE_BEFORE_SOURCE_CLEANUP)?;
|
||||
let observe_second = async {
|
||||
let reached = second_barrier.wait_reached(MOVEMENT_ACTIVE_TIMEOUT).await?;
|
||||
let status = rebalance_status_json(&run.dist.cluster).await?;
|
||||
let observed_id = status
|
||||
.get("id")
|
||||
.and_then(Value::as_str)
|
||||
.ok_or("rebalance status omitted id")?;
|
||||
if observed_id == first_id {
|
||||
return Err("the second rebalance reused the stopped run id".into());
|
||||
}
|
||||
wait_for_rebalance_active(&run.dist.cluster, observed_id, MOVEMENT_ACTIVE_TIMEOUT).await?;
|
||||
second_barrier.disarm()?;
|
||||
second_barrier.release_movement()?;
|
||||
Ok::<_, Box<dyn std::error::Error + Send + Sync>>((observed_id.to_string(), reached))
|
||||
};
|
||||
// Worker propagation can hold the start response until movement drains.
|
||||
let (second_id, (observed_id, second_reached)) = tokio::try_join!(start_rebalance(&run.dist.cluster), async {
|
||||
tokio::time::timeout(MOVEMENT_ACTIVE_TIMEOUT, observe_second).await?
|
||||
})?;
|
||||
if second_id != observed_id {
|
||||
return Err(format!("rebalance start acknowledged {second_id}, but the active run was {observed_id}").into());
|
||||
}
|
||||
wait_for_rebalance_complete(&run.dist.cluster, &second_id, MOVEMENT_COMPLETE_TIMEOUT).await?;
|
||||
verify_from_every_node(&run.dist, &manifest, "after the second rebalance completed").await?;
|
||||
verify_live_state(&run.dist.client(3)?, &scratch, &live_state, "after the second rebalance completed").await?;
|
||||
@@ -351,8 +413,8 @@ async fn scanner_heal_topology_rebalance_survives_live_writes_stop_and_restart()
|
||||
&json!({
|
||||
"gate": "G-REB",
|
||||
"manifest": manifest_summary(&manifest),
|
||||
"first_run": { "id": first_id, "outcome": first_outcome, "status": first_status, "versions_held_by_two_pools_after_stop": retained_after_stop },
|
||||
"second_run": { "id": second_id, "versions_off_seed_pool": moved },
|
||||
"first_run": { "id": first_id, "outcome": first_outcome, "status": first_status, "versions_held_by_two_pools_after_stop": retained_after_stop, "barrier_object": first_key, "barrier": first_reached },
|
||||
"second_run": { "id": second_id, "versions_off_seed_pool": moved, "barrier_object": second_key, "barrier": second_reached },
|
||||
"live_ops": ops,
|
||||
"probe_placement_during_rebalance": during_placement,
|
||||
"process_durability_strict_nodes": strict_nodes,
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1332,14 +1332,15 @@ mod tests {
|
||||
.collect();
|
||||
assert_eq!(confirmed.len(), 1);
|
||||
let mut receivers = Vec::new();
|
||||
for drive in 0..phase.disks.get(1).map_or(0, Vec::len) {
|
||||
let receiver_endpoints = phase.disks.get(1).map_or(&[][..], |disks| &phase.endpoints[1][..disks.len()]);
|
||||
for endpoint in receiver_endpoints {
|
||||
for cas in [prepares[1], commits[1]] {
|
||||
let matching: Vec<_> = events[1]
|
||||
.iter()
|
||||
.copied()
|
||||
.filter(|event| {
|
||||
event["kind"] == "receiver"
|
||||
&& event["disk"] == phase.endpoints[1][drive]
|
||||
&& event["disk"] == *endpoint
|
||||
&& event["dst_volume"] == ".rustfs.sys"
|
||||
&& event["dst_path"] == "pool.bin"
|
||||
&& event["etag"] == cas["etag"]
|
||||
|
||||
@@ -33,7 +33,7 @@ mod tests {
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::collections::HashSet;
|
||||
use std::error::Error;
|
||||
use std::io::{Read, Write};
|
||||
use std::io::{Read, Seek, SeekFrom, Write};
|
||||
use std::net::SocketAddr;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::Command;
|
||||
@@ -1654,6 +1654,89 @@ mod tests {
|
||||
TargetEndpointBlackhole,
|
||||
}
|
||||
|
||||
async fn release_heal_commit_barrier_after_shutdown(
|
||||
log: &Path,
|
||||
log_offset: u64,
|
||||
barrier: &Path,
|
||||
) -> Result<serde_json::Value, Box<dyn Error + Send + Sync>> {
|
||||
// Observe within the existing graceful process-stop budget.
|
||||
timeout(Duration::from_secs(45), async {
|
||||
loop {
|
||||
let mut file = std::fs::File::open(log)?;
|
||||
file.seek(SeekFrom::Start(log_offset))?;
|
||||
let mut tail = String::new();
|
||||
file.read_to_string(&mut tail)?;
|
||||
if let Some(event) = tail
|
||||
.lines()
|
||||
.filter_map(|line| serde_json::from_str::<serde_json::Value>(line).ok())
|
||||
.find(|event| {
|
||||
event["event"] == "server_shutdown_state"
|
||||
&& event["component"] == "main"
|
||||
&& event["subsystem"] == "startup"
|
||||
&& event["state"] == "stopping"
|
||||
})
|
||||
{
|
||||
// Local heal has stopped and HTTP shutdown has been signalled.
|
||||
// Let admitted peer RPCs finish; keeping this synthetic barrier
|
||||
// until process exit would make the fixture block its own drain.
|
||||
std::fs::remove_file(barrier)?;
|
||||
return Ok(event);
|
||||
}
|
||||
sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
})
|
||||
.await?
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn heal_commit_barrier_waits_for_new_shutdown_event() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
let directory = tempfile::tempdir()?;
|
||||
let log = directory.path().join("server.log");
|
||||
let barrier = directory.path().join("barrier");
|
||||
let event = serde_json::json!({
|
||||
"event": "server_shutdown_state", "component": "main", "subsystem": "startup", "state": "stopping"
|
||||
});
|
||||
std::fs::write(&log, format!("{event}\n"))?;
|
||||
std::fs::write(&barrier, "bucket/prefix/")?;
|
||||
let log_offset = log.metadata()?.len();
|
||||
let mut writer = std::fs::OpenOptions::new().append(true).open(&log)?;
|
||||
writeln!(writer, "{}", serde_json::json!({"event": "shutdown_signal_received"}))?;
|
||||
writeln!(writer, "{}", serde_json::json!({"event": "server_shutdown_state", "state": "stopping"}))?;
|
||||
|
||||
let release = release_heal_commit_barrier_after_shutdown(&log, log_offset, &barrier);
|
||||
tokio::pin!(release);
|
||||
assert!(
|
||||
futures::poll!(release.as_mut()).is_pending(),
|
||||
"old or unrelated events must not release the barrier"
|
||||
);
|
||||
assert!(barrier.is_file(), "the partial-rebuild barrier must remain armed before shutdown");
|
||||
tokio::time::advance(Duration::from_secs(6)).await;
|
||||
assert!(
|
||||
futures::poll!(release.as_mut()).is_pending(),
|
||||
"background shutdown may take longer than five seconds within the process-stop budget"
|
||||
);
|
||||
writeln!(writer, "{event}")?;
|
||||
tokio::time::advance(Duration::from_millis(10)).await;
|
||||
assert_eq!(release.await?, event, "retain the actual new shutdown witness");
|
||||
assert!(!barrier.exists(), "shutdown must release the synthetic hold");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn heal_commit_barrier_missing_shutdown_fails_without_releasing() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
let directory = tempfile::tempdir()?;
|
||||
let log = directory.path().join("server.log");
|
||||
let barrier = directory.path().join("barrier");
|
||||
std::fs::write(&log, "")?;
|
||||
std::fs::write(&barrier, "bucket/prefix/")?;
|
||||
let error = release_heal_commit_barrier_after_shutdown(&log, 0, &barrier)
|
||||
.await
|
||||
.expect_err("missing shutdown evidence must fail the fixture");
|
||||
assert!(error.is::<tokio::time::error::Elapsed>());
|
||||
assert!(barrier.is_file(), "a missing shutdown witness must not release repairs");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn run_cluster_root_heal_interruption(scenario: InterruptionScenario) -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
let server_binary = rustfs_binary_path();
|
||||
let evidence_case = match scenario {
|
||||
@@ -1700,6 +1783,15 @@ mod tests {
|
||||
);
|
||||
|
||||
let mut cluster = RustFSTestClusterEnvironment::with_topology(topology.cluster_topology()).await?;
|
||||
let bucket = "heal-restart-during-rebuild";
|
||||
let mrf_fallback_scope = if !background_enabled {
|
||||
let path = PathBuf::from(&cluster.temp_dir).join("skip-outage-mrf-fallback");
|
||||
std::fs::write(&path, format!("{bucket}/cluster/written-while-node-down-"))?;
|
||||
cluster.set_env("RUSTFS_E2E_SKIP_MRF_FALLBACK_PATH", path.to_string_lossy().into_owned());
|
||||
Some(path)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
cluster.set_env("RUSTFS_UNSAFE_BYPASS_DISK_CHECK", "true");
|
||||
cluster.set_env("RUSTFS_HEAL_ENABLED", "true");
|
||||
if let Some(storage_class) = evidence_case.and_then(|case| case.storage_class_standard) {
|
||||
@@ -1782,6 +1874,7 @@ mod tests {
|
||||
let mut created_online_objects = Vec::with_capacity(online_object_count);
|
||||
let mut expected_manifests = Vec::with_capacity(online_object_count);
|
||||
let mut unclean_shutdown_marker_observed = None;
|
||||
let mut shutdown_barrier_release = None;
|
||||
let mut attempt_count = 0usize;
|
||||
let max_online_attempts = online_object_count.saturating_mul(topology.total_drives().max(1));
|
||||
while expected_manifests.len() < online_object_count {
|
||||
@@ -1952,6 +2045,21 @@ mod tests {
|
||||
}
|
||||
};
|
||||
|
||||
if let Some(path) = &mrf_fallback_scope {
|
||||
let observed = path.with_extension("observed");
|
||||
let expected = format!("{bucket}/{outage_key}");
|
||||
let deadline = Instant::now() + Duration::from_secs(30);
|
||||
loop {
|
||||
if tokio::fs::read_to_string(&observed).await.ok().as_deref() == Some(expected.as_str()) {
|
||||
break;
|
||||
}
|
||||
if Instant::now() >= deadline {
|
||||
return Err("outage repair did not reach the isolated best-effort fallback before target rejoin".into());
|
||||
}
|
||||
sleep(Duration::from_millis(50)).await;
|
||||
}
|
||||
}
|
||||
|
||||
assert!(
|
||||
!outage_target_manifest_required
|
||||
|| target_set_peer_drives
|
||||
@@ -2045,7 +2153,12 @@ mod tests {
|
||||
} else {
|
||||
cluster_heal_is_idle(&recovered)
|
||||
};
|
||||
if ready {
|
||||
let pool_ready = if ready && background_enabled && !outage_write_deferred_until_rejoin {
|
||||
all_nodes_pool_metadata_readiness(&cluster, recovery_deadline).await.0
|
||||
} else {
|
||||
true
|
||||
};
|
||||
if ready && pool_ready {
|
||||
break;
|
||||
}
|
||||
if Instant::now() >= recovery_deadline {
|
||||
@@ -2344,7 +2457,21 @@ mod tests {
|
||||
| InterruptionScenario::BackgroundTargetRestartEc84MultiSet
|
||||
| InterruptionScenario::BackgroundCoordinatorRestart
|
||||
) {
|
||||
cluster.stop_node_gracefully(interruption_node).await?;
|
||||
if interruption_node == 1
|
||||
&& let Some(barrier) = &commit_barrier
|
||||
{
|
||||
let target_log = PathBuf::from(format!("{log_dir}/node1.log"));
|
||||
let log_offset = target_log.metadata()?.len();
|
||||
let (stopped, released) = tokio::join!(
|
||||
cluster.stop_node_gracefully(interruption_node),
|
||||
release_heal_commit_barrier_after_shutdown(&target_log, log_offset, barrier),
|
||||
);
|
||||
// Await both branches even on error so the child is reaped.
|
||||
stopped?;
|
||||
shutdown_barrier_release = Some(released?);
|
||||
} else {
|
||||
cluster.stop_node_gracefully(interruption_node).await?;
|
||||
}
|
||||
} else {
|
||||
cluster.stop_node(interruption_node)?;
|
||||
}
|
||||
@@ -2386,7 +2513,9 @@ mod tests {
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Some(barrier) = &commit_barrier {
|
||||
if shutdown_barrier_release.is_none()
|
||||
&& let Some(barrier) = &commit_barrier
|
||||
{
|
||||
std::fs::remove_file(barrier)?;
|
||||
}
|
||||
cluster.start_node_from_binary(interruption_node, &server_binary).await?;
|
||||
@@ -2542,6 +2671,10 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
if let Some(path) = &mrf_fallback_scope {
|
||||
std::fs::remove_file(path)?;
|
||||
}
|
||||
|
||||
if let Some(cycle_end) = scanner_cycle_floor {
|
||||
wait_for_scanner_cycle_after(&cluster, cycle_end).await?;
|
||||
}
|
||||
@@ -2713,6 +2846,7 @@ mod tests {
|
||||
"all_peers_bound_to_generation_window": true,
|
||||
"pid_before": target_pid, "pid_after": restarted_pid,
|
||||
"unclean_shutdown_marker": unclean_shutdown_marker_observed.unwrap_or(false),
|
||||
"graceful_barrier_release": shutdown_barrier_release,
|
||||
"objects": evidence_objects, "node_listings": node_listings,
|
||||
});
|
||||
if let Some(outage_write) = outage_write_diagnostic {
|
||||
|
||||
@@ -283,6 +283,9 @@ mod ssec_copy_test;
|
||||
#[cfg(test)]
|
||||
mod multipart_storage_class_test;
|
||||
|
||||
#[cfg(test)]
|
||||
mod multipart_upload_id_test;
|
||||
|
||||
#[cfg(test)]
|
||||
mod storage_class_capability_test;
|
||||
|
||||
|
||||
@@ -0,0 +1,175 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Multipart upload ID regression tests.
|
||||
//!
|
||||
//! Every operation that addresses an existing multipart upload must report an
|
||||
//! upload ID that does not name a live upload as `404 NoSuchUpload`, whether
|
||||
//! the ID is malformed or is a well-formed ID of an aborted upload.
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::common::{RustFSTestEnvironment, init_logging};
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError};
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
|
||||
|
||||
type TestResult = Result<(), Box<dyn std::error::Error + Send + Sync>>;
|
||||
|
||||
const BUCKET: &str = "multipart-upload-id";
|
||||
const KEY: &str = "target.bin";
|
||||
const COPY_SOURCE_KEY: &str = "copy-source.bin";
|
||||
// RustFS upload IDs are URL-safe base64; '.' is outside that alphabet, so
|
||||
// the object layer rejects this ID as malformed instead of looking it up.
|
||||
const MALFORMED_UPLOAD_ID: &str = "malformed.upload.id";
|
||||
const PART_NUMBER: i32 = 1;
|
||||
const PART_BODY: &[u8] = b"multipart upload id regression part";
|
||||
const PART_ETAG: &str = "\"00000000000000000000000000000000\"";
|
||||
const NO_SUCH_UPLOAD_STATUS: u16 = 404;
|
||||
const NO_SUCH_UPLOAD_CODE: &str = "NoSuchUpload";
|
||||
const OBJECT_NOT_FOUND_STATUS: u16 = 404;
|
||||
|
||||
/// Describes how `result` differs from `404 NoSuchUpload`, or returns `None` when it matches.
|
||||
fn no_such_upload_mismatch<T, E>(operation: &str, result: Result<T, SdkError<E>>) -> Option<String>
|
||||
where
|
||||
E: ProvideErrorMetadata + std::fmt::Debug,
|
||||
{
|
||||
let Err(error) = result else {
|
||||
return Some(format!("{operation}: succeeded"));
|
||||
};
|
||||
let status = error.raw_response().map(|response| response.status().as_u16());
|
||||
let code = error.as_service_error().and_then(ProvideErrorMetadata::code);
|
||||
(status != Some(NO_SUCH_UPLOAD_STATUS) || code != Some(NO_SUCH_UPLOAD_CODE))
|
||||
.then(|| format!("{operation}: status {status:?}, code {code:?}: {error:?}"))
|
||||
}
|
||||
|
||||
async fn start_server_with_copy_source() -> Result<(RustFSTestEnvironment, Client), Box<dyn std::error::Error + Send + Sync>>
|
||||
{
|
||||
init_logging();
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(Vec::new()).await?;
|
||||
env.create_test_bucket(BUCKET).await?;
|
||||
let client = env.create_s3_client();
|
||||
client
|
||||
.put_object()
|
||||
.bucket(BUCKET)
|
||||
.key(COPY_SOURCE_KEY)
|
||||
.body(ByteStream::from_static(PART_BODY))
|
||||
.send()
|
||||
.await?;
|
||||
Ok((env, client))
|
||||
}
|
||||
|
||||
/// Sends every operation that addresses an existing upload and asserts that all of them report `404 NoSuchUpload`.
|
||||
async fn assert_every_upload_operation_reports_no_such_upload(client: &Client, upload_id: &str) {
|
||||
let upload_part = client
|
||||
.upload_part()
|
||||
.bucket(BUCKET)
|
||||
.key(KEY)
|
||||
.upload_id(upload_id)
|
||||
.part_number(PART_NUMBER)
|
||||
.body(ByteStream::from_static(PART_BODY))
|
||||
.send()
|
||||
.await;
|
||||
let upload_part_copy = client
|
||||
.upload_part_copy()
|
||||
.bucket(BUCKET)
|
||||
.key(KEY)
|
||||
.upload_id(upload_id)
|
||||
.part_number(PART_NUMBER)
|
||||
.copy_source(format!("/{BUCKET}/{COPY_SOURCE_KEY}"))
|
||||
.send()
|
||||
.await;
|
||||
let list_parts = client.list_parts().bucket(BUCKET).key(KEY).upload_id(upload_id).send().await;
|
||||
let complete = client
|
||||
.complete_multipart_upload()
|
||||
.bucket(BUCKET)
|
||||
.key(KEY)
|
||||
.upload_id(upload_id)
|
||||
.multipart_upload(
|
||||
CompletedMultipartUpload::builder()
|
||||
.parts(CompletedPart::builder().part_number(PART_NUMBER).e_tag(PART_ETAG).build())
|
||||
.build(),
|
||||
)
|
||||
.send()
|
||||
.await;
|
||||
let abort = client
|
||||
.abort_multipart_upload()
|
||||
.bucket(BUCKET)
|
||||
.key(KEY)
|
||||
.upload_id(upload_id)
|
||||
.send()
|
||||
.await;
|
||||
|
||||
let mismatches: Vec<String> = [
|
||||
no_such_upload_mismatch("UploadPart", upload_part),
|
||||
no_such_upload_mismatch("UploadPartCopy", upload_part_copy),
|
||||
no_such_upload_mismatch("ListParts", list_parts),
|
||||
no_such_upload_mismatch("CompleteMultipartUpload", complete),
|
||||
no_such_upload_mismatch("AbortMultipartUpload", abort),
|
||||
]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.collect();
|
||||
assert!(
|
||||
mismatches.is_empty(),
|
||||
"upload ID {upload_id:?}: operations not reporting {NO_SUCH_UPLOAD_STATUS} {NO_SUCH_UPLOAD_CODE}:\n{}",
|
||||
mismatches.join("\n")
|
||||
);
|
||||
}
|
||||
|
||||
/// Asserts that the rejected operations left neither a multipart upload nor an object under `KEY`.
|
||||
async fn assert_no_upload_or_object_published(client: &Client) -> TestResult {
|
||||
let uploads = client.list_multipart_uploads().bucket(BUCKET).prefix(KEY).send().await?;
|
||||
assert!(uploads.uploads().is_empty(), "rejected operations must not leave an upload: {uploads:?}");
|
||||
let head = client.head_object().bucket(BUCKET).key(KEY).send().await;
|
||||
let status = head
|
||||
.as_ref()
|
||||
.err()
|
||||
.and_then(|error| error.raw_response())
|
||||
.map(|response| response.status().as_u16());
|
||||
assert_eq!(
|
||||
status,
|
||||
Some(OBJECT_NOT_FOUND_STATUS),
|
||||
"rejected operations must not publish {KEY}: {head:?}"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn malformed_upload_id_reports_no_such_upload_on_every_upload_operation() -> TestResult {
|
||||
let (_env, client) = start_server_with_copy_source().await?;
|
||||
|
||||
assert_every_upload_operation_reports_no_such_upload(&client, MALFORMED_UPLOAD_ID).await;
|
||||
assert_no_upload_or_object_published(&client).await
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn aborted_upload_id_reports_no_such_upload_on_every_upload_operation() -> TestResult {
|
||||
let (_env, client) = start_server_with_copy_source().await?;
|
||||
let created = client.create_multipart_upload().bucket(BUCKET).key(KEY).send().await?;
|
||||
let upload_id = created.upload_id().ok_or("CreateMultipartUpload returned no upload ID")?;
|
||||
client
|
||||
.abort_multipart_upload()
|
||||
.bucket(BUCKET)
|
||||
.key(KEY)
|
||||
.upload_id(upload_id)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
assert_every_upload_operation_reports_no_such_upload(&client, upload_id).await;
|
||||
assert_no_upload_or_object_published(&client).await
|
||||
}
|
||||
}
|
||||
@@ -77,9 +77,7 @@ core protocol surface plus cross-protocol consistency:
|
||||
existing path returns an error, bad-password authentication is rejected
|
||||
- Cross-protocol via aws-sdk-s3: SFTP write then S3 read with SHA256 match,
|
||||
S3 write then SFTP read with SHA256 match
|
||||
- Cross-API directory visibility: SFTP-created sub-directory visible via S3
|
||||
ListObjectsV2, S3-created `__XLDIR__` marker visible via SFTP readdir as a
|
||||
directory entry
|
||||
- Cross-API directory visibility: SFTP-created sub-directory visible via S3 ListObjectsV2, and an S3 trailing-slash directory object visible via SFTP readdir as a directory entry
|
||||
|
||||
#### sftp_compliance (`test_sftp_compliance_suite`)
|
||||
|
||||
|
||||
@@ -394,9 +394,9 @@ pub async fn test_sftp_core_operations() -> Result<()> {
|
||||
info!("PASS: S3-written object matches via S3 GetObject and SFTP READ");
|
||||
|
||||
// --- Cross-API directory visibility: SFTP mkdir, S3 ListObjectsV2 ---
|
||||
// SFTP mkdir writes a __XLDIR__ marker. The rustfs S3 listing path
|
||||
// decodes that marker back to a trailing-slash key, so the asserted
|
||||
// pattern is "subdir_sftp/".
|
||||
// SFTP mkdir writes a zero-byte object at the trailing-slash key.
|
||||
// Storage encodes that key to the __XLDIR__ marker, and the S3
|
||||
// listing path decodes the marker back to a trailing-slash key.
|
||||
info!("Testing SFTP: SFTP-created sub-directory visible via S3 ListObjectsV2");
|
||||
let sftp_subdir_name = "subdir_sftp";
|
||||
let sftp_subdir_path = format!("/{bucket}/{sftp_subdir_name}");
|
||||
@@ -422,20 +422,21 @@ pub async fn test_sftp_core_operations() -> Result<()> {
|
||||
);
|
||||
info!("PASS: SFTP mkdir visible to S3 ListObjectsV2");
|
||||
|
||||
// --- Cross-API directory visibility: S3 marker, SFTP readdir ---
|
||||
// aws-sdk-s3 PutObject writes a zero-byte marker keyed with the
|
||||
// __XLDIR__ suffix. SFTP readdir must decode the marker back to a
|
||||
// bare directory entry whose file_type reports as a directory.
|
||||
info!("Testing SFTP: S3-created __XLDIR__ marker visible via SFTP readdir");
|
||||
// --- Cross-API directory visibility: S3 directory object, SFTP readdir ---
|
||||
// aws-sdk-s3 PutObject writes a zero-byte object at the trailing-slash
|
||||
// key. The __XLDIR__ suffix is reserved on the client-key boundary, so
|
||||
// a direct put of that suffix is rejected. Storage encodes the slash
|
||||
// key, and SFTP readdir must show a directory entry.
|
||||
info!("Testing SFTP: S3-created directory object visible via SFTP readdir");
|
||||
let s3_subdir_name = "subdir_s3";
|
||||
let s3_subdir_marker_key = format!("{s3_subdir_name}{XLDIR_SUFFIX}");
|
||||
let s3_subdir_key = format!("{s3_subdir_name}/");
|
||||
s3.put_object()
|
||||
.bucket(bucket)
|
||||
.key(&s3_subdir_marker_key)
|
||||
.key(&s3_subdir_key)
|
||||
.body(ByteStream::from_static(b""))
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| anyhow!("S3 PutObject {}/{} failed: {:?}", bucket, s3_subdir_marker_key, e))?;
|
||||
.map_err(|e| anyhow!("S3 PutObject {}/{} failed: {:?}", bucket, s3_subdir_key, e))?;
|
||||
let bucket_entries: Vec<(String, bool)> = sftp
|
||||
.read_dir(&bucket_path)
|
||||
.await?
|
||||
@@ -444,9 +445,9 @@ pub async fn test_sftp_core_operations() -> Result<()> {
|
||||
let visible_via_sftp = bucket_entries.iter().any(|(name, is_dir)| name == s3_subdir_name && *is_dir);
|
||||
assert!(
|
||||
visible_via_sftp,
|
||||
"S3-created marker must appear as a directory in SFTP readdir: entries were {bucket_entries:?}"
|
||||
"S3-created directory object must appear as a directory in SFTP readdir: entries were {bucket_entries:?}"
|
||||
);
|
||||
info!("PASS: S3 marker visible to SFTP readdir as a directory");
|
||||
info!("PASS: S3 directory object visible to SFTP readdir as a directory");
|
||||
|
||||
// --- Pre-cleanup of cross-protocol fixtures ---
|
||||
// Removes the new files and sub-directories so the existing rmdir
|
||||
|
||||
@@ -412,6 +412,38 @@ mod integration_tests {
|
||||
);
|
||||
}
|
||||
|
||||
async fn upload_after_quota_reconciliation(
|
||||
env: &QuotaTestEnv,
|
||||
key: &str,
|
||||
size_bytes: usize,
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let upload = async {
|
||||
loop {
|
||||
let result = env
|
||||
.client
|
||||
.put_object()
|
||||
.bucket(&env.bucket_name)
|
||||
.key(key)
|
||||
.body(aws_sdk_s3::primitives::ByteStream::from(vec![0u8; size_bytes]))
|
||||
.send()
|
||||
.await;
|
||||
match result {
|
||||
Ok(_) => break,
|
||||
Err(error) => assert_quota_rejection(
|
||||
error.raw_response().map(|response| response.status().as_u16()),
|
||||
error.as_service_error(),
|
||||
&error,
|
||||
),
|
||||
}
|
||||
sleep(Duration::from_secs(1)).await;
|
||||
}
|
||||
};
|
||||
// The durable V1 ledger can retain the old charge for its 30-second reconciliation cooldown.
|
||||
timeout(Duration::from_secs(45), upload)
|
||||
.await
|
||||
.map_err(|_| format!("PUT {key} did not succeed after quota reconciliation within 45 seconds").into())
|
||||
}
|
||||
|
||||
async fn assert_put_rejected_by_quota(env: &QuotaTestEnv, key: &str, size_bytes: usize) {
|
||||
let error = env
|
||||
.client
|
||||
@@ -573,8 +605,7 @@ mod integration_tests {
|
||||
|
||||
assert!(!env.object_exists("file1.txt").await?);
|
||||
|
||||
// Now we should be able to upload again (quota freed up)
|
||||
env.upload_object("file3.txt", 256 * 1024).await?;
|
||||
upload_after_quota_reconciliation(&env, "file3.txt", 256 * 1024).await?;
|
||||
assert!(env.object_exists("file3.txt").await?);
|
||||
|
||||
env.cleanup_bucket().await?;
|
||||
@@ -981,8 +1012,7 @@ mod integration_tests {
|
||||
);
|
||||
assert_eq!(delete_result.deleted().len(), 2);
|
||||
|
||||
// Now should be able to upload again (quota freed up)
|
||||
env.upload_object("file3.txt", 256 * 1024).await?;
|
||||
upload_after_quota_reconciliation(&env, "file3.txt", 256 * 1024).await?;
|
||||
assert!(env.object_exists("file3.txt").await?);
|
||||
|
||||
env.cleanup_bucket().await?;
|
||||
|
||||
@@ -22,10 +22,11 @@ use rustfs_protos::{
|
||||
models::PingBodyBuilder,
|
||||
proto_gen::node_service::{
|
||||
BatchGenerallyLockRequest, BatchGenerallyLockResponse, BatchReadVersionRequest, BatchReadVersionResponse,
|
||||
GenerallyLockRequest, GenerallyLockResponse, GenerallyLockResult, PingRequest, PingResponse,
|
||||
ScannerPublicationLeaseReleaseRequest, ScannerPublicationLeaseReleaseResponse, ScannerPublicationLeaseRequest,
|
||||
ScannerPublicationLeaseResponse, SnapshotLeaseMutationResponse, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest,
|
||||
SnapshotLeaseRequest, SnapshotLeaseResponse, node_service_server::NodeService,
|
||||
GenerallyLockRequest, GenerallyLockResponse, GenerallyLockResult, MrfLifecycleRequest, MrfLifecycleResponse, PingRequest,
|
||||
PingResponse, ScannerPublicationLeaseReleaseRequest, ScannerPublicationLeaseReleaseResponse,
|
||||
ScannerPublicationLeaseRequest, ScannerPublicationLeaseResponse, SnapshotLeaseMutationResponse,
|
||||
SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest, SnapshotLeaseRequest, SnapshotLeaseResponse,
|
||||
node_service_server::NodeService,
|
||||
},
|
||||
};
|
||||
use std::pin::Pin;
|
||||
@@ -42,6 +43,7 @@ fn lock_result_from_response(response: rustfs_lock::LockResponse) -> GenerallyLo
|
||||
success: response.success,
|
||||
error_info: response.error,
|
||||
lock_info: response.lock_info.and_then(|info| serde_json::to_string(&info).ok()),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -50,6 +52,7 @@ fn lock_result_from_error(error: impl Into<String>) -> GenerallyLockResult {
|
||||
success: false,
|
||||
error_info: Some(error.into()),
|
||||
lock_info: None,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -59,6 +62,7 @@ fn lock_result_from_release(lock_id: &rustfs_lock::LockId, success: bool) -> Gen
|
||||
success: true,
|
||||
error_info: None,
|
||||
lock_info: None,
|
||||
..Default::default()
|
||||
}
|
||||
} else {
|
||||
lock_result_from_error(format!("lock not found for release: {lock_id}"))
|
||||
@@ -164,6 +168,7 @@ impl NodeService for MinimalLockNodeService {
|
||||
success: false,
|
||||
error_info: Some(format!("can not decode args, err: {err}")),
|
||||
lock_info: None,
|
||||
..Default::default()
|
||||
}));
|
||||
}
|
||||
};
|
||||
@@ -175,6 +180,7 @@ impl NodeService for MinimalLockNodeService {
|
||||
success: result.success,
|
||||
error_info: result.error,
|
||||
lock_info: lock_info_json,
|
||||
..Default::default()
|
||||
}))
|
||||
}
|
||||
Err(err) => Ok(Response::new(GenerallyLockResponse {
|
||||
@@ -184,6 +190,7 @@ impl NodeService for MinimalLockNodeService {
|
||||
args.resource, args.owner, err
|
||||
)),
|
||||
lock_info: None,
|
||||
..Default::default()
|
||||
})),
|
||||
}
|
||||
}
|
||||
@@ -197,6 +204,7 @@ impl NodeService for MinimalLockNodeService {
|
||||
success: false,
|
||||
error_info: Some(format!("can not decode args, err: {err}")),
|
||||
lock_info: None,
|
||||
..Default::default()
|
||||
}));
|
||||
}
|
||||
};
|
||||
@@ -208,6 +216,7 @@ impl NodeService for MinimalLockNodeService {
|
||||
success: result.success,
|
||||
error_info: result.error_info,
|
||||
lock_info: None,
|
||||
..Default::default()
|
||||
}))
|
||||
}
|
||||
Err(err) => Ok(Response::new(GenerallyLockResponse {
|
||||
@@ -217,6 +226,7 @@ impl NodeService for MinimalLockNodeService {
|
||||
args.resource, args.owner, err
|
||||
)),
|
||||
lock_info: None,
|
||||
..Default::default()
|
||||
})),
|
||||
}
|
||||
}
|
||||
@@ -230,6 +240,7 @@ impl NodeService for MinimalLockNodeService {
|
||||
success: false,
|
||||
error_info: Some(format!("can not decode args, err: {err}")),
|
||||
lock_info: None,
|
||||
..Default::default()
|
||||
}));
|
||||
}
|
||||
};
|
||||
@@ -241,6 +252,7 @@ impl NodeService for MinimalLockNodeService {
|
||||
success: result.success,
|
||||
error_info: result.error_info,
|
||||
lock_info: None,
|
||||
..Default::default()
|
||||
}))
|
||||
}
|
||||
Err(err) => Ok(Response::new(GenerallyLockResponse {
|
||||
@@ -250,6 +262,7 @@ impl NodeService for MinimalLockNodeService {
|
||||
args.resource, args.owner, err
|
||||
)),
|
||||
lock_info: None,
|
||||
..Default::default()
|
||||
})),
|
||||
}
|
||||
}
|
||||
@@ -263,6 +276,7 @@ impl NodeService for MinimalLockNodeService {
|
||||
success: false,
|
||||
error_info: Some(format!("can not decode args, err: {err}")),
|
||||
lock_info: None,
|
||||
..Default::default()
|
||||
}));
|
||||
}
|
||||
};
|
||||
@@ -272,11 +286,13 @@ impl NodeService for MinimalLockNodeService {
|
||||
success,
|
||||
error_info: None,
|
||||
lock_info: None,
|
||||
..Default::default()
|
||||
})),
|
||||
Err(err) => Ok(Response::new(GenerallyLockResponse {
|
||||
success: false,
|
||||
error_info: Some(format!("can not refresh, err: {err}")),
|
||||
lock_info: None,
|
||||
..Default::default()
|
||||
})),
|
||||
}
|
||||
}
|
||||
@@ -980,6 +996,10 @@ impl NodeService for MinimalLockNodeService {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
async fn mrf_lifecycle(&self, _request: Request<MrfLifecycleRequest>) -> Result<Response<MrfLifecycleResponse>, Status> {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
async fn background_heal_status(
|
||||
&self,
|
||||
_request: Request<rustfs_protos::proto_gen::node_service::BackgroundHealStatusRequest>,
|
||||
|
||||
@@ -31,8 +31,8 @@ use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use aws_sdk_s3::operation::list_object_versions::ListObjectVersionsOutput;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{
|
||||
BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, DeleteMarkerEntry, ObjectVersion, ServerSideEncryption,
|
||||
VersioningConfiguration,
|
||||
BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, Delete, DeleteMarkerEntry, ObjectIdentifier, ObjectVersion,
|
||||
ServerSideEncryption, VersioningConfiguration,
|
||||
};
|
||||
use base64_simd::STANDARD as BASE64_STANDARD;
|
||||
use bytes::Bytes;
|
||||
@@ -3729,6 +3729,8 @@ async fn test_bucket_replication_converges_delete_marker_and_version_purge() ->
|
||||
init_logging();
|
||||
|
||||
let mut source_env = RustFSTestEnvironment::new().await?;
|
||||
let source_log = format!("{}/source.log", source_env.temp_dir);
|
||||
source_env.capture_log_path = Some(source_log.clone());
|
||||
let mut source_env_vars = replication_fast_env();
|
||||
source_env_vars.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
|
||||
source_env.start_rustfs_server_with_env(vec![], &source_env_vars).await?;
|
||||
@@ -3791,19 +3793,66 @@ async fn test_bucket_replication_converges_delete_marker_and_version_purge() ->
|
||||
"target did not preserve the source delete-marker version ID"
|
||||
);
|
||||
|
||||
source_client
|
||||
.delete_object()
|
||||
let wait_for_source_writeback = || async {
|
||||
let metrics_url = format!("{}/{source_bucket}?replication-metrics", source_env.url);
|
||||
timeout(Duration::from_secs(30), async {
|
||||
loop {
|
||||
let response = signed_request(
|
||||
http::Method::GET,
|
||||
&metrics_url,
|
||||
&source_env.access_key,
|
||||
&source_env.secret_key,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
let metrics: serde_json::Value = response.error_for_status()?.json().await?;
|
||||
let queued = metrics["queued"]["curr"]["count"]
|
||||
.as_f64()
|
||||
.ok_or("missing replication queue count")?;
|
||||
if metrics["provider_available"] == true && queued == 0.0 {
|
||||
return Ok::<(), Box<dyn Error + Send + Sync>>(());
|
||||
}
|
||||
sleep(Duration::from_millis(50)).await;
|
||||
}
|
||||
})
|
||||
.await?
|
||||
};
|
||||
wait_for_source_writeback().await?;
|
||||
|
||||
let purged = source_client
|
||||
.delete_objects()
|
||||
.bucket(source_bucket)
|
||||
.key(object_key)
|
||||
.version_id(&purged_version_id)
|
||||
.delete(
|
||||
Delete::builder()
|
||||
.objects(
|
||||
ObjectIdentifier::builder()
|
||||
.key(object_key)
|
||||
.version_id(&purged_version_id)
|
||||
.build()?,
|
||||
)
|
||||
.objects(
|
||||
ObjectIdentifier::builder()
|
||||
.key(object_key)
|
||||
.version_id(delete_marker_version_id)
|
||||
.build()?,
|
||||
)
|
||||
.build()?,
|
||||
)
|
||||
.send()
|
||||
.await?;
|
||||
assert!(purged.errors().is_empty(), "batch version purge failed: {:?}", purged.errors());
|
||||
assert_eq!(purged.deleted().len(), 2, "both the data version and marker must be purged");
|
||||
assert_replication_converged(&source_client, source_bucket, &target_client, target_bucket).await?;
|
||||
let target_state = list_replication_state(&target_client, target_bucket).await?;
|
||||
assert!(
|
||||
target_state.iter().all(|entry| entry.version_id != purged_version_id),
|
||||
"target retained the explicitly purged object version"
|
||||
);
|
||||
assert!(
|
||||
target_state.iter().all(|entry| entry.version_id != delete_marker_version_id),
|
||||
"target retained the explicitly purged delete marker"
|
||||
);
|
||||
assert!(
|
||||
target_state
|
||||
.iter()
|
||||
@@ -3819,6 +3868,19 @@ async fn test_bucket_replication_converges_delete_marker_and_version_purge() ->
|
||||
.await?;
|
||||
assert_eq!(retained.body.collect().await?.into_bytes().as_ref(), b"versioned replication payload v2");
|
||||
|
||||
// Wait for the source workers to finish their final writeback, which runs
|
||||
// after the remote purge has already become visible (rustfs#9121).
|
||||
wait_for_source_writeback().await?;
|
||||
let logs = fs::read_to_string(source_log).await?;
|
||||
let writeback_failures: Vec<_> = logs
|
||||
.lines()
|
||||
.filter(|line| line.contains("apply_replication_delete_state"))
|
||||
.collect();
|
||||
assert!(
|
||||
writeback_failures.is_empty(),
|
||||
"a completed batch purge must not report a source writeback failure: {writeback_failures:?}"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user