mirror of
https://github.com/rustfs/rustfs.git
synced 2026-11-04 18:13:57 +00:00
Compare commits
67 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 2a5d4d882a | |||
| 279a88ab56 | |||
| b4abe64ada | |||
| add363f084 | |||
| 1f2dbe36e5 | |||
| 67fbafa13e | |||
| 1000860cb8 | |||
| b783f404ad | |||
| 11a991c777 | |||
| 3f2b6f690a | |||
| 88b626e7a0 | |||
| 8cf6187ff4 | |||
| a0a0cee9eb | |||
| 80fff4b5c5 | |||
| d91dfcedd2 | |||
| 79c4fedf8c | |||
| 73c846e09c | |||
| 6938c467d0 | |||
| 5323df7049 | |||
| 55b6fbd260 | |||
| dd677cc121 | |||
| 0c2f327974 | |||
| df42cf1ce8 | |||
| d290d46b34 | |||
| 0d297ed8cd | |||
| b8dffa2f1d | |||
| 62b6f5f3fc | |||
| be0986115c | |||
| 1bd91c8dad | |||
| dcb22bd4d5 | |||
| fa17718444 | |||
| 99abffc57a | |||
| fc061c7930 | |||
| d86bf1360f | |||
| f6097d1df2 | |||
| 7e5d32eecf | |||
| 72da0e21c4 | |||
| 78a0cf321c | |||
| cbe38a120f | |||
| db0828aa1c | |||
| 3ca889b099 | |||
| e6112c526f | |||
| c5967686f1 | |||
| 859de565ad | |||
| 1f7cc593e7 | |||
| 1d5a50f094 | |||
| 53245be9f4 | |||
| c47432e353 | |||
| f622c1bce0 | |||
| a99d8f4f89 | |||
| 4d15b81623 | |||
| 836d30192f | |||
| 860c61ee6d | |||
| f600af4704 | |||
| 9e747e8475 | |||
| 96fc067765 | |||
| e809b4899e | |||
| 584f03f300 | |||
| d52d6fade7 | |||
| 3623d6db2d | |||
| 484f1254ca | |||
| c9c16efbb6 | |||
| f2a59073ca | |||
| c1af94d1d1 | |||
| f84bbfe28f | |||
| 378a2ff7f8 | |||
| b88b53aa6a |
@@ -18,7 +18,6 @@ These implementation, style, and quality rules apply to all contributions, regar
|
||||
- Do not include agent, tool, contributor, account, or organization names in branch names.
|
||||
- Push to the user-requested remote or the repository's configured push remote. Do not hard-code or infer a remote from an account name.
|
||||
- Check free space before artifact-heavy builds, tests, coverage, or downloads. Re-check before a broad gate when space is tight.
|
||||
- Before handoff, remove the Cargo target directories and temporary worktrees the task created unless they stay intentionally or hold unmerged work. Identify build output by Cargo's markers (`.rustc_info.json`, `CACHEDIR.TAG`), not by directory name: `target` is also a source directory here.
|
||||
- Remove only task-owned temporary/build artifacts. Never delete another task's worktree or uncommitted data.
|
||||
- At handoff, mention disk or cleanup details only when they affected execution or artifacts/worktrees remain intentionally.
|
||||
- For Docker verification, ensure the tested image contains the intended binary. `buildx build` without `--load` leaves the image in the build cache; use the Makefile's loaded image and rebuild without cache only when stale artifacts are evidenced.
|
||||
|
||||
@@ -1,16 +0,0 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"rollout": "enforced",
|
||||
"core_packages": ["rustfs", "rustfs-ecstore", "rustfs-filemeta", "rustfs-iam", "rustfs-policy"],
|
||||
"connect_packages": [
|
||||
"rustfs-credentials", "rustfs-config", "rustfs-common", "rustfs-crypto",
|
||||
"rustfs-iam", "rustfs-policy", "rustfs-kms", "rustfs-license", "rustfs-security-governance",
|
||||
"rustfs-utils", "rustfs-tls-runtime", "rustfs-trusted-proxies", "rustfs-filemeta",
|
||||
"rustfs-lock", "rustfs-obs", "rustfs-signer", "rustfs-io-metrics", "rustfs-s3-ops"
|
||||
],
|
||||
"full_paths": [
|
||||
"Cargo.toml", "Cargo.lock", "*/Cargo.toml", "rust-toolchain*", "rustfmt.toml",
|
||||
"clippy.toml", ".cargo/*", ".config/*", ".github/*", "Makefile",
|
||||
"scripts/*", "build.rs", "*.proto"
|
||||
]
|
||||
}
|
||||
@@ -1,2 +1,2 @@
|
||||
sha256-linux=2f97cbbeb2dbf52919b704a3d07336c162a41f95079d941d74251f9279c494e6
|
||||
sha256-darwin=2f97cbbeb2dbf52919b704a3d07336c162a41f95079d941d74251f9279c494e6
|
||||
sha256-linux=eb33067d90ee66c0a97e8f3658e46c152e47f897d2fbfdc6cb5ad93cfa55414b
|
||||
sha256-darwin=eb33067d90ee66c0a97e8f3658e46c152e47f897d2fbfdc6cb5ad93cfa55414b
|
||||
|
||||
@@ -1,2 +1,2 @@
|
||||
sha256-darwin=7c705942d3a9e583ebaa85845daa21c4f81fdbf36def24db931aaef38636c53e
|
||||
sha256-linux=5c33dc395233671cb77452f8fb12900ecdbeb368632222b96835d310bce1aab8
|
||||
sha256-darwin=4fb92e88763982eabafd74d05eb3d269387624a447e427a8c6f84914e5b03bfe
|
||||
sha256-linux=bd0db7e846ce24f056a097655db8fe2265201913260d0e16feb38d99dcb954a7
|
||||
|
||||
@@ -1 +1 @@
|
||||
sha256=e4ee3898b840a8374cb02c64342cc15f8d7ecd1becaf4b4a2a4657290f9e7ff1
|
||||
sha256=4eac269f9a59b2f9be4e7a87ea6aa38454f5ebe10ede35c9b8d4218137d2cd35
|
||||
|
||||
@@ -1 +1 @@
|
||||
sha256=00f8a56b75990b487a594b97c36610b94476a6944ba00473e37187ea654d42bf
|
||||
sha256=879f421f1115d44f930e229510a2e533cfd5ebd4effe38730b82ef61f4fd7a86
|
||||
|
||||
@@ -70,6 +70,11 @@ s3s-footprint-check: ## Check the s3s dependency footprint ratchet stays frozen
|
||||
@echo "📦 Checking s3s footprint ratchet..."
|
||||
./scripts/check_s3s_footprint.sh
|
||||
|
||||
.PHONY: test-count-ratchet-check
|
||||
test-count-ratchet-check: ## Check no workspace crate lost tests or gained #[ignore] against the committed baseline
|
||||
@echo "🧪 Checking per-crate test-count ratchet..."
|
||||
./scripts/check_test_count_ratchet.sh
|
||||
|
||||
.PHONY: fips-wording-check
|
||||
fips-wording-check: ## Check docs and crates/kms do not over-claim crypto capabilities
|
||||
@echo "📣 Checking cryptographic capability wording guard..."
|
||||
@@ -90,7 +95,6 @@ test-wiring-check: ## Check tests stay registered and selected by their intended
|
||||
@echo "🧪 Checking test wiring..."
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/check_test_wiring.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/ci_gate.py --check-workflow
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/validation_registry.py check
|
||||
|
||||
.PHONY: log-analyzer-rules-check
|
||||
log-analyzer-rules-check: core-deps ## Check log-analyzer rule anchors still exist verbatim in source
|
||||
|
||||
@@ -20,11 +20,11 @@ planning-docs-check: ## Check that no planning-type documents are committed
|
||||
./scripts/check_no_planning_docs.sh
|
||||
|
||||
.PHONY: pre-commit
|
||||
pre-commit: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check error-other-ratchet-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check quick-check ## Run fast pre-commit checks without clippy/full tests
|
||||
pre-commit: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check error-other-ratchet-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check test-count-ratchet-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check quick-check ## Run fast pre-commit checks without clippy/full tests
|
||||
@echo "✅ All pre-commit checks passed!"
|
||||
|
||||
.PHONY: pre-pr
|
||||
pre-pr: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check error-other-ratchet-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check log-analyzer-rules-check offline-enrollment-e2e-check clippy-check test ## Run full pre-PR checks with clippy and tests
|
||||
pre-pr: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check error-other-ratchet-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check test-count-ratchet-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check log-analyzer-rules-check offline-enrollment-e2e-check clippy-check test ## Run full pre-PR checks with clippy and tests
|
||||
@echo "✅ All pre-PR checks passed!"
|
||||
|
||||
.PHONY: dev-check
|
||||
|
||||
@@ -51,7 +51,9 @@ script-tests: ## Run shell script tests
|
||||
./scripts/test_python_bin.sh
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_check_doc_paths.py
|
||||
./scripts/check_embedded_secrets.sh --self-test
|
||||
./scripts/check_s3s_footprint.sh --self-test
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/check_test_wiring.py --self-test
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/codemods/s3_error_to_s3_types.py --self-test
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_e2e_binary.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_inspect_btf_prototypes.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_release_build_workflow.py
|
||||
@@ -59,18 +61,12 @@ script-tests: ## Run shell script tests
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_upgrade_workflow.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_migration_gate_evidence.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/ci_gate.py --self-test
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/check_connect_release_boundary.py --self-test
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_ci_test_artifact.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_ci_test_shards.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_ci_merge_queue.py
|
||||
$(RUSTFS_PYTHON_BIN) -m unittest discover -s scripts/ci -p 'test_*.py'
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/check_security_coverage.py --self-test
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/check_scheduled_validation_freshness.py --self-test
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_security_workflow.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_nightly_candidate.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_functional_chain.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_functional_chain_health.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_validation_registry.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_inspect_validation_runner.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_retain_validation_evidence.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_ci_timing_report.py
|
||||
|
||||
+24
-29
@@ -234,7 +234,7 @@ test-group = 'embedded-test-ports'
|
||||
# processes cannot exhaust those budgets. Keep deadlines and assertions intact.
|
||||
# The short HTTP fixtures are tracked in rustfs/backlog#2714.
|
||||
[[profile.default.overrides]]
|
||||
filter = 'package(rustfs) & (binary(/^connect_registration$/) | binary(connect_inventory) | binary(connect_perf_drive) | (binary(connect_perf_client) & test(=configured_proxy_and_direct_endpoint_failures_are_distinct)) | (binary(connect_perf_object) & test(/^(real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put|cleanup_uses_a_budget_independent_of_the_measurement_window|cleanup_accepts_missing_object_and_still_deletes_the_bucket|cleanup_diagnostics_identify_each_disconnected_transport_stage|cleanup_diagnostics_identify_each_failed_http_stage_without_private_context|cleanup_diagnostics_identify_timeouts_without_resetting_the_shared_deadline|oversized_response_is_rejected_without_buffering_it)$/)))'
|
||||
filter = 'package(rustfs) & (binary(/^connect_registration$/) | binary(connect_inventory) | binary(connect_perf_drive) | (binary(connect_perf_client) & test(=configured_proxy_and_direct_endpoint_failures_are_distinct)) | (binary(connect_perf_object) & test(/^(real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put|cleanup_uses_a_budget_independent_of_the_measurement_window|oversized_response_is_rejected_without_buffering_it)$/)))'
|
||||
threads-required = "num-test-threads"
|
||||
|
||||
# This cancellation probe hashes the serving executable before acquiring its
|
||||
@@ -356,6 +356,12 @@ threads-required = "num-test-threads"
|
||||
filter = 'package(rustfs-ecstore) & test(=store::init::tests::dispatch_manifest_rollback_bounded_concurrency_reaches_tail_behind_slow_member)'
|
||||
threads-required = "num-test-threads"
|
||||
|
||||
# These short-deadline real-disk progress probes need the test run's capacity
|
||||
# for scanner and object-lock handoffs; keep their assertions and deadlines.
|
||||
[[profile.default.overrides]]
|
||||
filter = '(package(rustfs-scanner) & (test(=scanner::tests::cycle_persist_failure::later_cycle_post_commit_failure_reloads_durable_state) | test(=scanner::tests::running_main_loop_catches_up_pause_cleared_after_startup_observe))) | (package(rustfs-ecstore) & test(=store::init::tests::restored_transitioned_delete_causally_enqueues_free_version))'
|
||||
threads-required = "num-test-threads"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ci profile — the strict CI gate (ci.yml `cargo nextest run --profile ci`)
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -424,6 +430,12 @@ threads-required = "num-test-threads"
|
||||
filter = 'package(rustfs-scanner) & (test(/^scanner::backlog::tests::native_/) | test(=scanner_folder::tests::mrf_ownership::mrf_ownership_manager_completion_preserves_scanner_pending) | test(/^scanner_io::tests::service_cohort::/))'
|
||||
threads-required = "num-test-threads"
|
||||
|
||||
# Keep short-deadline real-disk progress probes from competing with unrelated
|
||||
# tests for scanner and object-lock capacity. Assertions and deadlines stay on.
|
||||
[[profile.ci.overrides]]
|
||||
filter = '(package(rustfs-scanner) & (test(=scanner::tests::cycle_persist_failure::later_cycle_post_commit_failure_reloads_durable_state) | test(=scanner::tests::running_main_loop_catches_up_pause_cleared_after_startup_observe))) | (package(rustfs-ecstore) & test(=store::init::tests::restored_transitioned_delete_causally_enqueues_free_version))'
|
||||
threads-required = "num-test-threads"
|
||||
|
||||
# Emit this expensive fixture's stage timings in CI and use tmpfs when safe.
|
||||
[[profile.ci.overrides]]
|
||||
filter = 'package(rustfs-ecstore) & test(=store::init::tests::decommission_durable_ilm_receipt_pagination_fails_closed_on_second_page)'
|
||||
@@ -548,7 +560,7 @@ filter = 'package(rustfs) & (binary(/^embedded.*_test$/) | binary(admin_diagnost
|
||||
test-group = 'embedded-test-ports'
|
||||
|
||||
[[profile.ci.overrides]]
|
||||
filter = 'package(rustfs) & (binary(/^connect_registration$/) | binary(connect_inventory) | binary(connect_perf_drive) | (binary(connect_perf_client) & test(=configured_proxy_and_direct_endpoint_failures_are_distinct)) | (binary(connect_perf_object) & test(/^(real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put|cleanup_uses_a_budget_independent_of_the_measurement_window|cleanup_accepts_missing_object_and_still_deletes_the_bucket|cleanup_diagnostics_identify_each_disconnected_transport_stage|cleanup_diagnostics_identify_each_failed_http_stage_without_private_context|cleanup_diagnostics_identify_timeouts_without_resetting_the_shared_deadline|oversized_response_is_rejected_without_buffering_it)$/)))'
|
||||
filter = 'package(rustfs) & (binary(/^connect_registration$/) | binary(connect_inventory) | binary(connect_perf_drive) | (binary(connect_perf_client) & test(=configured_proxy_and_direct_endpoint_failures_are_distinct)) | (binary(connect_perf_object) & test(/^(real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put|cleanup_uses_a_budget_independent_of_the_measurement_window|oversized_response_is_rejected_without_buffering_it)$/)))'
|
||||
threads-required = "num-test-threads"
|
||||
|
||||
# Preserve the cancellation probe's startup budget and assertions in CI too.
|
||||
@@ -698,6 +710,14 @@ test-group = 'ecstore-serial-flaky'
|
||||
# the PR-lane e2e coverage for scanner usage snapshots consumed by quota and
|
||||
# admin surfaces. It uses the same single-node, random-port, isolated-temp-dir
|
||||
# fixture as the existing smoke modules.
|
||||
#
|
||||
# Dual-stack differential (rustfs/backlog#2740): `dual_stack_differential::`
|
||||
# starts two single-node servers from the same binary, one per
|
||||
# RUSTFS_S3_STACK value, on random ports with isolated temp dirs, and runs one
|
||||
# SDK script covering every operation of `impl S3 for FS` against both. Its
|
||||
# comparator unit tests need no server. It meets the admission criteria
|
||||
# unchanged; the module clause admits a new differential test without another
|
||||
# filter edit.
|
||||
[profile.e2e-smoke]
|
||||
default-filter = """
|
||||
package(e2e_test) & (
|
||||
@@ -706,6 +726,7 @@ default-filter = """
|
||||
| test(/^reliant::lifecycle::/)
|
||||
| test(/^conditional_delete_objects_test::/)
|
||||
| test(/^reliant::tiering::/)
|
||||
| test(/^dual_stack_differential::/)
|
||||
| test(/^upgrade_compatibility_test::upgrade_write_readiness_tests::/)
|
||||
| test(/^on_demand_migration::(get_basic_test::(get_miss_pulls_inline_and_serves_locally_afterwards|head_miss_answers_from_the_source_without_persisting)|interaction_test::test_odm_admin_config_is_redacted_and_status_counts_match_the_source)$/)
|
||||
)
|
||||
@@ -827,7 +848,7 @@ slow-timeout = { period = "120s", terminate-after = 12, grace-period = "10s" }
|
||||
# ---------------------------------------------------------------------------
|
||||
# e2e-distributed profile — 4-node 4-disk Actions suite
|
||||
# ---------------------------------------------------------------------------
|
||||
# Nightly and manual-dispatch lane owned by
|
||||
# Storage-sensitive PR / nightly / dispatch lane owned by
|
||||
# .github/workflows/e2e-distributed.yml.
|
||||
# Each case starts four rustfs processes (and for site replication, two
|
||||
# clusters). Upgrade cases also require RUSTFS_UPGRADE_SOURCE_BINARY.
|
||||
@@ -960,12 +981,6 @@ test-group = 'e2e-inline-boundaries'
|
||||
filter = 'package(e2e_test) & test(/^heal_erasure_disk_rebuild_test::/)'
|
||||
test-group = 'e2e-cluster-nightly'
|
||||
|
||||
# The multi-pool crash fixture seeds three nodes and 36 drives. Reserve this
|
||||
# nextest run's workers so other multi-node tests do not starve its deadline.
|
||||
[[profile.e2e-full.overrides]]
|
||||
filter = 'package(e2e_test) & test(=heal_erasure_disk_rebuild_test::tests::test_cluster_root_heal_recovers_ec84_shards_across_multi_pool_after_background_target_crash)'
|
||||
threads-required = "num-test-threads"
|
||||
|
||||
[[profile.e2e-full.overrides]]
|
||||
filter = 'package(e2e_test) & (test(/^kms::kms_vault_test::/) | test(/^kms::kms_rekey_sweep_test::/) | test(/^kms::configured_roundtrip_test::test_configured_vault_kms_admin_and_versioned_cleanup$/))'
|
||||
test-group = 'e2e-vault'
|
||||
@@ -975,23 +990,3 @@ test-group = 'e2e-vault'
|
||||
[[profile.e2e-full.overrides]]
|
||||
filter = 'package(e2e_test) & (test(/^list_objects_v2_pagination_test::tests::(test_list_objects_v2_delimiter_small_page_traverses_all|test_list_objects_v2_max_keys_above_limit_returns_token|test_list_objects_v2_maxkeys_above_limit_with_delimiter)$/) | test(=reliant::tiering::test_hermetic_transition_restore_failure_expiry_and_retry))'
|
||||
threads-required = "num-test-threads"
|
||||
|
||||
# The mc fixture uploads hundreds of objects concurrently before measuring listing.
|
||||
# Reserve the runner so other server fixtures do not starve its disk operations.
|
||||
[[profile.e2e-full.overrides]]
|
||||
filter = 'package(e2e_test) & test(=mc_mirror_small_bucket_test::test_mc_mirror_small_bucket_completes_without_list_timeout)'
|
||||
threads-required = "num-test-threads"
|
||||
|
||||
# Fixed, small contract for every executable server change. These existing
|
||||
# scenarios exercise write/read, anonymous denial and retained bytes on restart.
|
||||
[profile.core-contract]
|
||||
default-filter = '''
|
||||
package(e2e_test) & test(/^(anonymous_access_test::test_anonymous_access_denied_when_restrict_public_buckets_enabled|head_object_consistency_test::head_object_consistency_after_write_and_multipart_and_presigned_head|storage_class_capability_test::tests::historical_label_only_metadata_is_standard_without_hiding_a_real_transition_tier)$/)
|
||||
'''
|
||||
retries = 0
|
||||
fail-fast = false
|
||||
test-threads = 1
|
||||
slow-timeout = { period = "60s", terminate-after = 3 }
|
||||
|
||||
[profile.core-contract.junit]
|
||||
path = "junit.xml"
|
||||
|
||||
@@ -1,77 +0,0 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"plan_id": "rustfs-release-validation",
|
||||
"description": "Versioned certification plan for every public RustFS release exit (rustfs/backlog#2662, R01). preview, prerelease and stable all require this explicit plan. Scripts/release/release_gate.py fail-closes publication on any missing, duplicate, stale, substituted or non-PASS required item.",
|
||||
"release_types": ["preview", "prerelease", "stable"],
|
||||
"targets": [
|
||||
{"os": "linux", "arch": "x86_64", "abi": "musl"},
|
||||
{"os": "linux", "arch": "aarch64", "abi": "musl"},
|
||||
{"os": "linux", "arch": "x86_64", "abi": "gnu"},
|
||||
{"os": "linux", "arch": "aarch64", "abi": "gnu"},
|
||||
{"os": "macos", "arch": "aarch64", "abi": "none"},
|
||||
{"os": "windows", "arch": "x86_64", "abi": "msvc"}
|
||||
],
|
||||
"targets_basis": "Platform matrix of .github/workflows/build.yml (prepare-platform-matrix). gnu and macOS targets build with the pyroscope feature; musl and Windows targets build with no extra features.",
|
||||
"features": ["pyroscope"],
|
||||
"trusted_workflows": [
|
||||
{"repository": "rustfs/s3chaos", "workflow_id": "release-gate.yml"}
|
||||
],
|
||||
"trusted_workflows_basis": "The shared evidence envelope producer lives in rustfs/s3chaos (G01, fault/release_gate.rs) and its plan/live split workflow is .github/workflows/release-gate.yml there. Only results from these identities may enter a release certification; PR-uploaded JSON never gains release authority. R02 must extend this list when additional verifier workflows land, never widen it to untrusted sources.",
|
||||
"lanes": [
|
||||
{
|
||||
"lane": "identity",
|
||||
"harness_repository": "rustfs/s3chaos",
|
||||
"harness_sha": "9a48f55c1d6d58ce63a8fb1ea7317e024f32f11d",
|
||||
"harness_basis": "Fixed G01 evidence-envelope producer revision (rustfs/s3chaos PR #102 head recorded in rustfs/backlog#2658). R02 re-pins this when the live verifier ships.",
|
||||
"timeout_seconds": 1800,
|
||||
"environment_capabilities": ["process-exec"],
|
||||
"cases": ["artifact-identity", "startup"],
|
||||
"applies_to": {
|
||||
"release_types": ["preview", "prerelease", "stable"],
|
||||
"platforms": "all"
|
||||
},
|
||||
"applies_basis": "Artifact identity and startup checks are mandatory for every release type, preview included, on every shipped target (rustfs/backlog#2662 frozen design item 5)."
|
||||
},
|
||||
{
|
||||
"lane": "upgrade",
|
||||
"harness_repository": "rustfs/s3chaos",
|
||||
"harness_sha": "9a48f55c1d6d58ce63a8fb1ea7317e024f32f11d",
|
||||
"harness_basis": "Same fixed envelope producer as the identity lane; the workflow SHA and the product build SHA are bound separately and need not be equal.",
|
||||
"timeout_seconds": 5400,
|
||||
"environment_capabilities": ["process-exec", "previous-release-artifact"],
|
||||
"cases": [
|
||||
"direct-upgrade",
|
||||
"mixed-version-upgrade",
|
||||
"bucket-config-upgrade",
|
||||
"bucket-config-rollback",
|
||||
"multipart-layout-upgrade",
|
||||
"multipart-layout-baseline"
|
||||
],
|
||||
"cases_basis": "Case IDs of the pinned previous-release matrix in .github/workflows/e2e-upgrade.yml (source.version/asset/sha256 and the case list mapped there).",
|
||||
"applies_to": {
|
||||
"release_types": ["prerelease", "stable"],
|
||||
"platforms": [{"os": "linux", "arch": "x86_64", "abi": "gnu"}]
|
||||
},
|
||||
"applies_basis": "preview releases use the pre-declared smaller set (identity lane only). The upgrade lane is only exercised on linux-x86_64-gnu, the platform of the pinned source assets."
|
||||
}
|
||||
],
|
||||
"upgrade_sources": {
|
||||
"basis": "Exactly the two previous-release sources pinned in .github/workflows/e2e-upgrade.yml are exercised today (source.version / source.asset / source.sha256). No other N-1/N-2 compatibility is claimed, and these digests are not inferred from tags.",
|
||||
"sources": [
|
||||
{
|
||||
"version": "1.0.1",
|
||||
"role": "N-1",
|
||||
"artifact_name": "rustfs-linux-x86_64-gnu-v1.0.1.zip",
|
||||
"artifact_sha256": "9742e183cb0dfb0d48fa909797af572fe47b7f827569036baa4156e0d3958ee5"
|
||||
},
|
||||
{
|
||||
"version": "1.0.0-rc.5",
|
||||
"role": "N-2",
|
||||
"artifact_name": "rustfs-linux-x86_64-gnu-v1.0.0-rc.5.zip",
|
||||
"artifact_sha256": "3ee8df71e8edcfada533be452c4135868f697bc515460ae97b027313eade7a3d"
|
||||
}
|
||||
],
|
||||
"rollback_supported": false,
|
||||
"rollback_basis": "Only the bucket-config-rollback case (rollback_to_previous_release_reads_current_bucket_metadata) is exercised. Full data-plane rollback is not certified and is never a default claim."
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
# Files that may still reference s3s (`s3s::` or `s3s_sigv4`) when Phase 1 of
|
||||
# the gateway migration ends (rustfs/backlog#2734 task T0.6, rustfs/backlog#2741).
|
||||
# Consumed only by `scripts/check_s3s_footprint.sh --mode allowlist`.
|
||||
#
|
||||
# One entry per line: a repository-relative file path or a git `:(glob)`
|
||||
# pathspec. `**` matches across directories, `*` stays within one path
|
||||
# component, and a directory entry matches everything under it. Blank lines
|
||||
# and lines starting with `#` are ignored; an entry may not contain whitespace,
|
||||
# so a note belongs on its own comment line.
|
||||
#
|
||||
# Every Rust file listed by `git grep -l '\bs3s\(::\|_sigv4\)' -- '*.rs'` that
|
||||
# no entry matches fails the allowlist mode. Task T1.9 switches CI to that
|
||||
# mode; until then `--dry-run` prints the files still outside this list, which
|
||||
# is the Phase 1 tracking list.
|
||||
#
|
||||
# Never add crates/** other than crates/e2e_test: the engine crates leave s3s
|
||||
# entirely, and the script rejects such an entry. Only the s3s-facing edge of
|
||||
# the rustfs binary and the e2e oracle keep it.
|
||||
|
||||
# rustfs binary: the s3s-facing edge that survives Phase 1
|
||||
rustfs/src/storage/ecfs.rs
|
||||
rustfs/src/storage/s3_compat/**
|
||||
rustfs/src/storage/access.rs
|
||||
rustfs/src/auth.rs
|
||||
rustfs/src/error.rs
|
||||
rustfs/src/storage_api.rs
|
||||
rustfs/src/app/storage_api.rs
|
||||
rustfs/src/admin/router.rs
|
||||
rustfs/src/app/metadata_route.rs
|
||||
rustfs/src/server/http.rs
|
||||
rustfs/src/server/layer.rs
|
||||
rustfs/src/server/hybrid.rs
|
||||
rustfs/src/protocols/client.rs
|
||||
|
||||
# e2e oracle: the test target that verifies S3 behaviour against s3s
|
||||
crates/e2e_test/src/fake_s3_target/**
|
||||
crates/e2e_test/src/**
|
||||
@@ -0,0 +1,17 @@
|
||||
# Exact baseline for the direct s3s footprint, consumed only by
|
||||
# scripts/check_s3s_footprint.sh (rustfs/backlog#1677; rustfs/backlog#2734 T0.4).
|
||||
#
|
||||
# Each value is the exact measurement of the tree, not a ceiling. CI fails when a
|
||||
# measured counter differs from its line here in either direction:
|
||||
# - a larger measurement means new code widened the s3s surface: route it
|
||||
# through the gateway abstractions instead; never raise a value here;
|
||||
# - a smaller measurement means the PR shrank the surface: lower the matching
|
||||
# value to the measured number in the same PR.
|
||||
#
|
||||
# files Rust files referencing s3s paths, repo-wide minus crates/e2e_test
|
||||
# s3_error_lines lines holding s3_error!, same scope, minus those the script
|
||||
# header's rule clears (rustfs_s3_types's macro); fails closed
|
||||
# ecstore_files Rust files referencing s3s paths under crates/ecstore/src
|
||||
files=65
|
||||
s3_error_lines=289
|
||||
ecstore_files=0
|
||||
@@ -0,0 +1,65 @@
|
||||
# Per-crate test-count baseline consumed by scripts/check_test_count_ratchet.sh
|
||||
# (rustfs/backlog#2734 section 1 rule 3; T0.3 in rustfs/backlog#2738).
|
||||
#
|
||||
# One row per workspace member: <package>, tests, ignored, tab-separated.
|
||||
# tests counts #[test], #[tokio::test], #[tokio::test(...)] and #[test_case(...)]
|
||||
# attribute lines under the package directory; ignored counts #[ignore] and
|
||||
# #[ignore = "..."]. Both columns are exact: CI fails when a crate's tests fall
|
||||
# below or rise above its row, or when its ignored count changes. Raise a row in
|
||||
# the PR that adds tests; lowering one is a reviewed statement that tests were
|
||||
# intentionally removed. Regenerate with:
|
||||
# scripts/check_test_count_ratchet.sh --update
|
||||
e2e_test 993 20
|
||||
rustfs 6078 64
|
||||
rustfs-audit 70 0
|
||||
rustfs-checksums 26 0
|
||||
rustfs-common 36 0
|
||||
rustfs-concurrency 54 0
|
||||
rustfs-config 43 0
|
||||
rustfs-connect-symbol-catalog 6 0
|
||||
rustfs-credentials 24 0
|
||||
rustfs-crypto 67 0
|
||||
rustfs-data-usage 56 0
|
||||
rustfs-ecstore 7415 2
|
||||
rustfs-extension-schema 23 0
|
||||
rustfs-filemeta 353 0
|
||||
rustfs-gateway-adapter 11 0
|
||||
rustfs-heal 1033 3
|
||||
rustfs-heal-contracts 8 0
|
||||
rustfs-iam 401 0
|
||||
rustfs-io-core 49 0
|
||||
rustfs-io-metrics 154 0
|
||||
rustfs-keystone 29 0
|
||||
rustfs-kms 715 25
|
||||
rustfs-license 2 0
|
||||
rustfs-lifecycle 155 0
|
||||
rustfs-lock 143 0
|
||||
rustfs-log-analyzer 90 0
|
||||
rustfs-madmin 210 0
|
||||
rustfs-notify 139 0
|
||||
rustfs-object-capacity 92 0
|
||||
rustfs-object-data-cache 140 0
|
||||
rustfs-obs 385 0
|
||||
rustfs-policy 734 0
|
||||
rustfs-protocols 748 1
|
||||
rustfs-protos 73 0
|
||||
rustfs-replication 209 0
|
||||
rustfs-rio 206 0
|
||||
rustfs-rio-v2 34 6
|
||||
rustfs-s3-client 69 0
|
||||
rustfs-s3-ops 8 0
|
||||
rustfs-s3-types 103 0
|
||||
rustfs-s3select-api 207 0
|
||||
rustfs-s3select-query 189 2
|
||||
rustfs-scanner 1261 20
|
||||
rustfs-scanner-metrics 90 0
|
||||
rustfs-security-governance 20 0
|
||||
rustfs-signer 31 0
|
||||
rustfs-storage-api 34 0
|
||||
rustfs-targets 603 26
|
||||
rustfs-test-utils 1 0
|
||||
rustfs-tls-runtime 43 0
|
||||
rustfs-trusted-proxies 68 0
|
||||
rustfs-utils 247 0
|
||||
rustfs-zip 30 0
|
||||
zstd 11 0
|
||||
@@ -13,7 +13,7 @@
|
||||
# limitations under the License.
|
||||
|
||||
name: Quick Checks
|
||||
description: Run the shared RustFS quality checks.
|
||||
description: Run the shared compile-free RustFS quality checks.
|
||||
|
||||
runs:
|
||||
using: composite
|
||||
@@ -24,7 +24,6 @@ runs:
|
||||
tool: |
|
||||
ripgrep@15.2.0
|
||||
shellcheck@0.11.0
|
||||
cargo-nextest@0.9.138
|
||||
|
||||
- name: Install actionlint
|
||||
shell: bash
|
||||
@@ -38,15 +37,6 @@ runs:
|
||||
rm "$actionlint_dir/actionlint.tar.gz"
|
||||
echo "$actionlint_dir" >> "$GITHUB_PATH"
|
||||
|
||||
- name: Install Python
|
||||
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- name: Install workflow parser
|
||||
shell: bash
|
||||
run: python3 -m pip install --disable-pip-version-check PyYAML==6.0.3
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
|
||||
with:
|
||||
@@ -96,6 +86,10 @@ runs:
|
||||
shell: bash
|
||||
run: ./scripts/check_s3s_footprint.sh
|
||||
|
||||
- name: Check per-crate test-count ratchet
|
||||
shell: bash
|
||||
run: ./scripts/check_test_count_ratchet.sh
|
||||
|
||||
- name: Check cryptographic capability wording
|
||||
shell: bash
|
||||
run: ./scripts/check_fips_wording.sh
|
||||
@@ -114,10 +108,7 @@ runs:
|
||||
|
||||
- name: Check test wiring
|
||||
shell: bash
|
||||
run: |
|
||||
python3 ./scripts/check_test_wiring.py
|
||||
python3 ./scripts/ci_gate.py --check-workflow
|
||||
python3 ./scripts/validation_registry.py check
|
||||
run: python3 ./scripts/check_test_wiring.py
|
||||
|
||||
- name: Check no planning docs committed
|
||||
shell: bash
|
||||
|
||||
@@ -1,107 +0,0 @@
|
||||
name: Runner capability and build pilot
|
||||
description: Inspect an existing runner and measure isolated fixed-source cold/warm builds
|
||||
inputs:
|
||||
workload:
|
||||
description: probe, server or workspace-tests
|
||||
required: true
|
||||
source-sha:
|
||||
description: Exact merged main commit for builds
|
||||
default: ""
|
||||
toolchain:
|
||||
description: Exact Rust version for builds
|
||||
default: ""
|
||||
scratch-root:
|
||||
description: Existing absolute scratch root
|
||||
default: ""
|
||||
build-jobs:
|
||||
description: Cargo concurrency
|
||||
default: "2"
|
||||
local-ssd:
|
||||
description: Require operator storage attestation
|
||||
default: "false"
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Validate pilot input and inspect existing capabilities
|
||||
shell: bash
|
||||
env:
|
||||
PILOT_WORKLOAD: ${{ inputs.workload }}
|
||||
PILOT_SHA: ${{ inputs.source-sha }}
|
||||
PILOT_TOOLCHAIN: ${{ inputs.toolchain }}
|
||||
PILOT_SCRATCH: ${{ inputs.scratch-root }}
|
||||
PILOT_JOBS: ${{ inputs.build-jobs }}
|
||||
PILOT_SSD: ${{ inputs.local-ssd }}
|
||||
run: |
|
||||
[[ "$PILOT_WORKLOAD" =~ ^(probe|server|workspace-tests)$ ]]
|
||||
[[ "$PILOT_JOBS" =~ ^([1-9]|1[0-6])$ ]]
|
||||
if [[ "$PILOT_WORKLOAD" != probe ]]; then
|
||||
[[ "$PILOT_SHA" =~ ^[a-f0-9]{40}$ ]]
|
||||
[[ "$PILOT_TOOLCHAIN" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]
|
||||
fi
|
||||
scratch="${PILOT_SCRATCH:-$RUNNER_TEMP}"
|
||||
[[ "$scratch" == /* && -d "$scratch" ]]
|
||||
minimum_disk=1
|
||||
minimum_memory=0
|
||||
if [[ "$PILOT_WORKLOAD" == server ]]; then minimum_disk=24; minimum_memory=8; fi
|
||||
if [[ "$PILOT_WORKLOAD" == workspace-tests ]]; then minimum_disk=90; minimum_memory=16; fi
|
||||
extra=()
|
||||
if [[ "$PILOT_SSD" == true ]]; then extra+=(--local-ssd-attestation /etc/rustfs-runner/storage.json); fi
|
||||
python3 scripts/ci/runner_pilot.py --root "$scratch" \
|
||||
--output "$RUNNER_TEMP/runner-pilot-evidence" \
|
||||
--min-free-gib "$minimum_disk" --min-memory-gib "$minimum_memory" --architecture x86_64 "${extra[@]}"
|
||||
- name: Checkout the trusted baseline history
|
||||
if: inputs.workload != 'probe'
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
ref: main
|
||||
path: .runner-pilot-source
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
- name: Bind the build to a merged main commit
|
||||
if: inputs.workload != 'probe'
|
||||
shell: bash
|
||||
env:
|
||||
PILOT_SHA: ${{ inputs.source-sha }}
|
||||
run: |
|
||||
git -C .runner-pilot-source merge-base --is-ancestor "$PILOT_SHA" HEAD
|
||||
git -C .runner-pilot-source checkout --detach "$PILOT_SHA"
|
||||
- name: Install build dependencies
|
||||
if: inputs.workload != 'probe'
|
||||
shell: bash
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y build-essential pkg-config libssl-dev cmake clang time
|
||||
- name: Install the fixed protobuf compiler
|
||||
if: inputs.workload != 'probe'
|
||||
uses: rustfs/setup-protoc@a3705324d8f9bf5b6c3573fb6cf8ae421db55dd6 # v3.0.1
|
||||
with:
|
||||
version: "35.1"
|
||||
repo-token: ${{ github.token }}
|
||||
- name: Install the fixed flatbuffer compiler
|
||||
if: inputs.workload != 'probe'
|
||||
uses: Nugine/setup-flatc@698800de72a96bfb22cf60431dc21a2ff9a7e07b # v1
|
||||
with:
|
||||
version: "25.12.19"
|
||||
- name: Install the requested exact Rust toolchain
|
||||
if: inputs.workload != 'probe'
|
||||
uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
|
||||
with:
|
||||
toolchain: ${{ inputs.toolchain }}
|
||||
- name: Measure cold and warm builds without shared target caches
|
||||
if: inputs.workload != 'probe'
|
||||
shell: bash
|
||||
env:
|
||||
PILOT_WORKLOAD: ${{ inputs.workload }}
|
||||
PILOT_SHA: ${{ inputs.source-sha }}
|
||||
PILOT_TOOLCHAIN: ${{ inputs.toolchain }}
|
||||
PILOT_SCRATCH: ${{ inputs.scratch-root }}
|
||||
PILOT_JOBS: ${{ inputs.build-jobs }}
|
||||
PILOT_SSD: ${{ inputs.local-ssd }}
|
||||
run: |
|
||||
extra=()
|
||||
if [[ "$PILOT_SSD" == true ]]; then extra+=(--local-ssd-attestation /etc/rustfs-runner/storage.json); fi
|
||||
python3 scripts/ci/runner_pilot.py --root "${PILOT_SCRATCH:-$RUNNER_TEMP}" \
|
||||
--output "$RUNNER_TEMP/runner-pilot-evidence" \
|
||||
--source .runner-pilot-source --source-sha "$PILOT_SHA" \
|
||||
--toolchain "$PILOT_TOOLCHAIN" --benchmark "$PILOT_WORKLOAD" \
|
||||
--jobs "$PILOT_JOBS" --timeout-seconds 3600 "${extra[@]}"
|
||||
@@ -1,16 +0,0 @@
|
||||
name: Verification runtime
|
||||
description: Install the runtime for prebuilt test archives without a target cache
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Install runtime tools
|
||||
shell: bash
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y libssl-dev ripgrep
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
|
||||
- name: Install cargo-nextest
|
||||
uses: taiki-e/install-action@96c7780c1d8a2b8723e12031def873a434d39d8d # nextest
|
||||
with:
|
||||
tool: cargo-nextest@0.9.138
|
||||
@@ -32,8 +32,6 @@ updates:
|
||||
- "overtrue"
|
||||
- "majinghe"
|
||||
ignore:
|
||||
- dependency-name: "pulsar"
|
||||
versions: [ "6.9.x" ]
|
||||
- dependency-name: "object_store"
|
||||
versions: [ "0.13.x" ]
|
||||
- dependency-name: "libunftp"
|
||||
|
||||
+25
-1731
File diff suppressed because it is too large
Load Diff
@@ -32,7 +32,7 @@ A job runs on `ubuntu-latest` unless it has one of these reasons, and the reason
|
||||
|
||||
Jobs that do only the following have no reason to be self-hosted: shell or Python scripts, `cargo fmt`, `cargo deny`, typos, actionlint, API calls (`gh`, dependency review, issue and stale bots, CLA checks), downloading an artifact and running a prebuilt binary, or a `cargo build -p rustfs` with its own cache key. `e2e-distributed.yml` and `e2e-upgrade.yml` build the server and the `e2e_test` harness on `ubuntu-latest` inside their budgets, so "it compiles something" alone is not a reason.
|
||||
|
||||
The functional chain's `prepare` and `complete-chain` jobs, the performance runner availability check, and runner hygiene use `ubuntu-latest`: they inspect APIs or artifacts, not the lab hosts. The actual functional and performance suites remain on their lab runners. Functional-chain health collection uses `ubuntu-latest` so a lab outage cannot also stop its independent monitor. Docker image packaging also uses `ubuntu-latest` for both musl and glibc variants; it downloads release binaries rather than compiling the workspace, and still sets up QEMU and Buildx explicitly.
|
||||
The functional chain's `prepare` and `complete-chain` jobs, the performance runner availability check, and runner hygiene use `ubuntu-latest`: they inspect APIs or artifacts, not the lab hosts. The actual functional and performance suites remain on their lab runners. Functional-chain health collection stays on the operator-managed `smoke-testing` runner as part of the automation environment. Docker image packaging also uses `ubuntu-latest` for both musl and glibc variants; it downloads release binaries rather than compiling the workspace, and still sets up QEMU and Buildx explicitly.
|
||||
|
||||
Moving these jobs off AKS reduces capacity demand, not necessarily the Azure bill immediately. VM costs fall only when nodes scale down or smaller capacity replaces them.
|
||||
|
||||
|
||||
+5
-311
@@ -814,18 +814,13 @@ jobs:
|
||||
export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
|
||||
export AWS_DEFAULT_REGION="auto"
|
||||
|
||||
# Determine upload path based on build type. Release and prerelease
|
||||
# builds only stage their bytes here: the official
|
||||
# artifacts/rustfs/release/ path is written solely by the
|
||||
# certification-gated promote-release-artifacts job, from these same
|
||||
# staged objects. Development builds keep the uncertified dev channel
|
||||
# and never touch official or latest channels.
|
||||
# Determine upload path based on build type
|
||||
if [[ "$BUILD_TYPE" == "development" ]]; then
|
||||
R2_PATH="s3://${R2_BUCKET}/artifacts/rustfs/dev/"
|
||||
echo "📤 Uploading development build to R2 dev directory"
|
||||
else
|
||||
R2_PATH="s3://${R2_BUCKET}/artifacts/rustfs/staging/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}/"
|
||||
echo "📤 Staging release build for certification-gated promotion"
|
||||
R2_PATH="s3://${R2_BUCKET}/artifacts/rustfs/release/"
|
||||
echo "📤 Uploading release build to R2 release directory"
|
||||
fi
|
||||
|
||||
# Upload all rustfs zip files to R2 using S3-compatible API
|
||||
@@ -874,221 +869,6 @@ jobs:
|
||||
echo "This run does not trigger Docker publishing."
|
||||
fi
|
||||
|
||||
# Aggregate the parent candidate batch. This job always collects results and
|
||||
# retains its evidence, including on failure, and every publication job
|
||||
# below requires its success. Missing, stale, substituted or non-PASS
|
||||
# evidence fails closed here: only the trusted verification sources
|
||||
# (rustfs/s3chaos release-gate envelopes, rustfs/backlog#2658) can satisfy
|
||||
# it, so a PR-uploaded JSON set never gains release authority.
|
||||
release-certification:
|
||||
name: Release Certification
|
||||
needs: [ build-check, build-rustfs ]
|
||||
if: ${{ !cancelled() && needs.build-check.outputs.should_publish == 'true' && startsWith(github.ref, 'refs/tags/') }}
|
||||
runs-on: ubuntu-latest # gh CLI required; the sm-standard-2 fleet does not ship it
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Download all build artifacts
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
path: ./artifacts
|
||||
pattern: rustfs-*
|
||||
merge-multiple: true
|
||||
|
||||
- name: Write candidate and collect trusted validation results
|
||||
env:
|
||||
# Cross-repo read of the trusted verification run. Falls back to the
|
||||
# workflow token, which cannot read rustfs/s3chaos; a missing token
|
||||
# therefore fail-closes instead of certifying.
|
||||
GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
|
||||
PRODUCT_COMMIT: ${{ github.sha }}
|
||||
VERSION: ${{ needs.build-check.outputs.version }}
|
||||
BUILD_TYPE: ${{ needs.build-check.outputs.build_type }}
|
||||
FEATURES: ${{ needs.build-check.outputs.build_type == 'development' && '' || 'pyroscope' }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
mkdir -p results release-gate-output
|
||||
PLAN_SHA256=$(python3 -c "import hashlib,json,sys; sys.path.insert(0,'scripts/release'); import release_gate; print(release_gate.canonical_sha256(json.load(open('.config/release-validation-plan.json'))))")
|
||||
|
||||
python3 - "$PLAN_SHA256" > candidate.json <<'PY'
|
||||
import hashlib, json, os, sys, zipfile
|
||||
|
||||
plan_sha256 = sys.argv[1]
|
||||
profile = {"release": "stable", "prerelease": "prerelease", "preview": "preview"}.get(os.environ["BUILD_TYPE"])
|
||||
if profile is None:
|
||||
raise SystemExit("release certification only covers preview, prerelease and stable builds")
|
||||
|
||||
def sha256(path):
|
||||
digest = hashlib.sha256()
|
||||
with open(path, "rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
deliverables = []
|
||||
for name in sorted(os.listdir("artifacts")):
|
||||
if not name.startswith("rustfs-") or not name.endswith(".zip"):
|
||||
continue
|
||||
path = os.path.join("artifacts", name)
|
||||
members = zipfile.ZipFile(path).namelist()
|
||||
binary_name = "rustfs.exe" if "rustfs.exe" in members else "rustfs"
|
||||
with zipfile.ZipFile(path) as archive:
|
||||
binary_bytes = archive.read(binary_name)
|
||||
import tempfile
|
||||
with tempfile.NamedTemporaryFile() as handle:
|
||||
handle.write(binary_bytes)
|
||||
handle.flush()
|
||||
binary_sha = sha256(handle.name)
|
||||
deliverables.append({
|
||||
"kind": "zip",
|
||||
"name": name,
|
||||
"location": "https://github.com/rustfs/rustfs/releases/download/{}/{}".format(os.environ["VERSION"], name),
|
||||
"sha256": sha256(path),
|
||||
"size": os.path.getsize(path),
|
||||
"binary_sha256": binary_sha,
|
||||
})
|
||||
if not deliverables:
|
||||
raise SystemExit("no built deliverables found to certify")
|
||||
candidate = {
|
||||
"schema_version": 1,
|
||||
"product_commit": os.environ["PRODUCT_COMMIT"],
|
||||
"version": os.environ["VERSION"],
|
||||
"release_profile": profile,
|
||||
"features": [f for f in os.environ.get("FEATURES", "").split(",") if f],
|
||||
"plan_sha256": plan_sha256,
|
||||
"validation_batch": {"run_id": os.environ["GITHUB_RUN_ID"], "run_attempt": int(os.environ["GITHUB_RUN_ATTEMPT"])},
|
||||
"deliverables": deliverables,
|
||||
}
|
||||
json.dump(candidate, sys.stdout, indent=2, sort_keys=True)
|
||||
print()
|
||||
PY
|
||||
|
||||
# Trusted verification evidence only: the release-gate envelopes of
|
||||
# rustfs/s3chaos for this exact product commit. Absence blocks below.
|
||||
TRUSTED_REPO="rustfs/s3chaos"
|
||||
TRUSTED_WORKFLOW="release-gate.yml"
|
||||
TRUSTED_RUN=$(gh api "repos/${TRUSTED_REPO}/actions/workflows/${TRUSTED_WORKFLOW}/runs?head_sha=${PRODUCT_COMMIT}&status=success&per_page=1" --jq '.workflow_runs[0].id // empty' || true)
|
||||
if [[ -z "$TRUSTED_RUN" ]]; then
|
||||
echo "❌ No trusted verification run of ${TRUSTED_REPO}/${TRUSTED_WORKFLOW} for ${PRODUCT_COMMIT}; refusing to certify" >&2
|
||||
else
|
||||
gh run download "$TRUSTED_RUN" -R "$TRUSTED_REPO" -n release-validation-evidence -D results || \
|
||||
echo "❌ Trusted verification run ${TRUSTED_RUN} published no release-validation-evidence artifact" >&2
|
||||
fi
|
||||
|
||||
python3 scripts/release/release_gate.py validate \
|
||||
--plan .config/release-validation-plan.json \
|
||||
--candidate candidate.json \
|
||||
--results results \
|
||||
--output release-gate-output
|
||||
|
||||
- name: Upload certification evidence
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: release-certification
|
||||
path: |
|
||||
candidate.json
|
||||
results/
|
||||
release-gate-output/
|
||||
if-no-files-found: warn
|
||||
retention-days: 30
|
||||
|
||||
# The only writer of the official R2 release path. Promotion re-checks the
|
||||
# exact bytes about to be published against the passed certification list
|
||||
# and copies the staged objects byte-for-byte: no rebuild-after-verify.
|
||||
promote-release-artifacts:
|
||||
name: Promote Release Artifacts
|
||||
needs: [ build-check, build-rustfs, release-certification ]
|
||||
# Target-channel lock for the official R2 release path.
|
||||
concurrency:
|
||||
group: release-channel-r2-release
|
||||
cancel-in-progress: false
|
||||
if: needs.build-check.outputs.should_publish == 'true' && startsWith(github.ref, 'refs/tags/') && (needs.build-check.outputs.build_type == 'release' || needs.build-check.outputs.build_type == 'prerelease')
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
env:
|
||||
R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
|
||||
R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
|
||||
R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }}
|
||||
R2_BUCKET: ${{ secrets.R2_BUCKET }}
|
||||
AWS_EC2_METADATA_DISABLED: true
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Download build and certification artifacts
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
path: ./artifacts
|
||||
pattern: rustfs-*
|
||||
merge-multiple: true
|
||||
|
||||
- name: Download certification evidence
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: release-certification
|
||||
path: ./certification
|
||||
|
||||
- name: Verify and promote staged release artifacts
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
if [[ -z "$R2_ACCESS_KEY_ID" || -z "$R2_SECRET_ACCESS_KEY" || -z "$R2_ENDPOINT" || -z "$R2_BUCKET" ]]; then
|
||||
echo "❌ R2 credentials missing; refusing to promote to the official channel" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
for zip_file in artifacts/rustfs-*.zip; do
|
||||
[[ -f "$zip_file" ]] || continue
|
||||
name="$(basename "$zip_file")"
|
||||
python3 scripts/release/release_gate.py verify-promotion \
|
||||
--plan .config/release-validation-plan.json \
|
||||
--candidate certification/candidate.json \
|
||||
--summary certification/release-gate-output \
|
||||
--artifact "$zip_file" \
|
||||
--artifact-name "$name"
|
||||
done
|
||||
|
||||
if ! command -v aws >/dev/null 2>&1; then
|
||||
sudo apt-get update && sudo apt-get install -y awscli
|
||||
fi
|
||||
export AWS_ACCESS_KEY_ID="$R2_ACCESS_KEY_ID"
|
||||
export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
|
||||
export AWS_DEFAULT_REGION="auto"
|
||||
|
||||
STAGING_PATH="s3://${R2_BUCKET}/artifacts/rustfs/staging/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}/"
|
||||
RELEASE_PATH="s3://${R2_BUCKET}/artifacts/rustfs/release/"
|
||||
for zip_file in artifacts/rustfs-*.zip; do
|
||||
[[ -f "$zip_file" ]] || continue
|
||||
name="$(basename "$zip_file")"
|
||||
staged="$(mktemp)"
|
||||
aws s3 cp "${STAGING_PATH}${name}" "$staged" --endpoint-url "$R2_ENDPOINT" --only-show-errors
|
||||
staged_sha="$(sha256sum "$staged" | awk '{print $1}')"
|
||||
built_sha="$(sha256sum "$zip_file" | awk '{print $1}')"
|
||||
if [[ "$staged_sha" != "$built_sha" ]]; then
|
||||
echo "❌ Staged object ${name} does not match the verified bytes" >&2
|
||||
exit 1
|
||||
fi
|
||||
aws s3 cp "$staged" "${RELEASE_PATH}${name}" --endpoint-url "$R2_ENDPOINT" --only-show-errors
|
||||
rm -f "$staged"
|
||||
echo "✅ Promoted ${name} (${built_sha})"
|
||||
done
|
||||
|
||||
# Create GitHub Release for every valid release tag, including previews
|
||||
create-release:
|
||||
name: Create GitHub Release
|
||||
@@ -1248,10 +1028,6 @@ jobs:
|
||||
update-latest-version:
|
||||
name: Update Latest Version
|
||||
needs: [ build-check, publish-release ]
|
||||
# Target-channel lock for the latest.json version pointer.
|
||||
concurrency:
|
||||
group: release-channel-oss-latest
|
||||
cancel-in-progress: false
|
||||
if: needs.build-check.outputs.should_publish == 'true' && startsWith(github.ref, 'refs/tags/') && (needs.build-check.outputs.build_type == 'release' || needs.build-check.outputs.build_type == 'prerelease')
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
@@ -1292,45 +1068,6 @@ jobs:
|
||||
rm -rf "$OSSUTIL_DIR" "$OSSUTIL_ZIP"
|
||||
chmod +x "$OSSUTIL_BIN"
|
||||
|
||||
# Candidate/pointer ordering: an older candidate must never overwrite
|
||||
# a newer pointer, even if its promotion job finishes later.
|
||||
CURRENT_LATEST="${RUNNER_TEMP}/current-latest.json"
|
||||
if "$OSSUTIL_BIN" cp oss://rustfs-version/latest.json "$CURRENT_LATEST" 2>/dev/null; then
|
||||
python3 - "$CURRENT_LATEST" "$VERSION" <<'PY'
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
|
||||
def key(version):
|
||||
match = re.match(r"^(\d+)\.(\d+)\.(\d+)(?:-([0-9A-Za-z.-]+))?$", version)
|
||||
if not match:
|
||||
raise SystemExit(3)
|
||||
core = tuple(int(part) for part in match.groups()[:3])
|
||||
prerelease = match.group(4)
|
||||
if prerelease is None:
|
||||
return core + (1,)
|
||||
return core + (0, tuple(int(p) if p.isdigit() else p for p in re.split(r"[.-]", prerelease)))
|
||||
|
||||
try:
|
||||
current = json.load(open(sys.argv[1], encoding="utf-8")).get("version", "")
|
||||
proposed = sys.argv[2]
|
||||
if current and key(current) > key(proposed):
|
||||
raise SystemExit(1)
|
||||
except SystemExit:
|
||||
raise
|
||||
except (OSError, ValueError, TypeError):
|
||||
raise SystemExit(3)
|
||||
PY
|
||||
guard_status=$?
|
||||
if [[ "$guard_status" -eq 1 ]]; then
|
||||
echo "Current latest.json points at a newer version; refusing to downgrade" >&2
|
||||
exit 0
|
||||
elif [[ "$guard_status" -ne 0 ]]; then
|
||||
echo "Existing latest.json is unparsable; refusing to overwrite it blindly" >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# Create latest.json
|
||||
cat > latest.json << EOF
|
||||
{
|
||||
@@ -1347,59 +1084,16 @@ jobs:
|
||||
|
||||
echo "✅ Updated latest.json for ${RELEASE_TYPE} release $VERSION"
|
||||
|
||||
# Publish release (remove draft status). Requires the release-certification
|
||||
# aggregate: without a complete legal success there is no public write.
|
||||
# Publish release (remove draft status)
|
||||
publish-release:
|
||||
name: Publish Release
|
||||
needs: [ build-check, create-release, upload-release-assets, release-certification ]
|
||||
# Target-channel lock: concurrent releases serialize on this channel so an
|
||||
# older candidate cannot interleave its publish with a newer one.
|
||||
concurrency:
|
||||
group: release-channel-github-release
|
||||
cancel-in-progress: false
|
||||
needs: [ build-check, create-release, upload-release-assets ]
|
||||
if: needs.build-check.outputs.should_publish == 'true' && startsWith(github.ref, 'refs/tags/') && (needs.build-check.outputs.build_type == 'preview' || needs.build-check.outputs.build_type == 'release' || needs.build-check.outputs.build_type == 'prerelease')
|
||||
runs-on: ubuntu-latest # gh CLI required; the sm-standard-2 fleet does not ship it
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
contents: write
|
||||
actions: read
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Download certification evidence
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: release-certification
|
||||
path: ./certification
|
||||
|
||||
- name: Verify release assets against certification
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
TAG="${{ needs.build-check.outputs.version }}"
|
||||
mkdir -p "$RUNNER_TEMP/promotion-assets"
|
||||
gh release download "$TAG" --dir "$RUNNER_TEMP/promotion-assets" --pattern '*.zip'
|
||||
found=0
|
||||
for zip_file in "$RUNNER_TEMP"/promotion-assets/*.zip; do
|
||||
[[ -f "$zip_file" ]] || continue
|
||||
found=1
|
||||
python3 scripts/release/release_gate.py verify-promotion \
|
||||
--plan .config/release-validation-plan.json \
|
||||
--candidate certification/candidate.json \
|
||||
--summary certification/release-gate-output \
|
||||
--artifact "$zip_file" \
|
||||
--artifact-name "$(basename "$zip_file")"
|
||||
done
|
||||
if [[ "$found" -ne 1 ]]; then
|
||||
echo "No release zip assets to verify against the certification" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Publish release
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
|
||||
@@ -96,7 +96,7 @@ env:
|
||||
RUST_BACKTRACE: 1
|
||||
|
||||
jobs:
|
||||
# Readers: workspace-lint-and-build, workspace-tests (workspace and ILM), build-rustfs-debug-binary,
|
||||
# Readers: test-and-lint (workspace and ILM), build-rustfs-debug-binary,
|
||||
# e2e-tests, e2e-full.
|
||||
warm-ci-dev:
|
||||
name: Warm ci-dev
|
||||
@@ -146,7 +146,7 @@ jobs:
|
||||
if-no-files-found: error
|
||||
|
||||
# --all-targets covers the test binaries nextest builds, including
|
||||
# e2e_test, which the workspace run excludes. The second build adds
|
||||
# e2e_test, which test-and-lint's own run excludes. The second build adds
|
||||
# the e2e-test-hooks feature resolution that build-rustfs-debug-binary uses
|
||||
# and that no lint lane enables.
|
||||
- name: Build ci-dev superset
|
||||
|
||||
@@ -1,60 +0,0 @@
|
||||
name: CI Resource Accounting
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
since:
|
||||
description: "Inclusive run creation timestamp, for example 2026-10-01T00:00:00Z"
|
||||
required: true
|
||||
type: string
|
||||
until:
|
||||
description: "Inclusive end timestamp; keep the window below 1000 runs"
|
||||
required: true
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
pull-requests: read
|
||||
|
||||
concurrency:
|
||||
group: ci-resource-accounting
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
collect:
|
||||
if: github.repository == 'rustfs/rustfs'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Collect all workflow attempts in the window
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
ACCOUNTING_SINCE: ${{ inputs.since }}
|
||||
ACCOUNTING_UNTIL: ${{ inputs.until }}
|
||||
ACCOUNTING_REPOSITORY: ${{ github.repository }}
|
||||
run: |
|
||||
python3 scripts/ci/actions_cost_report.py \
|
||||
--repository "$ACCOUNTING_REPOSITORY" \
|
||||
--since "$ACCOUNTING_SINCE" --until "$ACCOUNTING_UNTIL" \
|
||||
--output "$RUNNER_TEMP/ci-accounting.json" \
|
||||
--summary "$RUNNER_TEMP/ci-accounting.md"
|
||||
- name: Show available accounting summary
|
||||
if: always()
|
||||
run: |
|
||||
if [[ -f "$RUNNER_TEMP/ci-accounting.md" ]]; then
|
||||
cat "$RUNNER_TEMP/ci-accounting.md" >> "$GITHUB_STEP_SUMMARY"
|
||||
fi
|
||||
- name: Retain recomputable accounting evidence
|
||||
if: always()
|
||||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
|
||||
with:
|
||||
name: ci-accounting-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: |
|
||||
${{ runner.temp }}/ci-accounting.json
|
||||
${{ runner.temp }}/ci-accounting.md
|
||||
if-no-files-found: error
|
||||
retention-days: 30
|
||||
@@ -1,92 +0,0 @@
|
||||
name: CI Runner Pilot
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
pool:
|
||||
description: "Existing runner pool; no infrastructure is created"
|
||||
required: true
|
||||
type: choice
|
||||
options: [hosted, arc]
|
||||
default: hosted
|
||||
workload:
|
||||
description: "Probe only, or perform cold/warm fixed-source builds"
|
||||
required: true
|
||||
type: choice
|
||||
options: [probe, server, workspace-tests]
|
||||
default: probe
|
||||
source_sha:
|
||||
description: "Exact 40-character merged main SHA; required for builds"
|
||||
type: string
|
||||
toolchain:
|
||||
description: "Exact Rust version such as 1.92.0; required for builds"
|
||||
type: string
|
||||
scratch_root:
|
||||
description: "Existing absolute scratch directory; empty uses runner.temp"
|
||||
type: string
|
||||
build_jobs:
|
||||
description: "Fixed Cargo concurrency for both comparisons (1-16)"
|
||||
type: string
|
||||
default: "2"
|
||||
local_ssd:
|
||||
description: "Require operator /etc/rustfs-runner/storage.json attestation"
|
||||
type: boolean
|
||||
default: false
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ci-runner-pilot
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
hosted:
|
||||
if: github.repository == 'rustfs/rustfs' && github.ref == 'refs/heads/main' && inputs.pool == 'hosted'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 150
|
||||
steps:
|
||||
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: ./.github/actions/runner-pilot
|
||||
with:
|
||||
workload: ${{ inputs.workload }}
|
||||
source-sha: ${{ inputs.source_sha }}
|
||||
toolchain: ${{ inputs.toolchain }}
|
||||
scratch-root: ${{ inputs.scratch_root }}
|
||||
build-jobs: ${{ inputs.build_jobs }}
|
||||
local-ssd: ${{ inputs.local_ssd }}
|
||||
- name: Retain hosted pilot evidence
|
||||
if: always()
|
||||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
|
||||
with:
|
||||
name: runner-pilot-hosted-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/runner-pilot-evidence
|
||||
if-no-files-found: error
|
||||
retention-days: 30
|
||||
arc:
|
||||
if: github.repository == 'rustfs/rustfs' && github.ref == 'refs/heads/main' && inputs.pool == 'arc'
|
||||
# Disk and hardware: measure the existing workspace build pool's storage.
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 150
|
||||
steps:
|
||||
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: ./.github/actions/runner-pilot
|
||||
with:
|
||||
workload: ${{ inputs.workload }}
|
||||
source-sha: ${{ inputs.source_sha }}
|
||||
toolchain: ${{ inputs.toolchain }}
|
||||
scratch-root: ${{ inputs.scratch_root }}
|
||||
build-jobs: ${{ inputs.build_jobs }}
|
||||
local-ssd: ${{ inputs.local_ssd }}
|
||||
- name: Retain ARC pilot evidence
|
||||
if: always()
|
||||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
|
||||
with:
|
||||
name: runner-pilot-arc-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/runner-pilot-evidence
|
||||
if-no-files-found: error
|
||||
retention-days: 30
|
||||
+273
-419
File diff suppressed because it is too large
Load Diff
@@ -329,75 +329,13 @@ jobs:
|
||||
echo " - Create latest: $create_latest"
|
||||
echo " - Source ref: $source_ref"
|
||||
|
||||
# Consume the trusted release certification for the exact version this
|
||||
# workflow wraps and verify the certified input digests. A manual dispatch
|
||||
# without that certification fails here and pushes no official tag; only
|
||||
# results from the trusted verification sources carry release authority.
|
||||
release-certification:
|
||||
name: Release Certification
|
||||
needs: build-check
|
||||
if: needs.build-check.outputs.should_build == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
ref: ${{ needs.build-check.outputs.source_ref }}
|
||||
|
||||
- name: Download and verify release certification
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
|
||||
TRIGGERING_RUN_ID: ${{ github.event.workflow_run.id }}
|
||||
VERSION: ${{ needs.build-check.outputs.version }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mkdir -p certification
|
||||
run_id="$TRIGGERING_RUN_ID"
|
||||
if [[ -z "$run_id" ]]; then
|
||||
run_id="$(gh api --method GET \
|
||||
"repos/${GITHUB_REPOSITORY}/actions/workflows/build.yml/runs" \
|
||||
-f branch="$VERSION" -f status=success -F per_page=1 \
|
||||
--jq '.workflow_runs[0].id // empty' || true)"
|
||||
fi
|
||||
if [[ -z "$run_id" ]]; then
|
||||
echo "No certified Build and Release run found for ${VERSION}; failing closed" >&2
|
||||
exit 1
|
||||
fi
|
||||
if ! gh run download "$run_id" -n release-certification -D certification; then
|
||||
echo "Build run ${run_id} published no release-certification artifact; failing closed" >&2
|
||||
exit 1
|
||||
fi
|
||||
python3 scripts/release/release_gate.py verify-promotion \
|
||||
--plan .config/release-validation-plan.json \
|
||||
--candidate certification/candidate.json \
|
||||
--summary certification/release-gate-output \
|
||||
--artifact-sha256 "$(jq -r '.deliverables[0].sha256' certification/candidate.json)" \
|
||||
--artifact-name "$(jq -r '.deliverables[0].name' certification/candidate.json)"
|
||||
|
||||
# Input digest binding: every zip this image wraps must be the
|
||||
# certified byte-for-byte content, never a mutable latest pointer.
|
||||
while IFS=$'\t' read -r name location expected; do
|
||||
echo "Verifying certified input ${name}"
|
||||
actual="$(curl -fsSL "$location" | sha256sum | awk '{print $1}')"
|
||||
if [[ "$actual" != "$expected" ]]; then
|
||||
echo "Certified input ${name} does not match its recorded digest" >&2
|
||||
exit 1
|
||||
fi
|
||||
done < <(jq -r '.deliverables[] | select(.name | contains("linux")) | [.name, .location, .sha256] | @tsv' certification/candidate.json)
|
||||
|
||||
# Build multi-arch Docker images
|
||||
# Strategy: Build images using pre-built binaries from dl.rustfs.com
|
||||
# Supports both release and dev channel binaries based on build context
|
||||
# Only runs when should_build is true (which includes workflow success check)
|
||||
build-docker:
|
||||
name: Build Docker Images
|
||||
needs: [ build-check, release-certification ]
|
||||
needs: build-check
|
||||
if: needs.build-check.outputs.should_build == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 60
|
||||
@@ -505,14 +443,6 @@ jobs:
|
||||
# Output tags
|
||||
echo "tags=$TAGS" >> "$GITHUB_OUTPUT"
|
||||
|
||||
# Staging identity for this run's promotion: official tags are only
|
||||
# created after the staged image verifies against the certification,
|
||||
# and they point at this same digest (no rebuild-after-verify).
|
||||
STAGING_TAG="staging-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}${VARIANT_SUFFIX}"
|
||||
STAGING_TAGS="${{ env.REGISTRY_DOCKERHUB }}:${STAGING_TAG},${{ env.REGISTRY_GHCR }}:${STAGING_TAG},${{ env.REGISTRY_QUAY }}:${STAGING_TAG}"
|
||||
echo "staging_tags=$STAGING_TAGS" >> "$GITHUB_OUTPUT"
|
||||
echo "staging_tag=$STAGING_TAG" >> "$GITHUB_OUTPUT"
|
||||
|
||||
# Generate labels
|
||||
LABELS="org.opencontainers.image.title=RustFS"
|
||||
LABELS="$LABELS,org.opencontainers.image.description=RustFS distributed object storage system"
|
||||
@@ -537,9 +467,7 @@ jobs:
|
||||
file: ${{ matrix.file }}
|
||||
platforms: ${{ env.DOCKER_PLATFORMS }}
|
||||
push: ${{ needs.build-check.outputs.should_push == 'true' }}
|
||||
# Push to staging tags only: official tags are created by the
|
||||
# certification-gated promotion step below from this same digest.
|
||||
tags: ${{ steps.meta.outputs.staging_tags }}
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
# No layer cache. This build compiles nothing — it downloads a
|
||||
# release zip and runs apk/apt — so the cache could only save the
|
||||
@@ -565,114 +493,6 @@ jobs:
|
||||
no-cache: false
|
||||
pull: true
|
||||
|
||||
# A later-built wrapper: the staged image must complete this platform's
|
||||
# own binary identity verification before any official tag is written.
|
||||
# It may not inherit an unverified package's conclusion.
|
||||
- name: Verify staged image against certification
|
||||
if: needs.build-check.outputs.should_push == 'true'
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
|
||||
TRIGGERING_RUN_ID: ${{ github.event.workflow_run.id }}
|
||||
VERSION: ${{ needs.build-check.outputs.version }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mkdir -p certification
|
||||
run_id="$TRIGGERING_RUN_ID"
|
||||
if [[ -z "$run_id" ]]; then
|
||||
run_id="$(gh api --method GET \
|
||||
"repos/${GITHUB_REPOSITORY}/actions/workflows/build.yml/runs" \
|
||||
-f branch="$VERSION" -f status=success -F per_page=1 \
|
||||
--jq '.workflow_runs[0].id // empty' || true)"
|
||||
fi
|
||||
gh run download "$run_id" -n release-certification -D certification
|
||||
|
||||
STAGING_REF="${{ env.REGISTRY_DOCKERHUB }}:staging-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}${{ matrix.suffix }}"
|
||||
case "${{ matrix.variant }}" in
|
||||
musl) libc="musl" ;;
|
||||
glibc) libc="gnu" ;;
|
||||
esac
|
||||
for platform in linux/amd64 linux/arm64; do
|
||||
case "$platform" in
|
||||
linux/amd64) arch_sub="x86_64" ;;
|
||||
linux/arm64) arch_sub="aarch64" ;;
|
||||
esac
|
||||
docker pull --platform "$platform" "$STAGING_REF"
|
||||
container="$(docker create --platform "$platform" "$STAGING_REF")"
|
||||
docker cp "$container:/usr/bin/rustfs" "$RUNNER_TEMP/staged-rustfs"
|
||||
docker rm "$container" >/dev/null
|
||||
actual="$(sha256sum "$RUNNER_TEMP/staged-rustfs" | awk '{print $1}')"
|
||||
expected="$(jq -r --arg needle "linux-${arch_sub}-${libc}" \
|
||||
'[.deliverables[] | select(.name | contains($needle)) | .binary_sha256][0] // empty' \
|
||||
certification/candidate.json)"
|
||||
if [[ -z "$expected" || "$actual" != "$expected" ]]; then
|
||||
echo "Staged image binary for ${platform} does not match the certified binary digest" >&2
|
||||
exit 1
|
||||
fi
|
||||
python3 scripts/release/release_gate.py verify-promotion \
|
||||
--plan .config/release-validation-plan.json \
|
||||
--candidate certification/candidate.json \
|
||||
--summary certification/release-gate-output \
|
||||
--artifact "$RUNNER_TEMP/staged-rustfs" \
|
||||
--artifact-role binary
|
||||
done
|
||||
|
||||
- name: Promote staged image to official tags
|
||||
if: needs.build-check.outputs.should_push == 'true'
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
STAGING_REF="${{ env.REGISTRY_DOCKERHUB }}:staging-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}${{ matrix.suffix }}"
|
||||
IFS=',' read -r -a official_tags <<< "${{ steps.meta.outputs.tags }}"
|
||||
for tag in "${official_tags[@]}"; do
|
||||
docker buildx imagetools create -t "$tag" "$STAGING_REF"
|
||||
echo "Promoted $tag -> $STAGING_REF"
|
||||
done
|
||||
|
||||
# Downstream consumers (Helm) verify this record instead of inheriting
|
||||
# an unverified wrapper's conclusion.
|
||||
- name: Record image promotion
|
||||
if: needs.build-check.outputs.should_push == 'true'
|
||||
shell: bash
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
|
||||
TRIGGERING_RUN_ID: ${{ github.event.workflow_run.id }}
|
||||
VERSION: ${{ needs.build-check.outputs.version }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mkdir -p certification
|
||||
run_id="$TRIGGERING_RUN_ID"
|
||||
if [[ -z "$run_id" ]]; then
|
||||
run_id="$(gh api --method GET \
|
||||
"repos/${GITHUB_REPOSITORY}/actions/workflows/build.yml/runs" \
|
||||
-f branch="$VERSION" -f status=success -F per_page=1 \
|
||||
--jq '.workflow_runs[0].id // empty' || true)"
|
||||
fi
|
||||
gh run download "$run_id" -n release-certification -D certification
|
||||
STAGING_REF="${{ env.REGISTRY_DOCKERHUB }}:staging-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}${{ matrix.suffix }}"
|
||||
digest="$(docker buildx imagetools inspect "$STAGING_REF" --format '{{json .Manifest}}' | jq -r '.digest')"
|
||||
candidate_sha="$(python3 -c "import json,hashlib,sys; sys.path.insert(0,'scripts/release'); import release_gate; print(release_gate.canonical_sha256(json.load(open('certification/candidate.json'))))")"
|
||||
jq -n \
|
||||
--arg version "$VERSION" \
|
||||
--arg digest "$digest" \
|
||||
--arg staging_ref "$STAGING_REF" \
|
||||
--arg candidate_sha256 "$candidate_sha" \
|
||||
--arg tags "${{ steps.meta.outputs.tags }}" \
|
||||
--slurpfile candidate certification/candidate.json \
|
||||
'{schema_version: 1, version: $version, image_digest: $digest, staging_ref: $staging_ref,
|
||||
promoted_tags: ($tags | split(",")), candidate_sha256: $candidate_sha256,
|
||||
binary_sha256s: ($candidate[0].deliverables | map(.binary_sha256) | unique)}' \
|
||||
> image-promotion-record.json
|
||||
|
||||
- name: Upload image promotion record
|
||||
if: needs.build-check.outputs.should_push == 'true'
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: image-promotion-record${{ matrix.suffix }}
|
||||
path: image-promotion-record.json
|
||||
if-no-files-found: error
|
||||
retention-days: 30
|
||||
|
||||
# Note: Manifest creation is no longer needed as we only build one variant
|
||||
# Multi-arch manifests are automatically created by docker/build-push-action
|
||||
|
||||
|
||||
@@ -17,8 +17,8 @@
|
||||
# Each selected test starts a real localhost cluster via
|
||||
# `RustFSTestClusterEnvironment` (4 processes; 4 drives per node unless the
|
||||
# case is a two-site 4-node 1-drive pair or a 4-node upgrade). Membership is
|
||||
# `[profile.e2e-distributed]` in `.config/nextest.toml`. The nightly schedule
|
||||
# and manual dispatches execute the same fail-closed suite.
|
||||
# `[profile.e2e-distributed]` in `.config/nextest.toml`. Storage-sensitive PRs,
|
||||
# nightly runs, and manual dispatches all execute the same fail-closed suite.
|
||||
# Upgrade cases download the same pinned previous release as e2e-upgrade.yml.
|
||||
#
|
||||
# Isolated pool filesystems: expand/decommission/rebalance cases require
|
||||
@@ -33,6 +33,32 @@
|
||||
name: e2e-distributed
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- "Cargo.lock"
|
||||
- "Cargo.toml"
|
||||
- ".config/nextest.toml"
|
||||
- ".config/e2e-distributed-selection.txt"
|
||||
- ".github/workflows/e2e-distributed.yml"
|
||||
- "crates/audit/**"
|
||||
- "crates/common/**"
|
||||
- "crates/config/**"
|
||||
- "crates/e2e_test/**"
|
||||
- "crates/ecstore/**"
|
||||
- "crates/filemeta/**"
|
||||
- "crates/heal/**"
|
||||
- "crates/iam/**"
|
||||
- "crates/lock/**"
|
||||
- "crates/madmin/**"
|
||||
- "crates/notify/**"
|
||||
- "crates/replication/**"
|
||||
- "crates/s3-client/**"
|
||||
- "crates/s3-ops/**"
|
||||
- "crates/s3-types/**"
|
||||
- "crates/scanner/**"
|
||||
- "crates/storage-api/**"
|
||||
- "crates/utils/**"
|
||||
- "rustfs/**"
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
filter:
|
||||
|
||||
@@ -12,19 +12,30 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Single-node upgrade and rollback lane.
|
||||
#
|
||||
# The weekly Monday schedule, version tags, and workflow_dispatch run this
|
||||
# suite. Distributed upgrade cases stay on the nightly e2e-distributed lane.
|
||||
|
||||
name: Upgrade Compatibility
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- ".github/workflows/e2e-upgrade.yml"
|
||||
- "crates/e2e_test/src/common.rs"
|
||||
- "crates/e2e_test/src/distributed/**"
|
||||
- "crates/e2e_test/src/fake_s3_target/**"
|
||||
- "crates/e2e_test/src/lib.rs"
|
||||
- "crates/e2e_test/src/replication_extension_test.rs"
|
||||
- "crates/e2e_test/src/rpc_fault_proxy.rs"
|
||||
- "crates/e2e_test/src/upgrade_compatibility_test.rs"
|
||||
- "crates/ecstore/**"
|
||||
- "crates/filemeta/**"
|
||||
- "crates/heal/**"
|
||||
- "crates/kms/**"
|
||||
- "crates/storage-api/**"
|
||||
- "rustfs/**"
|
||||
- "Cargo.lock"
|
||||
push:
|
||||
tags:
|
||||
- "[0-9]*.[0-9]*.[0-9]*"
|
||||
schedule:
|
||||
# 03:17 UTC every Monday.
|
||||
- cron: "17 3 * * 1"
|
||||
workflow_dispatch:
|
||||
|
||||
@@ -33,7 +44,7 @@ permissions:
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: false
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
||||
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
|
||||
@@ -12,13 +12,44 @@ concurrency:
|
||||
jobs:
|
||||
collect:
|
||||
if: github.repository == 'rustfs/rustfs'
|
||||
# Health remains available when either laboratory fleet is offline.
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: smoke-testing
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Ensure GitHub CLI
|
||||
shell: bash
|
||||
run: |
|
||||
if command -v gh >/dev/null 2>&1; then
|
||||
gh --version
|
||||
exit 0
|
||||
fi
|
||||
|
||||
gh_version="2.101.0"
|
||||
case "$(uname -m)" in
|
||||
x86_64) gh_arch="amd64" ;;
|
||||
aarch64|arm64) gh_arch="arm64" ;;
|
||||
*)
|
||||
echo "Unsupported Linux architecture for GitHub CLI: $(uname -m)" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
gh_install_root="${RUNNER_TEMP:-$PWD}/gh-cli"
|
||||
rm -rf "$gh_install_root"
|
||||
mkdir -p "$gh_install_root/bin"
|
||||
archive="$gh_install_root/gh.tar.gz"
|
||||
curl --fail --location --retry 3 \
|
||||
"https://github.com/cli/cli/releases/download/v${gh_version}/gh_${gh_version}_linux_${gh_arch}.tar.gz" \
|
||||
--output "$archive"
|
||||
tar -xzf "$archive" -C "$gh_install_root"
|
||||
install -m 0755 \
|
||||
"$gh_install_root/gh_${gh_version}_linux_${gh_arch}/bin/gh" \
|
||||
"$gh_install_root/bin/gh"
|
||||
echo "$gh_install_root/bin" >> "$GITHUB_PATH"
|
||||
export PATH="$gh_install_root/bin:$PATH"
|
||||
gh --version
|
||||
- name: Collect and publish verified chain health
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
|
||||
@@ -30,102 +30,9 @@ permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
# Consume the same trusted release certification as every other publication
|
||||
# exit and verify the image promotion record of the Docker workflow. The
|
||||
# chart wraps no binary of its own, so it may only reference an image whose
|
||||
# platform verification already passed; it must not inherit an unverified
|
||||
# package's conclusion. A run without that evidence fails closed here and
|
||||
# never reaches the chart repository.
|
||||
release-certification:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
outputs:
|
||||
version: ${{ steps.version.outputs.raw_tag }}
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Normalize release version
|
||||
id: version
|
||||
env:
|
||||
RAW_INPUT: ${{ github.event.inputs.version }}
|
||||
RAW_BRANCH: ${{ github.event.workflow_run.head_branch }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
|
||||
RAW="$RAW_INPUT"
|
||||
else
|
||||
RAW="$RAW_BRANCH"
|
||||
fi
|
||||
case "$RAW" in
|
||||
refs/tags/*) RAW_TAG="${RAW#refs/tags/}" ;;
|
||||
*) RAW_TAG="$RAW" ;;
|
||||
esac
|
||||
echo "raw_tag=$RAW_TAG" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Download and verify release certification and image record
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
|
||||
TRIGGERING_RUN_ID: ${{ github.event.workflow_run.id }}
|
||||
RAW_TAG: ${{ steps.version.outputs.raw_tag }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mkdir -p certification image-record
|
||||
|
||||
build_run="$TRIGGERING_RUN_ID"
|
||||
if [[ -z "$build_run" ]]; then
|
||||
build_run="$(gh api --method GET \
|
||||
"repos/${GITHUB_REPOSITORY}/actions/workflows/build.yml/runs" \
|
||||
-f branch="$RAW_TAG" -f status=success -F per_page=1 \
|
||||
--jq '.workflow_runs[0].id // empty' || true)"
|
||||
fi
|
||||
if [[ -z "$build_run" ]]; then
|
||||
echo "No certified Build and Release run found for ${RAW_TAG}; failing closed" >&2
|
||||
exit 1
|
||||
fi
|
||||
if ! gh run download "$build_run" -n release-certification -D certification; then
|
||||
echo "Build run ${build_run} published no release-certification artifact; failing closed" >&2
|
||||
exit 1
|
||||
fi
|
||||
python3 scripts/release/release_gate.py verify-promotion \
|
||||
--plan .config/release-validation-plan.json \
|
||||
--candidate certification/candidate.json \
|
||||
--summary certification/release-gate-output \
|
||||
--artifact-sha256 "$(jq -r '.deliverables[0].sha256' certification/candidate.json)" \
|
||||
--artifact-name "$(jq -r '.deliverables[0].name' certification/candidate.json)"
|
||||
|
||||
docker_run="$(gh api --method GET \
|
||||
"repos/${GITHUB_REPOSITORY}/actions/workflows/docker.yml/runs" \
|
||||
-f branch="$RAW_TAG" -f status=success -F per_page=1 \
|
||||
--jq '.workflow_runs[0].id // empty' || true)"
|
||||
if [[ -z "$docker_run" ]] || ! gh run download "$docker_run" -n image-promotion-record -D image-record; then
|
||||
echo "No image promotion record for ${RAW_TAG}; a chart may not wrap an unverified image" >&2
|
||||
exit 1
|
||||
fi
|
||||
candidate_sha="$(python3 -c "import json,hashlib,sys; sys.path.insert(0,'scripts/release'); import release_gate; print(release_gate.canonical_sha256(json.load(open('certification/candidate.json'))))")"
|
||||
record_sha="$(jq -r '.candidate_sha256 // empty' image-record/image-promotion-record.json)"
|
||||
if [[ "$record_sha" != "$candidate_sha" ]]; then
|
||||
echo "Image promotion record does not belong to this certification" >&2
|
||||
exit 1
|
||||
fi
|
||||
overlap="$(jq -r --slurpfile candidate certification/candidate.json \
|
||||
'[.binary_sha256s[]] - ($candidate[0].deliverables | map(.binary_sha256)) | length' \
|
||||
image-record/image-promotion-record.json)"
|
||||
if [[ "$overlap" -ne 0 ]]; then
|
||||
echo "Image promotion record carries binary digests outside the certification" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
build-helm-package:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
needs: [ release-certification ]
|
||||
if: |
|
||||
(github.event_name == 'workflow_dispatch' && !contains(github.event.inputs.version, '-preview')) ||
|
||||
(
|
||||
|
||||
@@ -346,12 +346,12 @@ jobs:
|
||||
path: ${{ steps.publish.outputs.candidate_file }}
|
||||
if-no-files-found: error
|
||||
|
||||
# Publish the deb/rpm pair to the auto-testing repo's `assets` branch so
|
||||
# engineers can download and install the nightly directly. The branch is
|
||||
# a single-commit orphan rewritten on every build, which keeps the repo
|
||||
# small while the latest files stay reachable at stable raw URLs.
|
||||
# Publish the deb/rpm pair as assets of the rolling `nightly` release on
|
||||
# rustfs/auto-testing (see scripts/release/publish_nightly_assets.sh).
|
||||
# The dispatch below runs only after this upload and the immutable R2
|
||||
# candidate both succeed. It does not replace rustfs-functional-chain.yml,
|
||||
# which still starts from this workflow's workflow_run completion.
|
||||
# kms-vault-lane and kms-vault-ha-failover stay parallel with this job.
|
||||
- name: Publish packages to auto-testing release assets
|
||||
env:
|
||||
ASSETS_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
@@ -360,90 +360,6 @@ jobs:
|
||||
DEB_DATE: ${{ steps.deb.outputs.deb_date }}
|
||||
BUILD_REF: ${{ env.NIGHTLY_BUILD_REF }}
|
||||
run: bash scripts/release/publish_nightly_assets.sh
|
||||
|
||||
# Cross-repo repository_dispatch cannot use GITHUB_TOKEN. The secret is a
|
||||
# PAT or GitHub App token for rustfs/auto-testing: classic `repo` scope,
|
||||
# or fine-grained Contents write (the permission GitHub checks for
|
||||
# POST /repos/{owner}/{repo}/dispatches). A dispatch failure fails this
|
||||
# step, and therefore this job, so the scheduled failure alert fires.
|
||||
# GitHub allows at most 10 top-level client_payload properties, so the
|
||||
# schema 2 candidate object is nested under `candidate` and the release
|
||||
# fields sit beside it.
|
||||
- name: Dispatch nightly candidate to auto-testing
|
||||
if: ${{ steps.publish.outputs.candidate_file != '' }}
|
||||
env:
|
||||
DISPATCH_TOKEN: ${{ secrets.AUTO_TESTING_DISPATCH_TOKEN }}
|
||||
CANDIDATE_FILE: ${{ steps.publish.outputs.candidate_file }}
|
||||
DEB_DATE: ${{ steps.deb.outputs.deb_date }}
|
||||
SOURCE_REPO: ${{ github.repository }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
: "${DISPATCH_TOKEN:?AUTO_TESTING_DISPATCH_TOKEN is required to dispatch rustfs/auto-testing}"
|
||||
: "${CANDIDATE_FILE:?nightly candidate manifest is required}"
|
||||
: "${DEB_DATE:?DEB_DATE is required}"
|
||||
[[ "${DEB_DATE}" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]]
|
||||
[[ "${SOURCE_REPO}" == "rustfs/rustfs" ]]
|
||||
|
||||
payload="${RUNNER_TEMP}/nightly-candidate-dispatch-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.json"
|
||||
jq -n --slurpfile candidate "${CANDIDATE_FILE}" \
|
||||
--arg event_type "nightly-candidate-ready" \
|
||||
--arg source_repo "${SOURCE_REPO}" \
|
||||
--arg latest_alias_url "https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb" \
|
||||
--arg auto_testing_release "https://github.com/rustfs/auto-testing/releases/tag/nightly" \
|
||||
--arg channel "nightly" \
|
||||
--arg deb_date "${DEB_DATE}" \
|
||||
--arg nightly_workflow ".github/workflows/nightly-gnu.yml" \
|
||||
'{
|
||||
event_type: $event_type,
|
||||
source_repo: $source_repo,
|
||||
latest_alias_url: $latest_alias_url,
|
||||
auto_testing_release: $auto_testing_release,
|
||||
channel: $channel,
|
||||
deb_date: $deb_date,
|
||||
nightly_workflow: $nightly_workflow,
|
||||
candidate: $candidate[0]
|
||||
}' > "${payload}"
|
||||
if ! jq -e '
|
||||
.event_type == "nightly-candidate-ready"
|
||||
and .source_repo == "rustfs/rustfs"
|
||||
and .channel == "nightly"
|
||||
and .deb_date == env.DEB_DATE
|
||||
and .nightly_workflow == ".github/workflows/nightly-gnu.yml"
|
||||
and .latest_alias_url == "https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb"
|
||||
and .auto_testing_release == "https://github.com/rustfs/auto-testing/releases/tag/nightly"
|
||||
and .candidate.schema == 2
|
||||
and ((.candidate | keys | sort) == ["build_run_attempt","build_run_id","package_sha256","package_url","schema","source_ref","source_sha","workflow_sha"])
|
||||
and ((keys | sort) == ["auto_testing_release","candidate","channel","deb_date","event_type","latest_alias_url","nightly_workflow","source_repo"])
|
||||
' "${payload}" >/dev/null; then
|
||||
echo "nightly dispatch payload does not match the nightly-candidate-ready contract" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
body="${RUNNER_TEMP}/nightly-candidate-dispatch-body-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.json"
|
||||
jq -n --slurpfile client_payload "${payload}" \
|
||||
'{event_type: "nightly-candidate-ready", client_payload: $client_payload[0]}' > "${body}"
|
||||
response="${RUNNER_TEMP}/nightly-candidate-dispatch-response-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.txt"
|
||||
status="$(curl -sS --retry 3 --connect-timeout 15 --max-time 60 \
|
||||
-o "${response}" -w '%{http_code}' \
|
||||
-X POST \
|
||||
-H "Authorization: Bearer ${DISPATCH_TOKEN}" \
|
||||
-H "Accept: application/vnd.github+json" \
|
||||
-H "X-GitHub-Api-Version: 2022-11-28" \
|
||||
-H "Content-Type: application/json" \
|
||||
--data-binary @"${body}" \
|
||||
https://api.github.com/repos/rustfs/auto-testing/dispatches)" || {
|
||||
echo "repository_dispatch request to rustfs/auto-testing failed before an HTTP status" >&2
|
||||
exit 1
|
||||
}
|
||||
if [[ "${status}" != "204" ]]; then
|
||||
echo "repository_dispatch to rustfs/auto-testing failed: HTTP ${status}" >&2
|
||||
if [[ -s "${response}" ]]; then
|
||||
cat "${response}" >&2
|
||||
fi
|
||||
exit 1
|
||||
fi
|
||||
echo "dispatched nightly-candidate-ready to rustfs/auto-testing (deb_date=${DEB_DATE}, run=${GITHUB_RUN_ID}, attempt=${GITHUB_RUN_ATTEMPT})"
|
||||
# Live-Vault lane for the rustfs-kms suite (rustfs/backlog#1774).
|
||||
#
|
||||
# RUSTFS_KMS_VAULT_TOKEN is the single switch that adds the Vault KV2 and
|
||||
|
||||
+14
-210
@@ -225,57 +225,10 @@ jobs:
|
||||
echo " Build run ID: $BUILD_RUN_ID"
|
||||
echo " Build run number: $RUN_NUMBER"
|
||||
|
||||
# Consume the trusted release certification of the resolved build run and
|
||||
# verify the certified input digests before any package is wrapped. A run
|
||||
# without that certification fails closed here; packages built later must
|
||||
# complete their own verification and cannot inherit an unverified
|
||||
# conclusion.
|
||||
release-certification:
|
||||
name: Release Certification
|
||||
needs: resolve
|
||||
if: needs.resolve.result == 'success'
|
||||
runs-on: ubuntu-latest # gh CLI required; the sm-standard-2 fleet does not ship it
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Download and verify release certification
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
|
||||
BUILD_RUN_ID: ${{ needs.resolve.outputs.build_run_id }}
|
||||
BUILD_TYPE: ${{ needs.resolve.outputs.build_type }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mkdir -p certification
|
||||
if [[ "$BUILD_TYPE" == "development" ]]; then
|
||||
# Development artifacts are explicitly uncertified and never touch
|
||||
# official or latest channels; only the staging/dev paths remain.
|
||||
jq -n '{schema_version: 1, certified: false, reason: "development artifacts are explicitly uncertified"}' \
|
||||
> certification/development-uncertified.json
|
||||
exit 0
|
||||
fi
|
||||
if ! gh run download "$BUILD_RUN_ID" -n release-certification -D certification; then
|
||||
echo "Build run ${BUILD_RUN_ID} published no release-certification artifact; failing closed" >&2
|
||||
exit 1
|
||||
fi
|
||||
python3 scripts/release/release_gate.py verify-promotion \
|
||||
--plan .config/release-validation-plan.json \
|
||||
--candidate certification/candidate.json \
|
||||
--summary certification/release-gate-output \
|
||||
--artifact-sha256 "$(jq -r '.deliverables[0].sha256' certification/candidate.json)" \
|
||||
--artifact-name "$(jq -r '.deliverables[0].name' certification/candidate.json)"
|
||||
|
||||
# Build DEB and RPM packages for each architecture and libc variant
|
||||
package:
|
||||
name: Package (${{ matrix.arch }}-${{ matrix.libc }})
|
||||
needs: [ resolve, release-certification ]
|
||||
needs: resolve
|
||||
runs-on: ubuntu-latest # gh CLI required; the sm-standard-2 fleet does not ship it
|
||||
timeout-minutes: 30
|
||||
strategy:
|
||||
@@ -541,55 +494,6 @@ jobs:
|
||||
echo "rpm_file=$RPM_FILE" >> "$GITHUB_OUTPUT"
|
||||
echo "✅ RPM built: $RPM_FILE"
|
||||
|
||||
- name: Download certification evidence
|
||||
if: needs.resolve.outputs.build_type != 'development'
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: release-certification
|
||||
path: ./certification
|
||||
|
||||
# Later-built wrapper verification: the binary inside each package must
|
||||
# match the certified binary digest for this platform before anything is
|
||||
# uploaded, and the package itself stays on staging until promotion.
|
||||
- name: Verify wrapped binary against certification
|
||||
if: needs.resolve.outputs.build_type != 'development'
|
||||
shell: bash
|
||||
env:
|
||||
DEB_FILE: ${{ steps.deb.outputs.deb_file }}
|
||||
RPM_FILE: ${{ steps.rpm.outputs.rpm_file }}
|
||||
ARTIFACT_NAME: ${{ matrix.artifact_name }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
expected="$(jq -r --arg needle "$ARTIFACT_NAME" \
|
||||
'[.deliverables[] | select(.name | contains($needle)) | .binary_sha256][0] // empty' \
|
||||
certification/candidate.json)"
|
||||
if [[ -z "$expected" ]]; then
|
||||
echo "No certified binary digest recorded for ${ARTIFACT_NAME}" >&2
|
||||
exit 1
|
||||
fi
|
||||
if ! command -v rpm2cpio >/dev/null 2>&1; then
|
||||
sudo apt-get update && sudo apt-get install -y rpm2cpio
|
||||
fi
|
||||
if ! command -v cpio >/dev/null 2>&1; then
|
||||
sudo apt-get update && sudo apt-get install -y cpio
|
||||
fi
|
||||
for package_file in "$DEB_FILE" "$RPM_FILE"; do
|
||||
case "$package_file" in
|
||||
*.deb)
|
||||
dpkg-deb --fsys-tarfile "$package_file" | tar -xOf - ./usr/bin/rustfs > "$RUNNER_TEMP/wrapped-rustfs"
|
||||
;;
|
||||
*.rpm)
|
||||
rpm2cpio "$package_file" | cpio -i --to-stdout ./usr/bin/rustfs > "$RUNNER_TEMP/wrapped-rustfs"
|
||||
;;
|
||||
esac
|
||||
python3 scripts/release/release_gate.py verify-promotion \
|
||||
--plan .config/release-validation-plan.json \
|
||||
--candidate certification/candidate.json \
|
||||
--summary certification/release-gate-output \
|
||||
--artifact "$RUNNER_TEMP/wrapped-rustfs" \
|
||||
--artifact-role binary
|
||||
done
|
||||
|
||||
- name: Upload packages to artifacts
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
@@ -627,14 +531,10 @@ jobs:
|
||||
export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
|
||||
export AWS_DEFAULT_REGION="auto"
|
||||
|
||||
# Build-phase writes stay on staging for release types: the official
|
||||
# packages/release and packages/latest paths are written only by the
|
||||
# certification-gated promote-packages job, from these same staged
|
||||
# objects. Development packages keep the uncertified dev channel.
|
||||
if [[ "$BUILD_TYPE" == "development" ]]; then
|
||||
R2_PREFIX="artifacts/rustfs/packages/dev"
|
||||
else
|
||||
R2_PREFIX="artifacts/rustfs/packages/staging/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}"
|
||||
R2_PREFIX="artifacts/rustfs/packages/release"
|
||||
fi
|
||||
R2_PATH="s3://${R2_BUCKET}/${R2_PREFIX}/"
|
||||
|
||||
@@ -649,6 +549,18 @@ jobs:
|
||||
|
||||
echo "✅ Upload complete"
|
||||
|
||||
# Also upload as latest for release/prerelease
|
||||
if [[ "$BUILD_TYPE" == "release" || "$BUILD_TYPE" == "prerelease" ]]; then
|
||||
LATEST_PATH="s3://${R2_BUCKET}/artifacts/rustfs/packages/latest/"
|
||||
for f in "$DEB_FILE" "$RPM_FILE"; do
|
||||
if [[ -n "$f" && -f "$f" ]]; then
|
||||
echo "Uploading latest: $(basename "$f")"
|
||||
aws s3 cp "$f" "$LATEST_PATH" --endpoint-url "$R2_ENDPOINT" --only-show-errors
|
||||
fi
|
||||
done
|
||||
echo "✅ Latest packages updated"
|
||||
fi
|
||||
|
||||
- name: Upload packages to GitHub Release
|
||||
if: needs.resolve.outputs.tag != ''
|
||||
env:
|
||||
@@ -702,114 +614,6 @@ jobs:
|
||||
|
||||
echo "✅ GitHub release assets updated"
|
||||
|
||||
# The only writer of the official packages/release and packages/latest R2
|
||||
# paths. Promotion re-checks each package's wrapped binary against the
|
||||
# passed certification list and copies the staged objects byte-for-byte:
|
||||
# no rebuild-after-verify, no mutable latest resolution downstream.
|
||||
promote-packages:
|
||||
name: Promote Packages
|
||||
needs: [ resolve, package, release-certification ]
|
||||
# Target-channel lock for the official package paths.
|
||||
concurrency:
|
||||
group: release-channel-r2-packages
|
||||
cancel-in-progress: false
|
||||
if: needs.resolve.outputs.tag != '' && (needs.resolve.outputs.build_type == 'release' || needs.resolve.outputs.build_type == 'prerelease')
|
||||
runs-on: ubuntu-latest # gh CLI required; the sm-standard-2 fleet does not ship it
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
env:
|
||||
R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
|
||||
R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
|
||||
R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }}
|
||||
R2_BUCKET: ${{ secrets.R2_BUCKET }}
|
||||
AWS_EC2_METADATA_DISABLED: true
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Download packages and certification
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
path: ./packages
|
||||
pattern: packages-*
|
||||
merge-multiple: true
|
||||
|
||||
- name: Download certification evidence
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: release-certification
|
||||
path: ./certification
|
||||
|
||||
- name: Verify and promote staged packages
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
if [[ -z "$R2_ACCESS_KEY_ID" || -z "$R2_SECRET_ACCESS_KEY" || -z "$R2_ENDPOINT" || -z "$R2_BUCKET" ]]; then
|
||||
echo "❌ R2 credentials missing; refusing to promote to the official channel" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
STAGING_PREFIX="s3://${R2_BUCKET}/artifacts/rustfs/packages/staging/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}/"
|
||||
RELEASE_PATH="s3://${R2_BUCKET}/artifacts/rustfs/packages/release/"
|
||||
LATEST_PATH="s3://${R2_BUCKET}/artifacts/rustfs/packages/latest/"
|
||||
|
||||
if ! command -v aws >/dev/null 2>&1; then
|
||||
sudo apt-get update && sudo apt-get install -y awscli
|
||||
fi
|
||||
export AWS_ACCESS_KEY_ID="$R2_ACCESS_KEY_ID"
|
||||
export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
|
||||
export AWS_DEFAULT_REGION="auto"
|
||||
|
||||
if ! command -v rpm2cpio >/dev/null 2>&1; then
|
||||
sudo apt-get install -y rpm2cpio
|
||||
fi
|
||||
if ! command -v cpio >/dev/null 2>&1; then
|
||||
sudo apt-get install -y cpio
|
||||
fi
|
||||
|
||||
promoted=0
|
||||
for package_file in packages/*.deb packages/*.rpm; do
|
||||
[[ -f "$package_file" ]] || continue
|
||||
name="$(basename "$package_file")"
|
||||
case "$package_file" in
|
||||
*.deb)
|
||||
dpkg-deb --fsys-tarfile "$package_file" | tar -xOf - ./usr/bin/rustfs > "$RUNNER_TEMP/wrapped-rustfs"
|
||||
;;
|
||||
*.rpm)
|
||||
rpm2cpio "$package_file" | cpio -i --to-stdout ./usr/bin/rustfs > "$RUNNER_TEMP/wrapped-rustfs"
|
||||
;;
|
||||
esac
|
||||
python3 scripts/release/release_gate.py verify-promotion \
|
||||
--plan .config/release-validation-plan.json \
|
||||
--candidate certification/candidate.json \
|
||||
--summary certification/release-gate-output \
|
||||
--artifact "$RUNNER_TEMP/wrapped-rustfs" \
|
||||
--artifact-role binary
|
||||
|
||||
staged="$(mktemp)"
|
||||
aws s3 cp "${STAGING_PREFIX}${name}" "$staged" --endpoint-url "$R2_ENDPOINT" --only-show-errors
|
||||
staged_sha="$(sha256sum "$staged" | awk '{print $1}')"
|
||||
built_sha="$(sha256sum "$package_file" | awk '{print $1}')"
|
||||
if [[ "$staged_sha" != "$built_sha" ]]; then
|
||||
echo "❌ Staged package ${name} does not match the verified bytes" >&2
|
||||
exit 1
|
||||
fi
|
||||
aws s3 cp "$staged" "${RELEASE_PATH}${name}" --endpoint-url "$R2_ENDPOINT" --only-show-errors
|
||||
aws s3 cp "$staged" "${LATEST_PATH}${name}" --endpoint-url "$R2_ENDPOINT" --only-show-errors
|
||||
rm -f "$staged"
|
||||
promoted=$((promoted + 1))
|
||||
echo "✅ Promoted ${name} (${built_sha})"
|
||||
done
|
||||
if [[ "$promoted" -eq 0 ]]; then
|
||||
echo "No staged packages found to promote" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Summary
|
||||
summary:
|
||||
name: Summary
|
||||
|
||||
@@ -93,12 +93,6 @@ jobs:
|
||||
printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
|
||||
} >> "${GITHUB_ENV}"
|
||||
|
||||
- name: Checkout environment guard
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
path: rustfs-tools
|
||||
|
||||
- name: Checkout auto-testing scripts
|
||||
timeout-minutes: 5
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
@@ -124,13 +118,6 @@ jobs:
|
||||
echo "auto-testing: ${AUTO_TESTING_SHA}"
|
||||
df -h /data | tail -1
|
||||
|
||||
- name: Admit clean shared environment
|
||||
id: environment
|
||||
timeout-minutes: 5
|
||||
env:
|
||||
FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-matrix.json
|
||||
run: python3 rustfs-tools/scripts/functional_environment.py enter --suite fault-tolerance
|
||||
|
||||
- name: Cleanup environment (before)
|
||||
timeout-minutes: 5
|
||||
if: ${{ inputs.cleanup_before != 'false' }}
|
||||
@@ -228,24 +215,7 @@ jobs:
|
||||
if-no-files-found: warn
|
||||
|
||||
- name: Cleanup environment (after)
|
||||
id: environment_cleanup
|
||||
timeout-minutes: 5
|
||||
if: ${{ always() && inputs.cleanup_after != 'false' && steps.environment.outcome == 'success' }}
|
||||
if: ${{ always() && inputs.cleanup_after != 'false' }}
|
||||
run: |
|
||||
./auto-testing/rustfs-fault-tolerance-test.sh --cleanup -y --log-file "${LOG_FILE}"
|
||||
|
||||
- name: Release environment only after successful cleanup
|
||||
if: ${{ always() && steps.environment.outcome == 'success' }}
|
||||
timeout-minutes: 5
|
||||
env:
|
||||
FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-matrix.json
|
||||
CLEANUP_OUTCOME: ${{ steps.environment_cleanup.outcome }}
|
||||
run: python3 rustfs-tools/scripts/functional_environment.py finish --suite fault-tolerance --cleanup-outcome "$CLEANUP_OUTCOME"
|
||||
|
||||
- name: Retain environment ownership and cleanup receipt
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: functional-environment-matrix-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/environment-matrix.json
|
||||
if-no-files-found: error
|
||||
./auto-testing/rustfs-fault-tolerance-test.sh --cleanup -y --log-file "${LOG_FILE}" || true
|
||||
|
||||
@@ -50,6 +50,10 @@ on:
|
||||
description: 'Reset the nodes after the test (DESTROYS test data/config)'
|
||||
type: boolean
|
||||
default: true
|
||||
repository_dispatch:
|
||||
# Chain handoff: dispatched when the replication suite finishes, ahead of
|
||||
# the performance suite.
|
||||
types: [rustfs-chain-fault-tolerance]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -76,7 +80,7 @@ jobs:
|
||||
fault-tolerance-test:
|
||||
runs-on: smoke-testing
|
||||
timeout-minutes: 60
|
||||
if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' }}
|
||||
if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
|
||||
steps:
|
||||
- name: Checkout repository (for report parser)
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
@@ -132,13 +136,6 @@ jobs:
|
||||
echo "auto-testing: ${AUTO_TESTING_SHA}"
|
||||
df -h /data | tail -1
|
||||
|
||||
- name: Admit clean shared environment
|
||||
id: environment
|
||||
timeout-minutes: 5
|
||||
env:
|
||||
FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-fault-tolerance.json
|
||||
run: python3 scripts/functional_environment.py enter --suite fault-tolerance
|
||||
|
||||
- name: Cleanup environment (before)
|
||||
timeout-minutes: 5
|
||||
if: ${{ inputs.cleanup_before != 'false' }}
|
||||
@@ -197,21 +194,21 @@ jobs:
|
||||
jq -e --arg testing_sha "${AUTO_TESTING_SHA}" '
|
||||
.schema_version == 1 and
|
||||
.auto_testing_sha == $testing_sha and
|
||||
.selected_scenarios == ["A", "B", "C", "C2", "D", "E", "F"] and
|
||||
.selected_scenarios == ["A", "B", "C", "C2", "D", "E"] and
|
||||
.complete == true and
|
||||
.counts.expected == 54 and
|
||||
.counts.seen == 54 and
|
||||
.counts.expected == 44 and
|
||||
.counts.seen == 44 and
|
||||
.counts.harness_errors == 0 and
|
||||
(.expected_cases | length) == 54 and
|
||||
(.expected_cases | unique | length) == 54 and
|
||||
(.seen_cases | length) == 54 and
|
||||
(.seen_cases | unique | length) == 54 and
|
||||
(.expected_cases | length) == 44 and
|
||||
(.expected_cases | unique | length) == 44 and
|
||||
(.seen_cases | length) == 44 and
|
||||
(.seen_cases | unique | length) == 44 and
|
||||
(.seen_cases | sort) == (.expected_cases | sort) and
|
||||
(.missing_cases | length) == 0 and
|
||||
(.harness_errors | length) == 0 and
|
||||
(.cases | length) == 54 and
|
||||
(.cases | length) == 44 and
|
||||
([.cases[].case_id] | sort) == (.expected_cases | sort) and
|
||||
([.cases[].case_id] | unique | length) == 54 and
|
||||
([.cases[].case_id] | unique | length) == 44 and
|
||||
all(.cases[]; .verdict == "pass" or .verdict == "known-divergence" or .verdict == "UNEXPECTED") and
|
||||
.counts.unexpected == ([.cases[] | select(.verdict == "UNEXPECTED")] | length) and
|
||||
.counts.known_divergence == ([.cases[] | select(.verdict == "known-divergence")] | length)
|
||||
@@ -308,27 +305,58 @@ jobs:
|
||||
if-no-files-found: warn
|
||||
|
||||
- name: Cleanup environment (after)
|
||||
id: environment_cleanup
|
||||
timeout-minutes: 5
|
||||
if: ${{ always() && inputs.cleanup_after != 'false' && steps.environment.outcome == 'success' }}
|
||||
if: ${{ always() && inputs.cleanup_after != 'false' }}
|
||||
run: |
|
||||
./auto-testing/rustfs-fault-tolerance-test.sh --cleanup -y --log-file "${LOG_FILE}"
|
||||
./auto-testing/rustfs-fault-tolerance-test.sh --cleanup -y --log-file "${LOG_FILE}" || true
|
||||
|
||||
- name: Release environment only after successful cleanup
|
||||
if: ${{ always() && steps.environment.outcome == 'success' }}
|
||||
timeout-minutes: 5
|
||||
- name: "Continue functional chain (next: Performance)"
|
||||
# Only chain-triggered runs forward to the next suite; standalone
|
||||
# workflow_dispatch runs stop after their own cleanup. A failed
|
||||
# handoff retries, then files an alert issue in rustfs/backlog.
|
||||
if: ${{ always() && github.event_name == 'repository_dispatch' }}
|
||||
continue-on-error: true
|
||||
env:
|
||||
FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-fault-tolerance.json
|
||||
CLEANUP_OUTCOME: ${{ steps.environment_cleanup.outcome }}
|
||||
run: python3 scripts/functional_environment.py finish --suite fault-tolerance --cleanup-outcome "$CLEANUP_OUTCOME"
|
||||
|
||||
- name: Retain environment ownership and cleanup receipt
|
||||
if: ${{ always() }}
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: functional-environment-fault-tolerance-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/environment-fault-tolerance.json
|
||||
if-no-files-found: error
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
run: |
|
||||
set -uo pipefail
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
|
||||
exit 1
|
||||
fi
|
||||
DISPATCHED=0
|
||||
for attempt in 1 2 3; do
|
||||
if gh api --method POST repos/rustfs/rustfs/dispatches \
|
||||
-f event_type='rustfs-chain-performance' \
|
||||
-F 'client_payload[from_suite]=fault-tolerance'; then
|
||||
echo "dispatched next suite Performance (attempt ${attempt})"
|
||||
DISPATCHED=1
|
||||
break
|
||||
fi
|
||||
echo "dispatch attempt ${attempt} failed; retrying in ${attempt}0s" >&2
|
||||
sleep "${attempt}0"
|
||||
done
|
||||
if [ "${DISPATCHED:-0}" -ne 1 ]; then
|
||||
echo "ERROR: functional chain stalled: could not dispatch Performance after 3 attempts" >&2
|
||||
TITLE="[functional][chain] stalled after fault-tolerance (run ${GITHUB_RUN_ID})"
|
||||
BODY_FILE="$(mktemp)"
|
||||
{
|
||||
echo "The functional chain could not hand off from **fault-tolerance** to **Performance** after 3 attempts."
|
||||
echo ""
|
||||
echo "- Failed suite job: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
|
||||
echo "- Expected next event: 'rustfs-chain-performance'"
|
||||
echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
|
||||
echo "- Recovery: re-dispatch manually with"
|
||||
FENCE="$(printf "\x60\x60\x60")"; echo " ${FENCE}"
|
||||
echo " gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-performance'"
|
||||
FENCE="$(printf "\x60\x60\x60")"; echo " ${FENCE}"
|
||||
} > "${BODY_FILE}"
|
||||
gh issue create -R rustfs/backlog --title "${TITLE}" \
|
||||
--body-file "${BODY_FILE}" --label functional-test \
|
||||
|| gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}" \
|
||||
|| echo "could not file the stall alert issue either; check the token" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Notify on failure
|
||||
if: failure()
|
||||
@@ -363,13 +391,3 @@ jobs:
|
||||
name: functional-chain-fault-tolerance-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/chain-fault-tolerance-${{ github.run_id }}-${{ github.run_attempt }}/fault-tolerance.json
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Enforce suite and report result
|
||||
if: ${{ always() }}
|
||||
env:
|
||||
SUITE_OUTCOME: ${{ steps.test.outcome }}
|
||||
REPORT_OUTCOME: ${{ steps.chain_report.outcome }}
|
||||
run: |
|
||||
test "$SUITE_OUTCOME" = success
|
||||
test "$REPORT_OUTCOME" = success
|
||||
python3 scripts/functional_chain_evidence.py verify-report --suite fault-tolerance --report "${FUNCTIONAL_ARTIFACTS_DIR}/suite.log"
|
||||
|
||||
@@ -13,23 +13,12 @@
|
||||
# limitations under the License.
|
||||
|
||||
# Reusable workflows run from this driver's commit in one Actions run. Each
|
||||
# selected suite still runs after an earlier failure; the final job checks its plan.
|
||||
# suite still runs after an earlier suite fails; the final job requires all twelve.
|
||||
name: RustFS Functional Chain
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
validation_plan:
|
||||
description: Registered cadence plan (existing suite frequencies are preserved)
|
||||
type: choice
|
||||
options: [nightly, weekly, monthly]
|
||||
required: false
|
||||
default: nightly
|
||||
suites:
|
||||
description: Comma-separated suite IDs, or empty for the complete registered plan
|
||||
type: string
|
||||
required: false
|
||||
default: ''
|
||||
build_run_id:
|
||||
description: Successful nightly or validation package run; empty for package-only build
|
||||
type: string
|
||||
@@ -227,25 +216,10 @@ jobs:
|
||||
outputs:
|
||||
manifest: ${{ steps.candidate.outputs.manifest }}
|
||||
performance_ready: ${{ steps.perf_probe.outputs.performance_ready }}
|
||||
plan: ${{ steps.selection.outputs.plan }}
|
||||
suites: ${{ steps.selection.outputs.suites }}
|
||||
optional_suites: ${{ steps.selection.outputs.optional_suites }}
|
||||
steps:
|
||||
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Select registered suites
|
||||
id: selection
|
||||
env:
|
||||
VALIDATION_PLAN: ${{ inputs.validation_plan || 'nightly' }}
|
||||
VALIDATION_SUITES: ${{ inputs.suites }}
|
||||
run: python3 scripts/validation_registry.py select --output "${RUNNER_TEMP}/suite-plan.json"
|
||||
- name: Retain selected responsibilities
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: functional-plan-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/suite-plan.json
|
||||
if-no-files-found: error
|
||||
- name: Resolve published candidate
|
||||
id: candidate
|
||||
env:
|
||||
@@ -265,7 +239,6 @@ jobs:
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Check shared functional fleet runner before scheduling suites
|
||||
if: ${{ contains(fromJSON(steps.selection.outputs.environments), 'functional-fleet') }}
|
||||
# The eleven functional suites run on the smoke-testing fleet; a hard
|
||||
# requirement. The performance lane runs on pf-testing, which is a
|
||||
# separate fleet - probe it without failing the whole chain so an
|
||||
@@ -275,26 +248,15 @@ jobs:
|
||||
run: python3 scripts/check_functional_runners.py smoke-testing
|
||||
|
||||
- name: Probe performance fleet runner
|
||||
if: ${{ contains(fromJSON(steps.selection.outputs.suites), 'performance') }}
|
||||
id: perf_probe
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
PERFORMANCE_OPTIONAL: ${{ contains(fromJSON(steps.selection.outputs.optional_suites), 'performance') }}
|
||||
run: |
|
||||
# TEMPORARY: the pf-testing environment is broken, so the
|
||||
# performance lane is disabled until it is repaired. Forcing
|
||||
# offline keeps the chain on its designed skip path: the lane job
|
||||
# is skipped, aggregate receives --allow-skipped performance
|
||||
# (required=false in scheduled-validations.json), and the report
|
||||
# records the optional-lane skip with its reason.
|
||||
# Restore the probe below once the environment is healthy again.
|
||||
# if python3 scripts/check_functional_runners.py pf-testing; then
|
||||
# echo "performance_ready=online" >> "$GITHUB_OUTPUT"
|
||||
# else
|
||||
# echo "performance_ready=offline" >> "$GITHUB_OUTPUT"
|
||||
# test "$PERFORMANCE_OPTIONAL" = true
|
||||
# fi
|
||||
echo "performance_ready=offline" >> "$GITHUB_OUTPUT"
|
||||
if python3 scripts/check_functional_runners.py pf-testing; then
|
||||
echo "performance_ready=online" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "performance_ready=offline" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
validation:
|
||||
if: ${{ github.event_name == 'workflow_dispatch' && !inputs.validation_runner_inspect_only && inputs.validation_retain_evidence_run_id == '' && inputs.validation_package_sha == '' && (inputs.validation_case != '' || inputs.validation_tools_only || inputs.validation_native_tools || inputs.validation_shared_tools || inputs.validation_manager_tools || inputs.validation_cleanup_run_id != '' || inputs.validation_cleanup_retained_evidence) }}
|
||||
@@ -618,10 +580,6 @@ jobs:
|
||||
> "${VALIDATION_ROOT}/shared-fixture-manifest.json"
|
||||
printf 'VALIDATION_HEAL_SHARED_DIRECTORY=%s\n' "${SHARED_DIRECTORY}" >> "${GITHUB_ENV}"
|
||||
|
||||
- name: Reject quarantined suite environment before canonical validation
|
||||
if: ${{ !inputs.validation_tools_only && inputs.validation_cleanup_run_id == '' && inputs.validation_case != 'environment.inspect' }}
|
||||
run: python3 scripts/functional_environment.py check --suite heal
|
||||
|
||||
- name: Run selected canonical validation case
|
||||
if: ${{ !inputs.validation_tools_only && inputs.validation_cleanup_run_id == '' && inputs.validation_case != 'environment.inspect' }}
|
||||
env:
|
||||
@@ -707,7 +665,7 @@ jobs:
|
||||
|
||||
upgrade:
|
||||
needs: [prepare]
|
||||
if: ${{ always() && needs.prepare.result == 'success' && contains(fromJSON(needs.prepare.outputs.suites || '[]'), 'upgrade') }}
|
||||
if: ${{ always() && needs.prepare.result == 'success' }}
|
||||
uses: ./.github/workflows/rustfs-upgrade-test.yml
|
||||
with:
|
||||
chain_manifest: ${{ needs.prepare.outputs.manifest }}
|
||||
@@ -715,7 +673,7 @@ jobs:
|
||||
|
||||
s3:
|
||||
needs: [prepare, upgrade]
|
||||
if: ${{ always() && needs.prepare.result == 'success' && contains(fromJSON(needs.prepare.outputs.suites || '[]'), 's3') }}
|
||||
if: ${{ always() && needs.prepare.result == 'success' }}
|
||||
uses: ./.github/workflows/rustfs-s3-compat-test.yml
|
||||
with:
|
||||
chain_manifest: ${{ needs.prepare.outputs.manifest }}
|
||||
@@ -723,7 +681,7 @@ jobs:
|
||||
|
||||
kms:
|
||||
needs: [prepare, s3]
|
||||
if: ${{ always() && needs.prepare.result == 'success' && contains(fromJSON(needs.prepare.outputs.suites || '[]'), 'kms') }}
|
||||
if: ${{ always() && needs.prepare.result == 'success' }}
|
||||
uses: ./.github/workflows/rustfs-kms-test.yml
|
||||
with:
|
||||
chain_manifest: ${{ needs.prepare.outputs.manifest }}
|
||||
@@ -731,7 +689,7 @@ jobs:
|
||||
|
||||
tier:
|
||||
needs: [prepare, kms]
|
||||
if: ${{ always() && needs.prepare.result == 'success' && contains(fromJSON(needs.prepare.outputs.suites || '[]'), 'tier') }}
|
||||
if: ${{ always() && needs.prepare.result == 'success' }}
|
||||
uses: ./.github/workflows/rustfs-tier-test.yml
|
||||
with:
|
||||
chain_manifest: ${{ needs.prepare.outputs.manifest }}
|
||||
@@ -739,7 +697,7 @@ jobs:
|
||||
|
||||
storage:
|
||||
needs: [prepare, tier]
|
||||
if: ${{ always() && needs.prepare.result == 'success' && contains(fromJSON(needs.prepare.outputs.suites || '[]'), 'storage') }}
|
||||
if: ${{ always() && needs.prepare.result == 'success' }}
|
||||
uses: ./.github/workflows/rustfs-storage-test.yml
|
||||
with:
|
||||
chain_manifest: ${{ needs.prepare.outputs.manifest }}
|
||||
@@ -747,7 +705,7 @@ jobs:
|
||||
|
||||
heal:
|
||||
needs: [prepare, storage]
|
||||
if: ${{ always() && needs.prepare.result == 'success' && contains(fromJSON(needs.prepare.outputs.suites || '[]'), 'heal') }}
|
||||
if: ${{ always() && needs.prepare.result == 'success' }}
|
||||
uses: ./.github/workflows/rustfs-heal-test.yml
|
||||
with:
|
||||
chain_manifest: ${{ needs.prepare.outputs.manifest }}
|
||||
@@ -755,7 +713,7 @@ jobs:
|
||||
|
||||
pool:
|
||||
needs: [prepare, heal]
|
||||
if: ${{ always() && needs.prepare.result == 'success' && contains(fromJSON(needs.prepare.outputs.suites || '[]'), 'pool') }}
|
||||
if: ${{ always() && needs.prepare.result == 'success' }}
|
||||
uses: ./.github/workflows/rustfs-pool-expand-test.yml
|
||||
with:
|
||||
chain_manifest: ${{ needs.prepare.outputs.manifest }}
|
||||
@@ -763,7 +721,7 @@ jobs:
|
||||
|
||||
security:
|
||||
needs: [prepare, pool]
|
||||
if: ${{ always() && needs.prepare.result == 'success' && contains(fromJSON(needs.prepare.outputs.suites || '[]'), 'security') }}
|
||||
if: ${{ always() && needs.prepare.result == 'success' }}
|
||||
uses: ./.github/workflows/rustfs-security-test.yml
|
||||
with:
|
||||
chain_manifest: ${{ needs.prepare.outputs.manifest }}
|
||||
@@ -771,7 +729,7 @@ jobs:
|
||||
|
||||
replication:
|
||||
needs: [prepare, security]
|
||||
if: ${{ always() && needs.prepare.result == 'success' && contains(fromJSON(needs.prepare.outputs.suites || '[]'), 'replication') }}
|
||||
if: ${{ always() && needs.prepare.result == 'success' }}
|
||||
uses: ./.github/workflows/rustfs-replication-test.yml
|
||||
with:
|
||||
chain_manifest: ${{ needs.prepare.outputs.manifest }}
|
||||
@@ -779,7 +737,7 @@ jobs:
|
||||
|
||||
fault-tolerance:
|
||||
needs: [prepare, replication]
|
||||
if: ${{ always() && needs.prepare.result == 'success' && contains(fromJSON(needs.prepare.outputs.suites || '[]'), 'fault-tolerance') }}
|
||||
if: ${{ always() && needs.prepare.result == 'success' }}
|
||||
uses: ./.github/workflows/rustfs-fault-tolerance-test.yml
|
||||
with:
|
||||
chain_manifest: ${{ needs.prepare.outputs.manifest }}
|
||||
@@ -787,15 +745,15 @@ jobs:
|
||||
|
||||
table:
|
||||
needs: [prepare, fault-tolerance]
|
||||
if: ${{ always() && needs.prepare.result == 'success' && contains(fromJSON(needs.prepare.outputs.suites || '[]'), 'table') }}
|
||||
if: ${{ always() && needs.prepare.result == 'success' }}
|
||||
uses: ./.github/workflows/rustfs-table-test.yml
|
||||
with:
|
||||
chain_manifest: ${{ needs.prepare.outputs.manifest }}
|
||||
secrets: inherit
|
||||
|
||||
performance:
|
||||
needs: [prepare]
|
||||
if: ${{ always() && needs.prepare.result == 'success' && contains(fromJSON(needs.prepare.outputs.suites || '[]'), 'performance') && needs.prepare.outputs.performance_ready == 'online' }}
|
||||
needs: [prepare, table]
|
||||
if: ${{ always() && needs.prepare.result == 'success' && needs.prepare.outputs.performance_ready == 'online' }}
|
||||
uses: ./.github/workflows/rustfs-performance-test.yml
|
||||
with:
|
||||
chain_manifest: ${{ needs.prepare.outputs.manifest }}
|
||||
@@ -823,10 +781,8 @@ jobs:
|
||||
env:
|
||||
CHAIN_MANIFEST: ${{ needs.prepare.outputs.manifest }}
|
||||
CHAIN_NEEDS: ${{ toJSON(needs) }}
|
||||
CHAIN_PLAN: ${{ needs.prepare.outputs.plan }}
|
||||
run: >-
|
||||
python3 scripts/functional_chain_evidence.py summarize
|
||||
--allow-skipped "${{ contains(fromJSON(needs.prepare.outputs.optional_suites || '[]'), 'performance') && needs.prepare.outputs.performance_ready != 'online' && 'performance' || '' }}"
|
||||
--directory "${RUNNER_TEMP}/chain-evidence"
|
||||
--output "${RUNNER_TEMP}/chain-report.json"
|
||||
- name: Retain chain report regardless of test verdict
|
||||
@@ -843,10 +799,9 @@ jobs:
|
||||
env:
|
||||
CHAIN_MANIFEST: ${{ needs.prepare.outputs.manifest }}
|
||||
CHAIN_NEEDS: ${{ toJSON(needs) }}
|
||||
CHAIN_PLAN: ${{ needs.prepare.outputs.plan }}
|
||||
run: >-
|
||||
python3 scripts/functional_chain_evidence.py aggregate
|
||||
--allow-skipped "${{ contains(fromJSON(needs.prepare.outputs.optional_suites || '[]'), 'performance') && needs.prepare.outputs.performance_ready != 'online' && 'performance' || '' }}"
|
||||
--allow-skipped "${{ needs.prepare.outputs.performance_ready == 'online' && '' || 'performance' }}"
|
||||
--directory "${RUNNER_TEMP}/chain-evidence"
|
||||
--output "${RUNNER_TEMP}/chain-complete.json"
|
||||
- name: Upload complete-chain evidence
|
||||
|
||||
@@ -29,6 +29,11 @@ on:
|
||||
description: 'Reset the nodes after the test (DESTROYS test data/config)'
|
||||
type: boolean
|
||||
default: true
|
||||
repository_dispatch:
|
||||
# Chain handoff: dispatched when the storage suite finishes. Heal runs
|
||||
# exactly once per chain; the pool expansion workflow no longer embeds
|
||||
# its own heal pass.
|
||||
types: [rustfs-chain-heal]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -55,11 +60,10 @@ env:
|
||||
jobs:
|
||||
heal-test:
|
||||
runs-on: smoke-testing
|
||||
# 90-minute heal run (see the step below) plus setup, report and cleanup.
|
||||
timeout-minutes: 120
|
||||
timeout-minutes: 60
|
||||
# Standalone manual run, or one link of the nightly functional chain
|
||||
# (storage -> heal -> pool). Pool expansion no longer re-runs heal.
|
||||
if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' }}
|
||||
if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
|
||||
steps:
|
||||
- name: Checkout chain tooling
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
@@ -105,13 +109,6 @@ jobs:
|
||||
warp --version || true
|
||||
df -h /data | tail -1
|
||||
|
||||
- name: Admit clean shared environment
|
||||
id: environment
|
||||
timeout-minutes: 5
|
||||
env:
|
||||
FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-heal.json
|
||||
run: python3 scripts/functional_environment.py enter --suite heal
|
||||
|
||||
- name: Cleanup environment (before)
|
||||
timeout-minutes: 5
|
||||
if: ${{ inputs.cleanup_before != 'false' }}
|
||||
@@ -155,9 +152,10 @@ jobs:
|
||||
./auto-testing/rustfs_heal_test.sh "${ARGS[@]}" --log-file "${LOG_FILE}"
|
||||
|
||||
- name: Run heal test (write -> outage -> heal -> verify)
|
||||
timeout-minutes: 90
|
||||
timeout-minutes: 45
|
||||
id: test
|
||||
# Collect reports and backlog evidence before the final suite verdict.
|
||||
# Case failures keep the run green: the report and the backlog
|
||||
# issue manager carry the product signal.
|
||||
continue-on-error: true
|
||||
run: |
|
||||
./auto-testing/rustfs_heal_test.sh \
|
||||
@@ -243,8 +241,9 @@ jobs:
|
||||
RESULT=success
|
||||
fi
|
||||
|
||||
# Preserve harness completion separately; the final verdict checks
|
||||
# every case after retaining reports and cleanup evidence.
|
||||
# Product gate: failing steps keep the run green — they are reported
|
||||
# below and tracked in rustfs/backlog. Only harness/environment
|
||||
# breakdowns turn the workflow red.
|
||||
STEPS_TOTAL="$(grep -cE '^\[HEAL-STEP\]' "${LOG_FILE}" 2>/dev/null || true)"
|
||||
STEPS_FAIL="$(grep -cE '^\[HEAL-STEP\].* FAIL$' "${LOG_FILE}" 2>/dev/null || true)"
|
||||
STEPS_TOTAL=$(( ${STEPS_TOTAL:-0} + 0 )); STEPS_FAIL=$(( ${STEPS_FAIL:-0} + 0 ))
|
||||
@@ -282,7 +281,7 @@ jobs:
|
||||
fi
|
||||
} | tee "${REPORT_FILE}"
|
||||
cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
|
||||
# Preserve the harness verdict separately from the final case verdict.
|
||||
# Red only for harness/environment breakdowns; step failures stay green.
|
||||
[ "${HARNESS_OK}" = "1" ]
|
||||
|
||||
- name: Upload functional report to dashboard
|
||||
@@ -378,9 +377,8 @@ jobs:
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Cleanup environment (after)
|
||||
id: environment_cleanup
|
||||
timeout-minutes: 5
|
||||
if: ${{ always() && inputs.cleanup_after != 'false' && steps.environment.outcome == 'success' }}
|
||||
if: ${{ always() && inputs.cleanup_after != 'false' }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
|
||||
@@ -398,21 +396,54 @@ jobs:
|
||||
'
|
||||
done
|
||||
|
||||
- name: Release environment only after successful cleanup
|
||||
if: ${{ always() && steps.environment.outcome == 'success' }}
|
||||
timeout-minutes: 5
|
||||
- name: "Continue functional chain (next: Pool expansion)"
|
||||
# Only chain-triggered runs forward to the next suite; standalone
|
||||
# workflow_dispatch runs stop after their own cleanup. A failed
|
||||
# handoff must never pass silently: it retries, then files an alert
|
||||
# issue in rustfs/backlog so a stalled chain is visible.
|
||||
if: ${{ always() && github.event_name == 'repository_dispatch' }}
|
||||
continue-on-error: true
|
||||
env:
|
||||
FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-heal.json
|
||||
CLEANUP_OUTCOME: ${{ steps.environment_cleanup.outcome }}
|
||||
run: python3 scripts/functional_environment.py finish --suite heal --cleanup-outcome "$CLEANUP_OUTCOME"
|
||||
|
||||
- name: Retain environment ownership and cleanup receipt
|
||||
if: ${{ always() }}
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: functional-environment-heal-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/environment-heal.json
|
||||
if-no-files-found: error
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
run: |
|
||||
set -uo pipefail
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
|
||||
exit 1
|
||||
fi
|
||||
DISPATCHED=0
|
||||
for attempt in 1 2 3; do
|
||||
if gh api --method POST repos/rustfs/rustfs/dispatches \
|
||||
-f event_type='rustfs-chain-pool' \
|
||||
-F 'client_payload[from_suite]=heal'; then
|
||||
echo "dispatched next suite Pool expansion (attempt ${attempt})"
|
||||
DISPATCHED=1
|
||||
break
|
||||
fi
|
||||
echo "dispatch attempt ${attempt} failed; retrying in ${attempt}0s" >&2
|
||||
sleep "${attempt}0"
|
||||
done
|
||||
if [ "${DISPATCHED:-0}" -ne 1 ]; then
|
||||
echo "ERROR: functional chain stalled: could not dispatch Pool expansion after 3 attempts" >&2
|
||||
TITLE="[functional][chain] stalled after heal (run ${GITHUB_RUN_ID})"
|
||||
BODY_FILE="$(mktemp)"
|
||||
{
|
||||
echo "The functional chain could not hand off from **heal** to **Pool expansion** after 3 attempts."
|
||||
echo ""
|
||||
echo "- Failed suite job: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
|
||||
echo "- Expected next event: 'rustfs-chain-pool'"
|
||||
echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
|
||||
echo "- Recovery: re-dispatch manually with"
|
||||
FENCE="$(printf "\x60\x60\x60")"; echo " ${FENCE}"
|
||||
echo " gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-pool'"
|
||||
FENCE="$(printf "\x60\x60\x60")"; echo " ${FENCE}"
|
||||
} > "${BODY_FILE}"
|
||||
gh issue create -R rustfs/backlog --title "${TITLE}" \
|
||||
--body-file "${BODY_FILE}" --label functional-test \
|
||||
|| gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}" \
|
||||
|| echo "could not file the stall alert issue either; check the token" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Notify on failure
|
||||
if: failure()
|
||||
@@ -441,13 +472,3 @@ jobs:
|
||||
name: functional-chain-heal-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/chain-heal-${{ github.run_id }}-${{ github.run_attempt }}/heal.json
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Enforce suite and report result
|
||||
if: ${{ always() }}
|
||||
env:
|
||||
SUITE_OUTCOME: ${{ steps.test.outcome }}
|
||||
REPORT_OUTCOME: ${{ steps.chain_report.outcome }}
|
||||
run: |
|
||||
test "$SUITE_OUTCOME" = success
|
||||
test "$REPORT_OUTCOME" = success
|
||||
python3 scripts/functional_chain_evidence.py verify-report --suite heal --report "${FUNCTIONAL_ARTIFACTS_DIR}/steps.md"
|
||||
|
||||
@@ -29,6 +29,9 @@ on:
|
||||
description: 'Set RUSTFS_KMS_CONFIG_SECRET (runs KMS-107 config sealing)'
|
||||
required: false
|
||||
type: string
|
||||
repository_dispatch:
|
||||
# Chain handoff: dispatched when the S3 compatibility suite finishes.
|
||||
types: [rustfs-chain-kms]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -53,7 +56,7 @@ jobs:
|
||||
kms-test:
|
||||
runs-on: smoke-testing
|
||||
timeout-minutes: 60
|
||||
if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' }}
|
||||
if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
|
||||
steps:
|
||||
- name: Checkout repository (for report parser)
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
@@ -97,13 +100,6 @@ jobs:
|
||||
openssl version
|
||||
docker --version || true
|
||||
|
||||
- name: Admit clean shared environment
|
||||
id: environment
|
||||
timeout-minutes: 5
|
||||
env:
|
||||
FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-kms.json
|
||||
run: python3 scripts/functional_environment.py enter --suite kms
|
||||
|
||||
- name: Cleanup environment (before)
|
||||
timeout-minutes: 5
|
||||
run: |
|
||||
@@ -135,7 +131,8 @@ jobs:
|
||||
- name: Run KMS suite
|
||||
id: test
|
||||
timeout-minutes: 45
|
||||
# Collect reports and backlog evidence before the final suite verdict.
|
||||
# Case failures keep the run green: the report and the backlog
|
||||
# issue manager carry the product signal.
|
||||
continue-on-error: true
|
||||
run: |
|
||||
set -euo pipefail
|
||||
@@ -188,8 +185,10 @@ jobs:
|
||||
RESULT=success
|
||||
fi
|
||||
|
||||
# Report harness completion here; the final verdict checks every
|
||||
# case result after diagnostics and cleanup have been retained.
|
||||
# Product gate: failing cases keep the run green — they are reported
|
||||
# below and tracked in rustfs/backlog. Only harness/environment
|
||||
# breakdowns (the suite never reached case level, or a wholesale
|
||||
# failure with zero passes) turn the workflow red.
|
||||
CASES_TOTAL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| (PASS|FAIL|UNSUPPORTED|RUNNING) \|' "${CASE_TABLE}" 2>/dev/null || true)"
|
||||
CASES_FAIL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| FAIL \|' "${CASE_TABLE}" 2>/dev/null || true)"
|
||||
CASES_TOTAL=$(( ${CASES_TOTAL:-0} + 0 )); CASES_FAIL=$(( ${CASES_FAIL:-0} + 0 ))
|
||||
@@ -227,7 +226,7 @@ jobs:
|
||||
fi
|
||||
} | tee "${REPORT_FILE}"
|
||||
cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
|
||||
# Preserve the harness verdict separately from the final case verdict.
|
||||
# Red only for harness/environment breakdowns; case failures stay green.
|
||||
[ "${HARNESS_OK}" = "1" ]
|
||||
|
||||
- name: Upload functional report to dashboard
|
||||
@@ -326,9 +325,8 @@ jobs:
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Cleanup environment (after)
|
||||
id: environment_cleanup
|
||||
timeout-minutes: 5
|
||||
if: ${{ always() && steps.environment.outcome == 'success' }}
|
||||
if: always()
|
||||
run: |
|
||||
set -euo pipefail
|
||||
read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
|
||||
@@ -346,21 +344,54 @@ jobs:
|
||||
'
|
||||
done
|
||||
|
||||
- name: Release environment only after successful cleanup
|
||||
if: ${{ always() && steps.environment.outcome == 'success' }}
|
||||
timeout-minutes: 5
|
||||
- name: "Continue functional chain (next: Tier)"
|
||||
# Only chain-triggered runs forward to the next suite; standalone
|
||||
# workflow_dispatch runs stop after their own cleanup. A failed
|
||||
# handoff must never pass silently: it retries, then files an alert
|
||||
# issue in rustfs/backlog so a stalled chain is visible.
|
||||
if: ${{ always() && github.event_name == 'repository_dispatch' }}
|
||||
continue-on-error: true
|
||||
env:
|
||||
FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-kms.json
|
||||
CLEANUP_OUTCOME: ${{ steps.environment_cleanup.outcome }}
|
||||
run: python3 scripts/functional_environment.py finish --suite kms --cleanup-outcome "$CLEANUP_OUTCOME"
|
||||
|
||||
- name: Retain environment ownership and cleanup receipt
|
||||
if: ${{ always() }}
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: functional-environment-kms-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/environment-kms.json
|
||||
if-no-files-found: error
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
run: |
|
||||
set -uo pipefail
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
|
||||
exit 1
|
||||
fi
|
||||
DISPATCHED=0
|
||||
for attempt in 1 2 3; do
|
||||
if gh api --method POST repos/rustfs/rustfs/dispatches \
|
||||
-f event_type='rustfs-chain-tier' \
|
||||
-F 'client_payload[from_suite]=kms'; then
|
||||
echo "dispatched next suite Tier (attempt ${attempt})"
|
||||
DISPATCHED=1
|
||||
break
|
||||
fi
|
||||
echo "dispatch attempt ${attempt} failed; retrying in ${attempt}0s" >&2
|
||||
sleep "${attempt}0"
|
||||
done
|
||||
if [ "${DISPATCHED:-0}" -ne 1 ]; then
|
||||
echo "ERROR: functional chain stalled: could not dispatch Tier after 3 attempts" >&2
|
||||
TITLE="[functional][chain] stalled after kms (run ${GITHUB_RUN_ID})"
|
||||
BODY_FILE="$(mktemp)"
|
||||
{
|
||||
echo "The functional chain could not hand off from **kms** to **Tier** after 3 attempts."
|
||||
echo ""
|
||||
echo "- Failed suite job: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
|
||||
echo "- Expected next event: 'rustfs-chain-tier'"
|
||||
echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
|
||||
echo "- Recovery: re-dispatch manually with"
|
||||
FENCE="$(printf "\x60\x60\x60")"; echo " ${FENCE}"
|
||||
echo " gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-tier'"
|
||||
FENCE="$(printf "\x60\x60\x60")"; echo " ${FENCE}"
|
||||
} > "${BODY_FILE}"
|
||||
gh issue create -R rustfs/backlog --title "${TITLE}" \
|
||||
--body-file "${BODY_FILE}" --label functional-test \
|
||||
|| gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}" \
|
||||
|| echo "could not file the stall alert issue either; check the token" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Notify on failure
|
||||
if: failure()
|
||||
@@ -388,13 +419,3 @@ jobs:
|
||||
name: functional-chain-kms-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/chain-kms-${{ github.run_id }}-${{ github.run_attempt }}/kms.json
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Enforce suite and report result
|
||||
if: ${{ always() }}
|
||||
env:
|
||||
SUITE_OUTCOME: ${{ steps.test.outcome }}
|
||||
REPORT_OUTCOME: ${{ steps.chain_report.outcome }}
|
||||
run: |
|
||||
test "$SUITE_OUTCOME" = success
|
||||
test "$REPORT_OUTCOME" = success
|
||||
python3 scripts/functional_chain_evidence.py verify-report --suite kms --report "${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"
|
||||
|
||||
@@ -54,6 +54,9 @@ on:
|
||||
description: 'Reset the nodes after the test (DESTROYS test data/config)'
|
||||
type: boolean
|
||||
default: true
|
||||
repository_dispatch:
|
||||
# Chain handoff: dispatched when the replication suite finishes.
|
||||
types: [rustfs-chain-performance]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -86,7 +89,7 @@ env:
|
||||
|
||||
jobs:
|
||||
runner-check:
|
||||
if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' }}
|
||||
if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
@@ -110,7 +113,7 @@ jobs:
|
||||
RUSTFS_WARP_CONCURRENCY: ${{ inputs.warp_concurrency || '64' }}
|
||||
# Run on manual dispatch, or when the nightly build completed successfully.
|
||||
# Skipped when nightly failed.
|
||||
if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' }}
|
||||
if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
|
||||
steps:
|
||||
- name: Checkout chain tooling
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
@@ -156,13 +159,6 @@ jobs:
|
||||
warp --version || true
|
||||
df -h /data | tail -1
|
||||
|
||||
- name: Admit clean shared environment
|
||||
id: environment
|
||||
timeout-minutes: 5
|
||||
env:
|
||||
FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-performance.json
|
||||
run: python3 scripts/functional_environment.py enter --suite performance
|
||||
|
||||
- name: Reset test environment (before)
|
||||
if: ${{ inputs.cleanup_before != 'false' }}
|
||||
run: |
|
||||
@@ -327,27 +323,10 @@ jobs:
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Reset test environment (after)
|
||||
id: environment_cleanup
|
||||
if: ${{ always() && inputs.cleanup_after != 'false' && steps.environment.outcome == 'success' }}
|
||||
if: ${{ always() && inputs.cleanup_after != 'false' }}
|
||||
run: |
|
||||
./auto-testing/rustfs_performance_test.sh --step 7 -y --log-file "${LOG_FILE:-/dev/null}"
|
||||
|
||||
- name: Release environment only after successful cleanup
|
||||
if: ${{ always() && steps.environment.outcome == 'success' }}
|
||||
timeout-minutes: 5
|
||||
env:
|
||||
FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-performance.json
|
||||
CLEANUP_OUTCOME: ${{ steps.environment_cleanup.outcome }}
|
||||
run: python3 scripts/functional_environment.py finish --suite performance --cleanup-outcome "$CLEANUP_OUTCOME"
|
||||
|
||||
- name: Retain environment ownership and cleanup receipt
|
||||
if: ${{ always() }}
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: functional-environment-performance-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/environment-performance.json
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Notify on failure
|
||||
if: failure()
|
||||
run: |
|
||||
@@ -375,13 +354,3 @@ jobs:
|
||||
name: functional-chain-performance-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/chain-performance-${{ github.run_id }}-${{ github.run_attempt }}/performance.json
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Enforce suite and report result
|
||||
if: ${{ always() }}
|
||||
env:
|
||||
SUITE_OUTCOME: ${{ steps.benchmark.outcome }}
|
||||
REPORT_OUTCOME: ${{ steps.chain_report.outcome }}
|
||||
run: |
|
||||
test "$SUITE_OUTCOME" = success
|
||||
test "$REPORT_OUTCOME" = success
|
||||
python3 scripts/functional_chain_evidence.py verify-report --suite performance --report "${RUSTFS_RESULT_DIR}/summary.tsv"
|
||||
|
||||
@@ -56,6 +56,9 @@ on:
|
||||
description: 'Reset the nodes after the test (DESTROYS test data/config)'
|
||||
type: boolean
|
||||
default: true
|
||||
repository_dispatch:
|
||||
# Chain handoff: dispatched when the heal suite finishes.
|
||||
types: [rustfs-chain-pool]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -83,13 +86,14 @@ env:
|
||||
RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
|
||||
|
||||
jobs:
|
||||
# Pool expansion owns an independent reusable suite.
|
||||
# Pool expansion: dispatched by the heal suite's chain handoff. Heal
|
||||
# itself lives in rustfs-heal-test.yml and runs exactly once per chain.
|
||||
pool-expansion-test:
|
||||
name: Pool expansion / decommission test
|
||||
runs-on: smoke-testing
|
||||
# Leave setup and finalizer headroom beyond the default 240-minute test.
|
||||
timeout-minutes: 360
|
||||
if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' }}
|
||||
if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
|
||||
env:
|
||||
RUSTFS_POOL_ADMIN_ENDPOINT: ${{ secrets.RUSTFS_POOL_ADMIN_ENDPOINT || vars.RUSTFS_POOL_ADMIN_ENDPOINT || 'http://rustfs-node1:9000' }}
|
||||
RUSTFS_POOL_PROXY_ENDPOINT: http://127.0.0.1:19000
|
||||
@@ -145,13 +149,6 @@ jobs:
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Admit clean shared environment
|
||||
id: environment
|
||||
timeout-minutes: 5
|
||||
env:
|
||||
FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-pool.json
|
||||
run: python3 scripts/functional_environment.py enter --suite pool
|
||||
|
||||
- name: Cleanup environment (before)
|
||||
timeout-minutes: 5
|
||||
if: ${{ inputs.cleanup_before != 'false' }}
|
||||
@@ -273,7 +270,8 @@ jobs:
|
||||
# timeout-minutes requires; the bare || form yielded the string '240'
|
||||
# and the runner rejected the template (Unexpected value '240').
|
||||
timeout-minutes: ${{ fromJSON(inputs.pool_timeout_minutes || '240') }}
|
||||
# Collect reports and backlog evidence before the final suite verdict.
|
||||
# Case failures keep the run green: the report and the backlog
|
||||
# issue manager carry the product signal.
|
||||
continue-on-error: true
|
||||
run: |
|
||||
set -o pipefail
|
||||
@@ -700,9 +698,8 @@ jobs:
|
||||
./auto-testing/rustfs_pool_nginx_stage.sh cleanup
|
||||
|
||||
- name: Cleanup environment (after)
|
||||
id: environment_cleanup
|
||||
timeout-minutes: 5
|
||||
if: ${{ always() && steps.topology.outcome == 'success' && inputs.cleanup_after != 'false' && steps.environment.outcome == 'success' }}
|
||||
if: ${{ always() && steps.topology.outcome == 'success' && inputs.cleanup_after != 'false' }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
|
||||
@@ -720,21 +717,54 @@ jobs:
|
||||
'
|
||||
done
|
||||
|
||||
- name: Release environment only after successful cleanup
|
||||
if: ${{ always() && steps.environment.outcome == 'success' }}
|
||||
timeout-minutes: 5
|
||||
- name: "Continue functional chain (next: Security)"
|
||||
# Only chain-triggered runs forward to the next suite; standalone
|
||||
# workflow_dispatch runs stop after their own cleanup. A failed
|
||||
# handoff must never pass silently: it retries, then files an alert
|
||||
# issue in rustfs/backlog so a stalled chain is visible.
|
||||
if: ${{ always() && github.event_name == 'repository_dispatch' }}
|
||||
continue-on-error: true
|
||||
env:
|
||||
FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-pool.json
|
||||
CLEANUP_OUTCOME: ${{ steps.environment_cleanup.outcome }}
|
||||
run: python3 scripts/functional_environment.py finish --suite pool --cleanup-outcome "$CLEANUP_OUTCOME"
|
||||
|
||||
- name: Retain environment ownership and cleanup receipt
|
||||
if: ${{ always() }}
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: functional-environment-pool-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/environment-pool.json
|
||||
if-no-files-found: error
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
run: |
|
||||
set -uo pipefail
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
|
||||
exit 1
|
||||
fi
|
||||
DISPATCHED=0
|
||||
for attempt in 1 2 3; do
|
||||
if gh api --method POST repos/rustfs/rustfs/dispatches \
|
||||
-f event_type='rustfs-chain-security' \
|
||||
-F 'client_payload[from_suite]=pool'; then
|
||||
echo "dispatched next suite Security (attempt ${attempt})"
|
||||
DISPATCHED=1
|
||||
break
|
||||
fi
|
||||
echo "dispatch attempt ${attempt} failed; retrying in ${attempt}0s" >&2
|
||||
sleep "${attempt}0"
|
||||
done
|
||||
if [ "${DISPATCHED:-0}" -ne 1 ]; then
|
||||
echo "ERROR: functional chain stalled: could not dispatch Security after 3 attempts" >&2
|
||||
TITLE="[functional][chain] stalled after pool (run ${GITHUB_RUN_ID})"
|
||||
BODY_FILE="$(mktemp)"
|
||||
{
|
||||
echo "The functional chain could not hand off from **pool** to **Security** after 3 attempts."
|
||||
echo ""
|
||||
echo "- Failed suite job: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
|
||||
echo "- Expected next event: 'rustfs-chain-security'"
|
||||
echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
|
||||
echo "- Recovery: re-dispatch manually with"
|
||||
FENCE="$(printf "\x60\x60\x60")"; echo " ${FENCE}"
|
||||
echo " gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-security'"
|
||||
FENCE="$(printf "\x60\x60\x60")"; echo " ${FENCE}"
|
||||
} > "${BODY_FILE}"
|
||||
gh issue create -R rustfs/backlog --title "${TITLE}" \
|
||||
--body-file "${BODY_FILE}" --label functional-test \
|
||||
|| gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}" \
|
||||
|| echo "could not file the stall alert issue either; check the token" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Notify on failure
|
||||
if: failure()
|
||||
@@ -763,13 +793,3 @@ jobs:
|
||||
name: functional-chain-pool-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/chain-pool-${{ github.run_id }}-${{ github.run_attempt }}/pool.json
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Enforce suite and report result
|
||||
if: ${{ always() }}
|
||||
env:
|
||||
SUITE_OUTCOME: ${{ steps.pool_test.outcome }}
|
||||
REPORT_OUTCOME: ${{ steps.chain_report.outcome }}
|
||||
run: |
|
||||
test "$SUITE_OUTCOME" = success
|
||||
test "$REPORT_OUTCOME" = success
|
||||
python3 scripts/functional_chain_evidence.py verify-report --suite pool --report "${POOL_ARTIFACT_DIR}/pool-steps.md"
|
||||
|
||||
@@ -39,6 +39,9 @@ on:
|
||||
- bucket
|
||||
- site
|
||||
default: all
|
||||
repository_dispatch:
|
||||
# Chain handoff: dispatched when the security suite finishes.
|
||||
types: [rustfs-chain-replication]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -65,7 +68,7 @@ jobs:
|
||||
replication-test:
|
||||
runs-on: smoke-testing
|
||||
timeout-minutes: 60
|
||||
if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' }}
|
||||
if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
|
||||
steps:
|
||||
- name: Checkout repository (for report parser)
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
@@ -110,13 +113,6 @@ jobs:
|
||||
aws --version
|
||||
df -h /data | tail -1 || true
|
||||
|
||||
- name: Admit clean shared environment
|
||||
id: environment
|
||||
timeout-minutes: 5
|
||||
env:
|
||||
FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-replication.json
|
||||
run: python3 scripts/functional_environment.py enter --suite replication
|
||||
|
||||
- name: Cleanup environment (before)
|
||||
timeout-minutes: 5
|
||||
run: |
|
||||
@@ -139,7 +135,8 @@ jobs:
|
||||
- name: Run replication suite
|
||||
id: test
|
||||
timeout-minutes: 45
|
||||
# Collect reports and backlog evidence before the final suite verdict.
|
||||
# Case failures keep the run green: the report and the backlog
|
||||
# issue manager carry the product signal.
|
||||
continue-on-error: true
|
||||
run: |
|
||||
set -euo pipefail
|
||||
@@ -195,8 +192,10 @@ jobs:
|
||||
RESULT=success
|
||||
fi
|
||||
|
||||
# Report harness completion here; the final verdict checks every
|
||||
# case result after diagnostics and cleanup have been retained.
|
||||
# Product gate: failing cases keep the run green — they are reported
|
||||
# below and tracked in rustfs/backlog. Only harness/environment
|
||||
# breakdowns (the suite never reached case level, or a wholesale
|
||||
# failure with zero passes) turn the workflow red.
|
||||
CASES_TOTAL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| (PASS|FAIL|UNSUPPORTED|RUNNING) \|' "${CASE_TABLE}" 2>/dev/null || true)"
|
||||
CASES_FAIL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| FAIL \|' "${CASE_TABLE}" 2>/dev/null || true)"
|
||||
CASES_TOTAL=$(( ${CASES_TOTAL:-0} + 0 )); CASES_FAIL=$(( ${CASES_FAIL:-0} + 0 ))
|
||||
@@ -235,7 +234,7 @@ jobs:
|
||||
fi
|
||||
} | tee "${REPORT_FILE}"
|
||||
cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
|
||||
# Preserve the harness verdict separately from the final case verdict.
|
||||
# Red only for harness/environment breakdowns; case failures stay green.
|
||||
[ "${HARNESS_OK}" = "1" ]
|
||||
|
||||
- name: Upload functional report to dashboard
|
||||
@@ -334,9 +333,8 @@ jobs:
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Cleanup environment (after)
|
||||
id: environment_cleanup
|
||||
timeout-minutes: 5
|
||||
if: ${{ always() && steps.environment.outcome == 'success' }}
|
||||
if: always()
|
||||
run: |
|
||||
set -euo pipefail
|
||||
read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
|
||||
@@ -354,21 +352,50 @@ jobs:
|
||||
'
|
||||
done
|
||||
|
||||
- name: Release environment only after successful cleanup
|
||||
if: ${{ always() && steps.environment.outcome == 'success' }}
|
||||
timeout-minutes: 5
|
||||
- name: "Continue functional chain (next: Fault tolerance)"
|
||||
if: ${{ always() && github.event_name == 'repository_dispatch' }}
|
||||
env:
|
||||
FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-replication.json
|
||||
CLEANUP_OUTCOME: ${{ steps.environment_cleanup.outcome }}
|
||||
run: python3 scripts/functional_environment.py finish --suite replication --cleanup-outcome "$CLEANUP_OUTCOME"
|
||||
|
||||
- name: Retain environment ownership and cleanup receipt
|
||||
if: ${{ always() }}
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: functional-environment-replication-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/environment-replication.json
|
||||
if-no-files-found: error
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
run: |
|
||||
set -uo pipefail
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
|
||||
exit 1
|
||||
fi
|
||||
DISPATCHED=0
|
||||
for attempt in 1 2 3; do
|
||||
if gh api --method POST repos/rustfs/rustfs/dispatches \
|
||||
-f event_type='rustfs-chain-fault-tolerance' \
|
||||
-F 'client_payload[from_suite]=replication'; then
|
||||
echo "dispatched next suite Fault tolerance (attempt ${attempt})"
|
||||
DISPATCHED=1
|
||||
break
|
||||
fi
|
||||
echo "dispatch attempt ${attempt} failed; retrying in ${attempt}0s" >&2
|
||||
sleep "${attempt}0"
|
||||
done
|
||||
if [ "${DISPATCHED:-0}" -ne 1 ]; then
|
||||
echo "ERROR: functional chain stalled: could not dispatch Fault tolerance after 3 attempts" >&2
|
||||
TITLE="[functional][chain] stalled after replication (run ${GITHUB_RUN_ID})"
|
||||
BODY_FILE="$(mktemp)"
|
||||
trap 'rm -f "${BODY_FILE}"' EXIT
|
||||
{
|
||||
echo "The functional chain could not hand off from **replication** to **Fault tolerance** after 3 attempts."
|
||||
echo ""
|
||||
echo "- Failed suite job: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
|
||||
echo "- Expected next event: 'rustfs-chain-fault-tolerance'"
|
||||
echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
|
||||
echo "- Recovery: re-dispatch manually with"
|
||||
FENCE="$(printf "\x60\x60\x60")"; echo " ${FENCE}"
|
||||
echo " gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-fault-tolerance'"
|
||||
FENCE="$(printf "\x60\x60\x60")"; echo " ${FENCE}"
|
||||
} > "${BODY_FILE}"
|
||||
gh issue create -R rustfs/backlog --title "${TITLE}" \
|
||||
--body-file "${BODY_FILE}" --label functional-test \
|
||||
|| gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}" \
|
||||
|| echo "could not file the stall alert issue either; check the token" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Notify on failure
|
||||
if: failure()
|
||||
@@ -397,13 +424,3 @@ jobs:
|
||||
name: functional-chain-replication-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/chain-replication-${{ github.run_id }}-${{ github.run_attempt }}/replication.json
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Enforce suite and report result
|
||||
if: ${{ always() }}
|
||||
env:
|
||||
SUITE_OUTCOME: ${{ steps.test.outcome }}
|
||||
REPORT_OUTCOME: ${{ steps.chain_report.outcome }}
|
||||
run: |
|
||||
test "$SUITE_OUTCOME" = success
|
||||
test "$REPORT_OUTCOME" = success
|
||||
python3 scripts/functional_chain_evidence.py verify-report --suite replication --report "${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"
|
||||
|
||||
@@ -17,6 +17,9 @@ on:
|
||||
description: 'Direct .deb URL (nightly/R2/dev). Overrides rustfs_version.'
|
||||
required: false
|
||||
type: string
|
||||
repository_dispatch:
|
||||
# Chain handoff: dispatched when the upgrade suite finishes.
|
||||
types: [rustfs-chain-s3]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -41,7 +44,7 @@ jobs:
|
||||
s3-compat-test:
|
||||
runs-on: smoke-testing
|
||||
timeout-minutes: 60
|
||||
if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' }}
|
||||
if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
|
||||
steps:
|
||||
- name: Checkout repository (for report parser)
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
@@ -85,13 +88,6 @@ jobs:
|
||||
openssl version
|
||||
df -h /data | tail -1
|
||||
|
||||
- name: Admit clean shared environment
|
||||
id: environment
|
||||
timeout-minutes: 5
|
||||
env:
|
||||
FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-s3.json
|
||||
run: python3 scripts/functional_environment.py enter --suite s3
|
||||
|
||||
- name: Cleanup environment (before)
|
||||
timeout-minutes: 5
|
||||
run: |
|
||||
@@ -114,7 +110,8 @@ jobs:
|
||||
- name: Run S3 compatibility suite
|
||||
id: test
|
||||
timeout-minutes: 45
|
||||
# Collect reports and backlog evidence before the final suite verdict.
|
||||
# Case failures keep the run green: the report and the backlog
|
||||
# issue manager carry the product signal.
|
||||
continue-on-error: true
|
||||
run: |
|
||||
set -euo pipefail
|
||||
@@ -164,8 +161,10 @@ jobs:
|
||||
RESULT=success
|
||||
fi
|
||||
|
||||
# Report harness completion here; the final verdict checks every
|
||||
# case result after diagnostics and cleanup have been retained.
|
||||
# Product gate: failing cases keep the run green — they are reported
|
||||
# below and tracked in rustfs/backlog. Only harness/environment
|
||||
# breakdowns (the suite never reached case level, or a wholesale
|
||||
# failure with zero passes) turn the workflow red.
|
||||
CASES_TOTAL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| (PASS|FAIL|UNSUPPORTED|RUNNING) \|' "${CASE_TABLE}" 2>/dev/null || true)"
|
||||
CASES_FAIL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| FAIL \|' "${CASE_TABLE}" 2>/dev/null || true)"
|
||||
CASES_TOTAL=$(( ${CASES_TOTAL:-0} + 0 )); CASES_FAIL=$(( ${CASES_FAIL:-0} + 0 ))
|
||||
@@ -204,7 +203,7 @@ jobs:
|
||||
fi
|
||||
} | tee "${REPORT_FILE}"
|
||||
cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
|
||||
# Preserve the harness verdict separately from the final case verdict.
|
||||
# Red only for harness/environment breakdowns; case failures stay green.
|
||||
[ "${HARNESS_OK}" = "1" ]
|
||||
|
||||
- name: Upload functional report to dashboard
|
||||
@@ -303,9 +302,8 @@ jobs:
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Cleanup environment (after)
|
||||
id: environment_cleanup
|
||||
timeout-minutes: 5
|
||||
if: ${{ always() && steps.environment.outcome == 'success' }}
|
||||
if: always()
|
||||
run: |
|
||||
set -euo pipefail
|
||||
read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
|
||||
@@ -323,21 +321,54 @@ jobs:
|
||||
'
|
||||
done
|
||||
|
||||
- name: Release environment only after successful cleanup
|
||||
if: ${{ always() && steps.environment.outcome == 'success' }}
|
||||
timeout-minutes: 5
|
||||
- name: "Continue functional chain (next: KMS)"
|
||||
# Only chain-triggered runs forward to the next suite; standalone
|
||||
# workflow_dispatch runs stop after their own cleanup. A failed
|
||||
# handoff must never pass silently: it retries, then files an alert
|
||||
# issue in rustfs/backlog so a stalled chain is visible.
|
||||
if: ${{ always() && github.event_name == 'repository_dispatch' }}
|
||||
continue-on-error: true
|
||||
env:
|
||||
FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-s3.json
|
||||
CLEANUP_OUTCOME: ${{ steps.environment_cleanup.outcome }}
|
||||
run: python3 scripts/functional_environment.py finish --suite s3 --cleanup-outcome "$CLEANUP_OUTCOME"
|
||||
|
||||
- name: Retain environment ownership and cleanup receipt
|
||||
if: ${{ always() }}
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: functional-environment-s3-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/environment-s3.json
|
||||
if-no-files-found: error
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
run: |
|
||||
set -uo pipefail
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
|
||||
exit 1
|
||||
fi
|
||||
DISPATCHED=0
|
||||
for attempt in 1 2 3; do
|
||||
if gh api --method POST repos/rustfs/rustfs/dispatches \
|
||||
-f event_type='rustfs-chain-kms' \
|
||||
-F 'client_payload[from_suite]=s3'; then
|
||||
echo "dispatched next suite KMS (attempt ${attempt})"
|
||||
DISPATCHED=1
|
||||
break
|
||||
fi
|
||||
echo "dispatch attempt ${attempt} failed; retrying in ${attempt}0s" >&2
|
||||
sleep "${attempt}0"
|
||||
done
|
||||
if [ "${DISPATCHED:-0}" -ne 1 ]; then
|
||||
echo "ERROR: functional chain stalled: could not dispatch KMS after 3 attempts" >&2
|
||||
TITLE="[functional][chain] stalled after s3 (run ${GITHUB_RUN_ID})"
|
||||
BODY_FILE="$(mktemp)"
|
||||
{
|
||||
echo "The functional chain could not hand off from **s3** to **KMS** after 3 attempts."
|
||||
echo ""
|
||||
echo "- Failed suite job: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
|
||||
echo "- Expected next event: 'rustfs-chain-kms'"
|
||||
echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
|
||||
echo "- Recovery: re-dispatch manually with"
|
||||
FENCE="$(printf "\x60\x60\x60")"; echo " ${FENCE}"
|
||||
echo " gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-kms'"
|
||||
FENCE="$(printf "\x60\x60\x60")"; echo " ${FENCE}"
|
||||
} > "${BODY_FILE}"
|
||||
gh issue create -R rustfs/backlog --title "${TITLE}" \
|
||||
--body-file "${BODY_FILE}" --label functional-test \
|
||||
|| gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}" \
|
||||
|| echo "could not file the stall alert issue either; check the token" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Notify on failure
|
||||
if: failure()
|
||||
@@ -365,13 +396,3 @@ jobs:
|
||||
name: functional-chain-s3-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/chain-s3-${{ github.run_id }}-${{ github.run_attempt }}/s3.json
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Enforce suite and report result
|
||||
if: ${{ always() }}
|
||||
env:
|
||||
SUITE_OUTCOME: ${{ steps.test.outcome }}
|
||||
REPORT_OUTCOME: ${{ steps.chain_report.outcome }}
|
||||
run: |
|
||||
test "$SUITE_OUTCOME" = success
|
||||
test "$REPORT_OUTCOME" = success
|
||||
python3 scripts/functional_chain_evidence.py verify-report --suite s3 --report "${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"
|
||||
|
||||
@@ -52,6 +52,9 @@ on:
|
||||
description: 'Reset the nodes after the test (DESTROYS test data/config)'
|
||||
type: boolean
|
||||
default: true
|
||||
repository_dispatch:
|
||||
# Chain handoff: dispatched when the pool expansion suite finishes.
|
||||
types: [rustfs-chain-security]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -78,7 +81,7 @@ jobs:
|
||||
security-test:
|
||||
runs-on: smoke-testing
|
||||
timeout-minutes: 60
|
||||
if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' }}
|
||||
if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
|
||||
steps:
|
||||
# Checkout the repository into its own subdirectory. Checking out at
|
||||
# the workspace root would wipe the auto-testing clone above (that is
|
||||
@@ -123,13 +126,6 @@ jobs:
|
||||
docker --version || true
|
||||
df -h /data | tail -1
|
||||
|
||||
- name: Admit clean shared environment
|
||||
id: environment
|
||||
timeout-minutes: 5
|
||||
env:
|
||||
FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-security.json
|
||||
run: python3 rustfs-repo/scripts/functional_environment.py enter --suite security
|
||||
|
||||
- name: Cleanup environment (before)
|
||||
timeout-minutes: 5
|
||||
if: ${{ inputs.cleanup_before != 'false' || github.event_name != 'workflow_dispatch' }}
|
||||
@@ -153,7 +149,8 @@ jobs:
|
||||
- name: Run security suite
|
||||
id: test
|
||||
timeout-minutes: 45
|
||||
# Collect reports and backlog evidence before the final suite verdict.
|
||||
# Case failures keep the run green: the report and the backlog
|
||||
# issue manager carry the product signal.
|
||||
continue-on-error: true
|
||||
env:
|
||||
REPORT_FILE: ${{ env.SECURITY_ARTIFACTS_DIR }}/suite-report.md
|
||||
@@ -197,8 +194,9 @@ jobs:
|
||||
RESULT=success
|
||||
fi
|
||||
|
||||
# The final verdict checks cases after diagnostics and cleanup.
|
||||
# The suite log lines carry ANSI
|
||||
# Product gate: failing cases keep the run green — they are reported
|
||||
# below and tracked in rustfs/backlog. Only harness/environment
|
||||
# breakdowns turn the workflow red. The suite log lines carry ANSI
|
||||
# color escapes both before and after the verdict tag
|
||||
# (e.g. "\e[1;31m[FAIL]\e[0m STS-105 ..."), so the pattern allows
|
||||
# any number of escapes on each side of it.
|
||||
@@ -238,7 +236,7 @@ jobs:
|
||||
fi
|
||||
} > "${SECURITY_ARTIFACTS_DIR}/report.md"
|
||||
cat "${SECURITY_ARTIFACTS_DIR}/report.md" >> "${GITHUB_STEP_SUMMARY}"
|
||||
# Preserve the harness verdict separately from the final case verdict.
|
||||
# Red only for harness/environment breakdowns; case failures stay green.
|
||||
[ "${HARNESS_OK}" = "1" ]
|
||||
|
||||
- name: Upload functional report to dashboard
|
||||
@@ -338,9 +336,8 @@ jobs:
|
||||
retention-days: 3
|
||||
|
||||
- name: Cleanup environment (after)
|
||||
id: environment_cleanup
|
||||
timeout-minutes: 5
|
||||
if: ${{ always() && (inputs.cleanup_after != 'false' || github.event_name != 'workflow_dispatch') && steps.environment.outcome == 'success' }}
|
||||
if: ${{ always() && (inputs.cleanup_after != 'false' || github.event_name != 'workflow_dispatch') }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
|
||||
@@ -358,21 +355,23 @@ jobs:
|
||||
'
|
||||
done
|
||||
|
||||
- name: Release environment only after successful cleanup
|
||||
if: ${{ always() && steps.environment.outcome == 'success' }}
|
||||
timeout-minutes: 5
|
||||
- name: "Continue functional chain (next: Replication)"
|
||||
# Only chain-triggered runs forward to the next suite; standalone
|
||||
# workflow_dispatch runs stop after their own cleanup.
|
||||
if: ${{ always() && github.event_name == 'repository_dispatch' }}
|
||||
continue-on-error: true
|
||||
env:
|
||||
FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-security.json
|
||||
CLEANUP_OUTCOME: ${{ steps.environment_cleanup.outcome }}
|
||||
run: python3 rustfs-repo/scripts/functional_environment.py finish --suite security --cleanup-outcome "$CLEANUP_OUTCOME"
|
||||
|
||||
- name: Retain environment ownership and cleanup receipt
|
||||
if: ${{ always() }}
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: functional-environment-security-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/environment-security.json
|
||||
if-no-files-found: error
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "Dispatching next functional suite: Replication"
|
||||
gh api --method POST repos/rustfs/rustfs/dispatches \
|
||||
-f event_type='rustfs-chain-replication' \
|
||||
-F 'client_payload[from_suite]=security'
|
||||
|
||||
- name: Notify on failure
|
||||
if: failure()
|
||||
@@ -401,13 +400,3 @@ jobs:
|
||||
name: functional-chain-security-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/chain-security-${{ github.run_id }}-${{ github.run_attempt }}/security.json
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Enforce suite and report result
|
||||
if: ${{ always() }}
|
||||
env:
|
||||
SUITE_OUTCOME: ${{ steps.test.outcome }}
|
||||
REPORT_OUTCOME: ${{ steps.report.outcome }}
|
||||
run: |
|
||||
test "$SUITE_OUTCOME" = success
|
||||
test "$REPORT_OUTCOME" = success
|
||||
python3 rustfs-repo/scripts/functional_chain_evidence.py verify-report --suite security --report "${SECURITY_ARTIFACTS_DIR}/suite-report.md"
|
||||
|
||||
@@ -26,6 +26,9 @@ on:
|
||||
- single-multi
|
||||
- multi-multi
|
||||
default: all
|
||||
repository_dispatch:
|
||||
# Chain handoff: dispatched when the tier suite finishes.
|
||||
types: [rustfs-chain-storage]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -50,7 +53,7 @@ jobs:
|
||||
storage-test:
|
||||
runs-on: smoke-testing
|
||||
timeout-minutes: 60
|
||||
if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' }}
|
||||
if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
|
||||
steps:
|
||||
- name: Checkout repository (for report parser)
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
@@ -94,13 +97,6 @@ jobs:
|
||||
openssl version
|
||||
df -h /data | tail -1
|
||||
|
||||
- name: Admit clean shared environment
|
||||
id: environment
|
||||
timeout-minutes: 5
|
||||
env:
|
||||
FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-storage.json
|
||||
run: python3 scripts/functional_environment.py enter --suite storage
|
||||
|
||||
- name: Cleanup environment (before)
|
||||
timeout-minutes: 5
|
||||
run: |
|
||||
@@ -123,7 +119,8 @@ jobs:
|
||||
- name: Run storage engine suite
|
||||
id: test
|
||||
timeout-minutes: 45
|
||||
# Collect reports and backlog evidence before the final suite verdict.
|
||||
# Case failures keep the run green: the report and the backlog
|
||||
# issue manager carry the product signal.
|
||||
continue-on-error: true
|
||||
run: |
|
||||
set -euo pipefail
|
||||
@@ -179,8 +176,10 @@ jobs:
|
||||
RESULT=success
|
||||
fi
|
||||
|
||||
# Report harness completion here; the final verdict checks every
|
||||
# case result after diagnostics and cleanup have been retained.
|
||||
# Product gate: failing cases keep the run green — they are reported
|
||||
# below and tracked in rustfs/backlog. Only harness/environment
|
||||
# breakdowns (the suite never reached case level, or a wholesale
|
||||
# failure with zero passes) turn the workflow red.
|
||||
CASES_TOTAL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| (PASS|FAIL|UNSUPPORTED|RUNNING) \|' "${CASE_TABLE}" 2>/dev/null || true)"
|
||||
CASES_FAIL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| FAIL \|' "${CASE_TABLE}" 2>/dev/null || true)"
|
||||
CASES_TOTAL=$(( ${CASES_TOTAL:-0} + 0 )); CASES_FAIL=$(( ${CASES_FAIL:-0} + 0 ))
|
||||
@@ -219,7 +218,7 @@ jobs:
|
||||
fi
|
||||
} | tee "${REPORT_FILE}"
|
||||
cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
|
||||
# Preserve the harness verdict separately from the final case verdict.
|
||||
# Red only for harness/environment breakdowns; case failures stay green.
|
||||
[ "${HARNESS_OK}" = "1" ]
|
||||
|
||||
- name: Upload functional report to dashboard
|
||||
@@ -318,9 +317,8 @@ jobs:
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Cleanup environment (after)
|
||||
id: environment_cleanup
|
||||
timeout-minutes: 5
|
||||
if: ${{ always() && steps.environment.outcome == 'success' }}
|
||||
if: always()
|
||||
run: |
|
||||
set -euo pipefail
|
||||
read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
|
||||
@@ -338,21 +336,54 @@ jobs:
|
||||
'
|
||||
done
|
||||
|
||||
- name: Release environment only after successful cleanup
|
||||
if: ${{ always() && steps.environment.outcome == 'success' }}
|
||||
timeout-minutes: 5
|
||||
- name: "Continue functional chain (next: Heal)"
|
||||
# Only chain-triggered runs forward to the next suite; standalone
|
||||
# workflow_dispatch runs stop after their own cleanup. A failed
|
||||
# handoff must never pass silently: it retries, then files an alert
|
||||
# issue in rustfs/backlog so a stalled chain is visible.
|
||||
if: ${{ always() && github.event_name == 'repository_dispatch' }}
|
||||
continue-on-error: true
|
||||
env:
|
||||
FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-storage.json
|
||||
CLEANUP_OUTCOME: ${{ steps.environment_cleanup.outcome }}
|
||||
run: python3 scripts/functional_environment.py finish --suite storage --cleanup-outcome "$CLEANUP_OUTCOME"
|
||||
|
||||
- name: Retain environment ownership and cleanup receipt
|
||||
if: ${{ always() }}
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: functional-environment-storage-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/environment-storage.json
|
||||
if-no-files-found: error
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
run: |
|
||||
set -uo pipefail
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
|
||||
exit 1
|
||||
fi
|
||||
DISPATCHED=0
|
||||
for attempt in 1 2 3; do
|
||||
if gh api --method POST repos/rustfs/rustfs/dispatches \
|
||||
-f event_type='rustfs-chain-heal' \
|
||||
-F 'client_payload[from_suite]=storage'; then
|
||||
echo "dispatched next suite Heal (attempt ${attempt})"
|
||||
DISPATCHED=1
|
||||
break
|
||||
fi
|
||||
echo "dispatch attempt ${attempt} failed; retrying in ${attempt}0s" >&2
|
||||
sleep "${attempt}0"
|
||||
done
|
||||
if [ "${DISPATCHED:-0}" -ne 1 ]; then
|
||||
echo "ERROR: functional chain stalled: could not dispatch Heal after 3 attempts" >&2
|
||||
TITLE="[functional][chain] stalled after storage (run ${GITHUB_RUN_ID})"
|
||||
BODY_FILE="$(mktemp)"
|
||||
{
|
||||
echo "The functional chain could not hand off from **storage** to **Heal** after 3 attempts."
|
||||
echo ""
|
||||
echo "- Failed suite job: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
|
||||
echo "- Expected next event: 'rustfs-chain-heal'"
|
||||
echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
|
||||
echo "- Recovery: re-dispatch manually with"
|
||||
FENCE="$(printf "\x60\x60\x60")"; echo " ${FENCE}"
|
||||
echo " gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-heal'"
|
||||
FENCE="$(printf "\x60\x60\x60")"; echo " ${FENCE}"
|
||||
} > "${BODY_FILE}"
|
||||
gh issue create -R rustfs/backlog --title "${TITLE}" \
|
||||
--body-file "${BODY_FILE}" --label functional-test \
|
||||
|| gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}" \
|
||||
|| echo "could not file the stall alert issue either; check the token" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Notify on failure
|
||||
if: failure()
|
||||
@@ -380,13 +411,3 @@ jobs:
|
||||
name: functional-chain-storage-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/chain-storage-${{ github.run_id }}-${{ github.run_attempt }}/storage.json
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Enforce suite and report result
|
||||
if: ${{ always() }}
|
||||
env:
|
||||
SUITE_OUTCOME: ${{ steps.test.outcome }}
|
||||
REPORT_OUTCOME: ${{ steps.chain_report.outcome }}
|
||||
run: |
|
||||
test "$SUITE_OUTCOME" = success
|
||||
test "$REPORT_OUTCOME" = success
|
||||
python3 scripts/functional_chain_evidence.py verify-report --suite storage --report "${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"
|
||||
|
||||
@@ -89,13 +89,6 @@ jobs:
|
||||
aws --version
|
||||
python3 --version
|
||||
|
||||
- name: Admit clean shared environment
|
||||
id: environment
|
||||
timeout-minutes: 5
|
||||
env:
|
||||
FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-table.json
|
||||
run: python3 scripts/functional_environment.py enter --suite table
|
||||
|
||||
- name: Cleanup environment (before)
|
||||
timeout-minutes: 5
|
||||
continue-on-error: true
|
||||
@@ -169,8 +162,10 @@ jobs:
|
||||
RESULT=success
|
||||
fi
|
||||
|
||||
# Report harness completion here; the final verdict checks every
|
||||
# case result after diagnostics and cleanup have been retained.
|
||||
# Product gate: failing cases keep the run green — they are reported
|
||||
# below and tracked in rustfs/backlog. Only harness/environment
|
||||
# breakdowns (the suite never reached case level, or a wholesale
|
||||
# failure with zero passes) turn the workflow red.
|
||||
CASES_TOTAL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| (PASS|FAIL|UNSUPPORTED|RUNNING) \|' "${CASE_TABLE}" 2>/dev/null || true)"
|
||||
CASES_FAIL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| FAIL \|' "${CASE_TABLE}" 2>/dev/null || true)"
|
||||
CASES_TOTAL=$(( ${CASES_TOTAL:-0} + 0 )); CASES_FAIL=$(( ${CASES_FAIL:-0} + 0 ))
|
||||
@@ -208,7 +203,7 @@ jobs:
|
||||
fi
|
||||
} | tee "${REPORT_FILE}"
|
||||
cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
|
||||
# Preserve the harness verdict separately from the final case verdict.
|
||||
# Red only for harness/environment breakdowns; case failures stay green.
|
||||
[ "${HARNESS_OK}" = "1" ]
|
||||
|
||||
- name: Upload functional report to dashboard
|
||||
@@ -297,12 +292,11 @@ jobs:
|
||||
retention-days: 30
|
||||
|
||||
- name: Cleanup environment (after)
|
||||
id: environment_cleanup
|
||||
timeout-minutes: 5
|
||||
if: ${{ always() && steps.environment.outcome == 'success' }}
|
||||
if: always()
|
||||
continue-on-error: true
|
||||
run: |
|
||||
set -euo pipefail
|
||||
cleanup_status=0
|
||||
set -uo pipefail
|
||||
for node in ${RUSTFS_NODES}; do
|
||||
ssh -o BatchMode=yes -o ConnectTimeout=10 "${RUSTFS_SSH_USER}@${node}" '
|
||||
set -euo pipefail
|
||||
@@ -313,25 +307,26 @@ jobs:
|
||||
fi
|
||||
for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
|
||||
${SUDO} rm -f /etc/default/rustfs
|
||||
' || cleanup_status=$?
|
||||
' || echo "cleanup failed on ${node} (continuing)"
|
||||
done
|
||||
exit "${cleanup_status}"
|
||||
|
||||
- name: Release environment only after successful cleanup
|
||||
if: ${{ always() && steps.environment.outcome == 'success' }}
|
||||
timeout-minutes: 5
|
||||
- name: "Continue functional chain (next: Performance)"
|
||||
# Only chain-triggered runs forward to the next suite; standalone
|
||||
# workflow_dispatch runs stop after their own cleanup.
|
||||
if: ${{ always() && github.event_name == 'repository_dispatch' }}
|
||||
continue-on-error: true
|
||||
env:
|
||||
FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-table.json
|
||||
CLEANUP_OUTCOME: ${{ steps.environment_cleanup.outcome }}
|
||||
run: python3 scripts/functional_environment.py finish --suite table --cleanup-outcome "$CLEANUP_OUTCOME"
|
||||
|
||||
- name: Retain environment ownership and cleanup receipt
|
||||
if: ${{ always() }}
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: functional-environment-table-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/environment-table.json
|
||||
if-no-files-found: error
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "Dispatching next functional suite: Performance"
|
||||
gh api --method POST repos/rustfs/rustfs/dispatches \
|
||||
-f event_type='rustfs-chain-performance' \
|
||||
-F 'client_payload[from_suite]=table'
|
||||
|
||||
- name: Notify on failure
|
||||
if: failure()
|
||||
@@ -363,13 +358,3 @@ jobs:
|
||||
name: functional-chain-table-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/chain-table-${{ github.run_id }}-${{ github.run_attempt }}/table.json
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Enforce suite and report result
|
||||
if: ${{ always() }}
|
||||
env:
|
||||
SUITE_OUTCOME: ${{ steps.test.outcome }}
|
||||
REPORT_OUTCOME: ${{ steps.chain_report.outcome }}
|
||||
run: |
|
||||
test "$SUITE_OUTCOME" = success
|
||||
test "$REPORT_OUTCOME" = success
|
||||
python3 scripts/functional_chain_evidence.py verify-report --suite table --report "${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"
|
||||
|
||||
@@ -37,6 +37,9 @@ on:
|
||||
required: false
|
||||
default: false
|
||||
type: boolean
|
||||
repository_dispatch:
|
||||
# Chain handoff: dispatched when the KMS suite finishes.
|
||||
types: [rustfs-chain-tier]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -65,7 +68,7 @@ jobs:
|
||||
tier-test:
|
||||
runs-on: smoke-testing
|
||||
timeout-minutes: 60
|
||||
if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' }}
|
||||
if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
|
||||
steps:
|
||||
- name: Checkout chain tooling
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
@@ -213,13 +216,6 @@ jobs:
|
||||
openssl version
|
||||
df -h /data | tail -1
|
||||
|
||||
- name: Admit clean shared environment
|
||||
id: environment
|
||||
timeout-minutes: 5
|
||||
env:
|
||||
FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-tier.json
|
||||
run: python3 scripts/functional_environment.py enter --suite tier
|
||||
|
||||
- name: Cleanup environment (before)
|
||||
timeout-minutes: 5
|
||||
run: |
|
||||
@@ -273,7 +269,8 @@ jobs:
|
||||
- name: Run tier suite
|
||||
id: test
|
||||
timeout-minutes: 45
|
||||
# Collect reports and backlog evidence before the final suite verdict.
|
||||
# Case failures keep the run green: the report and the backlog
|
||||
# issue manager carry the product signal.
|
||||
continue-on-error: true
|
||||
env:
|
||||
PACKAGE_URL_INPUT: ${{ inputs.package_url }}
|
||||
@@ -470,9 +467,8 @@ jobs:
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Cleanup environment (after)
|
||||
id: environment_cleanup
|
||||
timeout-minutes: 5
|
||||
if: ${{ always() && steps.environment.outcome == 'success' }}
|
||||
if: always()
|
||||
run: |
|
||||
set -euo pipefail
|
||||
sudo docker rm -f rustfs-test-mqtt >/dev/null 2>&1 || true
|
||||
@@ -492,22 +488,6 @@ jobs:
|
||||
'
|
||||
done
|
||||
|
||||
- name: Release environment only after successful cleanup
|
||||
if: ${{ always() && steps.environment.outcome == 'success' }}
|
||||
timeout-minutes: 5
|
||||
env:
|
||||
FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-tier.json
|
||||
CLEANUP_OUTCOME: ${{ steps.environment_cleanup.outcome }}
|
||||
run: python3 scripts/functional_environment.py finish --suite tier --cleanup-outcome "$CLEANUP_OUTCOME"
|
||||
|
||||
- name: Retain environment ownership and cleanup receipt
|
||||
if: ${{ always() }}
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: functional-environment-tier-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/environment-tier.json
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Cleanup pinned RustFS CLI
|
||||
timeout-minutes: 2
|
||||
if: always()
|
||||
@@ -684,6 +664,55 @@ jobs:
|
||||
--package-source "${PACKAGE_SOURCE}" \
|
||||
--date "$(date -u +%Y-%m-%d)"
|
||||
|
||||
- name: "Continue functional chain (next: Storage engine)"
|
||||
# Only chain-triggered runs forward to the next suite; standalone
|
||||
# workflow_dispatch runs stop after their own cleanup. A failed
|
||||
# handoff must never pass silently: it retries, then files an alert
|
||||
# issue in rustfs/backlog so a stalled chain is visible.
|
||||
if: ${{ always() && github.event_name == 'repository_dispatch' }}
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
run: |
|
||||
set -uo pipefail
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
|
||||
exit 1
|
||||
fi
|
||||
DISPATCHED=0
|
||||
for attempt in 1 2 3; do
|
||||
if gh api --method POST repos/rustfs/rustfs/dispatches \
|
||||
-f event_type='rustfs-chain-storage' \
|
||||
-F 'client_payload[from_suite]=tier'; then
|
||||
echo "dispatched next suite Storage engine (attempt ${attempt})"
|
||||
DISPATCHED=1
|
||||
break
|
||||
fi
|
||||
echo "dispatch attempt ${attempt} failed; retrying in ${attempt}0s" >&2
|
||||
sleep "${attempt}0"
|
||||
done
|
||||
if [ "${DISPATCHED:-0}" -ne 1 ]; then
|
||||
echo "ERROR: functional chain stalled: could not dispatch Storage engine after 3 attempts" >&2
|
||||
TITLE="[functional][chain] stalled after tier (run ${GITHUB_RUN_ID})"
|
||||
BODY_FILE="$(mktemp)"
|
||||
{
|
||||
echo "The functional chain could not hand off from **tier** to **Storage engine** after 3 attempts."
|
||||
echo ""
|
||||
echo "- Failed suite job: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
|
||||
echo "- Expected next event: 'rustfs-chain-storage'"
|
||||
echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
|
||||
echo "- Recovery: re-dispatch manually with"
|
||||
FENCE="$(printf "\x60\x60\x60")"; echo " ${FENCE}"
|
||||
echo " gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-storage'"
|
||||
FENCE="$(printf "\x60\x60\x60")"; echo " ${FENCE}"
|
||||
} > "${BODY_FILE}"
|
||||
gh issue create -R rustfs/backlog --title "${TITLE}" \
|
||||
--body-file "${BODY_FILE}" --label functional-test \
|
||||
|| gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}" \
|
||||
|| echo "could not file the stall alert issue either; check the token" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Notify on failure
|
||||
if: failure()
|
||||
run: |
|
||||
@@ -710,13 +739,3 @@ jobs:
|
||||
name: functional-chain-tier-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/chain-tier-${{ github.run_id }}-${{ github.run_attempt }}/tier.json
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Enforce suite and report result
|
||||
if: ${{ always() }}
|
||||
env:
|
||||
SUITE_OUTCOME: ${{ steps.test.outcome }}
|
||||
REPORT_OUTCOME: ${{ steps.chain_report.outcome }}
|
||||
run: |
|
||||
test "$SUITE_OUTCOME" = success
|
||||
test "$REPORT_OUTCOME" = success
|
||||
python3 scripts/functional_chain_evidence.py verify-report --suite tier --report "${TIER_ARTIFACTS_DIR}/rustfs-tier-cases.md"
|
||||
|
||||
@@ -59,6 +59,9 @@ on:
|
||||
description: 'Reset the nodes after the test (DESTROYS test data/config)'
|
||||
type: boolean
|
||||
default: true
|
||||
repository_dispatch:
|
||||
# Functional-chain entry: dispatched by rustfs-functional-chain.yml.
|
||||
types: [rustfs-chain-upgrade]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -83,7 +86,7 @@ jobs:
|
||||
upgrade-test:
|
||||
runs-on: smoke-testing
|
||||
timeout-minutes: 60
|
||||
if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' }}
|
||||
if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
|
||||
steps:
|
||||
- name: Checkout repository (for report parser)
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
@@ -129,13 +132,6 @@ jobs:
|
||||
docker --version || true
|
||||
df -h /data | tail -1
|
||||
|
||||
- name: Admit clean shared environment
|
||||
id: environment
|
||||
timeout-minutes: 5
|
||||
env:
|
||||
FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-upgrade.json
|
||||
run: python3 scripts/functional_environment.py enter --suite upgrade
|
||||
|
||||
- name: Cleanup environment (before)
|
||||
timeout-minutes: 5
|
||||
if: ${{ inputs.cleanup_before != 'false' || github.event_name != 'workflow_dispatch' }}
|
||||
@@ -169,7 +165,8 @@ jobs:
|
||||
- name: Run upgrade compatibility suite
|
||||
id: test
|
||||
timeout-minutes: 45
|
||||
# Collect reports and backlog evidence before the final suite verdict.
|
||||
# Case failures keep the run green: the report and the backlog
|
||||
# issue manager carry the product signal.
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
@@ -261,8 +258,10 @@ jobs:
|
||||
RESULT=success
|
||||
fi
|
||||
|
||||
# Report harness completion here; the final verdict checks every
|
||||
# case result after diagnostics and cleanup have been retained.
|
||||
# Product gate: failing cases keep the run green — they are reported
|
||||
# below and tracked in rustfs/backlog. Only harness/environment
|
||||
# breakdowns (the suite never reached case level, or a wholesale
|
||||
# failure with zero passes) turn the workflow red.
|
||||
CASES_TOTAL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| (PASS|FAIL|UNSUPPORTED|RUNNING) \|' "${CASE_TABLE}" 2>/dev/null || true)"
|
||||
CASES_FAIL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| FAIL \|' "${CASE_TABLE}" 2>/dev/null || true)"
|
||||
CASES_TOTAL=$(( ${CASES_TOTAL:-0} + 0 )); CASES_FAIL=$(( ${CASES_FAIL:-0} + 0 ))
|
||||
@@ -305,7 +304,7 @@ jobs:
|
||||
fi
|
||||
} | tee "${REPORT_FILE}"
|
||||
cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
|
||||
# Preserve the harness verdict separately from the final case verdict.
|
||||
# Red only for harness/environment breakdowns; case failures stay green.
|
||||
[ "${HARNESS_OK}" = "1" ]
|
||||
|
||||
- name: Upload functional report to dashboard
|
||||
@@ -413,9 +412,8 @@ jobs:
|
||||
retention-days: 3
|
||||
|
||||
- name: Cleanup environment (after)
|
||||
id: environment_cleanup
|
||||
timeout-minutes: 5
|
||||
if: ${{ always() && (inputs.cleanup_after != 'false' || github.event_name != 'workflow_dispatch') && steps.environment.outcome == 'success' }}
|
||||
if: ${{ always() && (inputs.cleanup_after != 'false' || github.event_name != 'workflow_dispatch') }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
|
||||
@@ -433,21 +431,54 @@ jobs:
|
||||
'
|
||||
done
|
||||
|
||||
- name: Release environment only after successful cleanup
|
||||
if: ${{ always() && steps.environment.outcome == 'success' }}
|
||||
timeout-minutes: 5
|
||||
- name: "Continue functional chain (next: S3 compatibility)"
|
||||
# Only chain-triggered runs forward to the next suite; standalone
|
||||
# workflow_dispatch runs stop after their own cleanup. A failed
|
||||
# handoff must never pass silently: it retries, then files an alert
|
||||
# issue in rustfs/backlog so a stalled chain is visible.
|
||||
if: ${{ always() && github.event_name == 'repository_dispatch' }}
|
||||
continue-on-error: true
|
||||
env:
|
||||
FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-upgrade.json
|
||||
CLEANUP_OUTCOME: ${{ steps.environment_cleanup.outcome }}
|
||||
run: python3 scripts/functional_environment.py finish --suite upgrade --cleanup-outcome "$CLEANUP_OUTCOME"
|
||||
|
||||
- name: Retain environment ownership and cleanup receipt
|
||||
if: ${{ always() }}
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: functional-environment-upgrade-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/environment-upgrade.json
|
||||
if-no-files-found: error
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
run: |
|
||||
set -uo pipefail
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
|
||||
exit 1
|
||||
fi
|
||||
DISPATCHED=0
|
||||
for attempt in 1 2 3; do
|
||||
if gh api --method POST repos/rustfs/rustfs/dispatches \
|
||||
-f event_type='rustfs-chain-s3' \
|
||||
-F 'client_payload[from_suite]=upgrade'; then
|
||||
echo "dispatched next suite S3 compatibility (attempt ${attempt})"
|
||||
DISPATCHED=1
|
||||
break
|
||||
fi
|
||||
echo "dispatch attempt ${attempt} failed; retrying in ${attempt}0s" >&2
|
||||
sleep "${attempt}0"
|
||||
done
|
||||
if [ "${DISPATCHED:-0}" -ne 1 ]; then
|
||||
echo "ERROR: functional chain stalled: could not dispatch S3 compatibility after 3 attempts" >&2
|
||||
TITLE="[functional][chain] stalled after upgrade (run ${GITHUB_RUN_ID})"
|
||||
BODY_FILE="$(mktemp)"
|
||||
{
|
||||
echo "The functional chain could not hand off from **upgrade** to **S3 compatibility** after 3 attempts."
|
||||
echo ""
|
||||
echo "- Failed suite job: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
|
||||
echo "- Expected next event: 'rustfs-chain-s3'"
|
||||
echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
|
||||
echo "- Recovery: re-dispatch manually with"
|
||||
FENCE="$(printf "\x60\x60\x60")"; echo " ${FENCE}"
|
||||
echo " gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-s3'"
|
||||
FENCE="$(printf "\x60\x60\x60")"; echo " ${FENCE}"
|
||||
} > "${BODY_FILE}"
|
||||
gh issue create -R rustfs/backlog --title "${TITLE}" \
|
||||
--body-file "${BODY_FILE}" --label functional-test \
|
||||
|| gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}" \
|
||||
|| echo "could not file the stall alert issue either; check the token" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Notify on failure
|
||||
if: failure()
|
||||
@@ -477,13 +508,3 @@ jobs:
|
||||
name: functional-chain-upgrade-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/chain-upgrade-${{ github.run_id }}-${{ github.run_attempt }}/upgrade.json
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Enforce suite and report result
|
||||
if: ${{ always() }}
|
||||
env:
|
||||
SUITE_OUTCOME: ${{ steps.test.outcome }}
|
||||
REPORT_OUTCOME: ${{ steps.chain_report.outcome }}
|
||||
run: |
|
||||
test "$SUITE_OUTCOME" = success
|
||||
test "$REPORT_OUTCOME" = success
|
||||
python3 scripts/functional_chain_evidence.py verify-report --suite upgrade --report "${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"
|
||||
|
||||
@@ -1,457 +0,0 @@
|
||||
# Copyright 2024 RustFS Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
name: RustFS Validation Sweep
|
||||
|
||||
# Executes the migrated canonical validation cases (rustfs/auto-testing#113)
|
||||
# on the shared functional fleet. Everything with a lane executor is swept in
|
||||
# rotating nightly batches so the whole set is covered roughly weekly; a
|
||||
# single case or one module can be dispatched manually. That is the
|
||||
# implemented catalog (66 today) plus the eight planned cases whose dedicated
|
||||
# executors already exist and have live PASS evidence - fresh nightly
|
||||
# evidence feeds their acceptance review. heal.* cases stay on the
|
||||
# interactive validation flow: they require reviewed native/tool collectors
|
||||
# built by separate validation-tools runs before fleet execution.
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# After the nightly functional chain settles (chain starts ~16:48 UTC and
|
||||
# runs for about two hours).
|
||||
- cron: "30 19 * * *"
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
case:
|
||||
description: 'Single canonical validation case id (overrides module and rotation), e.g. pool.decommission'
|
||||
type: string
|
||||
required: false
|
||||
default: ''
|
||||
module:
|
||||
description: 'Restrict the sweep to one catalog module, e.g. bucket-replication'
|
||||
type: string
|
||||
required: false
|
||||
default: ''
|
||||
limit:
|
||||
description: 'Maximum cases per sweep (0 = run everything selected)'
|
||||
type: number
|
||||
required: false
|
||||
default: 12
|
||||
build_run_id:
|
||||
description: 'Successful nightly package run; empty resolves the latest'
|
||||
type: string
|
||||
required: false
|
||||
default: ''
|
||||
build_run_attempt:
|
||||
description: 'Attempt of that package run'
|
||||
type: string
|
||||
required: false
|
||||
default: '1'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: rustfs-validation-sweep
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES || 'vm000 vm001 vm002 vm003' }}
|
||||
RUSTFS_SSH_USER: ${{ secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER || 'azureuser' }}
|
||||
RUSTFS_ACCESS_KEY: ${{ secrets.RUSTFS_ACCESS_KEY }}
|
||||
RUSTFS_SECRET_KEY: ${{ secrets.RUSTFS_SECRET_KEY }}
|
||||
RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
|
||||
PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
|
||||
jobs:
|
||||
sweep:
|
||||
if: github.repository == 'rustfs/rustfs'
|
||||
runs-on: smoke-testing
|
||||
timeout-minutes: 150
|
||||
steps:
|
||||
- name: Checkout chain tooling
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Resolve nightly candidate
|
||||
id: candidate
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
BUILD_RUN_ID: ${{ inputs.build_run_id }}
|
||||
BUILD_RUN_ATTEMPT: ${{ inputs.build_run_attempt }}
|
||||
CHAIN_OUTPUT: ${{ runner.temp }}/validation-sweep-candidate.json
|
||||
run: |
|
||||
set -euo pipefail
|
||||
# Scheduled runs carry no inputs and the resolver only derives a
|
||||
# build from a workflow_run payload (which a schedule cannot
|
||||
# provide), so resolve the newest successful nightly build
|
||||
# explicitly before invoking it.
|
||||
if [ -z "${BUILD_RUN_ID:-}" ]; then
|
||||
LATEST="$(gh api "repos/rustfs/rustfs/actions/workflows/nightly-gnu.yml/runs?branch=main&status=success&per_page=1" --jq '.workflow_runs[0]')"
|
||||
BUILD_RUN_ID="$(printf '%s' "${LATEST}" | jq -r '.id')"
|
||||
BUILD_RUN_ATTEMPT="$(printf '%s' "${LATEST}" | jq -r '.run_attempt')"
|
||||
export BUILD_RUN_ID BUILD_RUN_ATTEMPT
|
||||
echo "scheduled sweep pinned to nightly ${BUILD_RUN_ID} attempt ${BUILD_RUN_ATTEMPT}"
|
||||
fi
|
||||
python3 scripts/resolve_functional_candidate.py
|
||||
python3 - <<'PY'
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
root = Path(os.environ["RUNNER_TEMP"]) / "validation-sweep"
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
chain = json.loads((Path(os.environ["RUNNER_TEMP"]) / "validation-sweep-candidate.json").read_text())
|
||||
manifest = chain["candidate"]["manifest"]
|
||||
(root / "candidate.json").write_text(json.dumps({
|
||||
"package_url": manifest["package_url"],
|
||||
"package_sha256": manifest["package_sha256"],
|
||||
"source_sha": manifest["source_sha"],
|
||||
}, sort_keys=True) + "\n")
|
||||
with open(os.environ["GITHUB_OUTPUT"], "a") as output:
|
||||
output.write("testing_sha=" + chain["testing_sha"] + "\n")
|
||||
output.write("package_url=" + manifest["package_url"] + "\n")
|
||||
output.write("source_sha=" + manifest["source_sha"] + "\n")
|
||||
PY
|
||||
|
||||
- name: Checkout auto-testing scripts
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
repository: rustfs/auto-testing
|
||||
ref: ${{ steps.candidate.outputs.testing_sha }}
|
||||
token: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
path: auto-testing
|
||||
persist-credentials: false
|
||||
|
||||
- name: Prepare validation Python runtime
|
||||
# The pool / ilm / replication validation runners require a managed
|
||||
# Python, exactly like the chain's validation job provides.
|
||||
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- name: Install validation Python requirements
|
||||
# Case scripts call python3 and import boto3 at startup. Install the
|
||||
# functional chain's client into a per-run env so that interpreter
|
||||
# sees it without changing the runner's shared Python.
|
||||
run: |
|
||||
set -euo pipefail
|
||||
validation_python="${RUNNER_TEMP}/rustfs-validation-python-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
|
||||
test ! -e "${validation_python}"
|
||||
python3 -m venv "${validation_python}"
|
||||
"${validation_python}/bin/python" -m pip install \
|
||||
-r auto-testing/scripts/validation_ilm/requirements.txt
|
||||
printf '%s\n' "${validation_python}/bin" >> "${GITHUB_PATH}"
|
||||
|
||||
- name: Select sweep cases
|
||||
id: select
|
||||
env:
|
||||
SELECTED_CASE: ${{ inputs.case }}
|
||||
SELECTED_MODULE: ${{ inputs.module }}
|
||||
CASE_LIMIT: ${{ inputs.limit }}
|
||||
SWEEP_EVENT: ${{ github.event_name }}
|
||||
SWEEP_ROOT: ${{ runner.temp }}/validation-sweep
|
||||
run: |
|
||||
set -euo pipefail
|
||||
python3 - <<'PY'
|
||||
import datetime
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
sweep_root = Path(os.environ["SWEEP_ROOT"])
|
||||
|
||||
catalog = json.loads(Path("auto-testing/validation-cases.json").read_text())
|
||||
def sweepable(case):
|
||||
status = case.get("source_status")
|
||||
if status == "implemented":
|
||||
return True
|
||||
# The remaining planned non-heal cases route to the lane
|
||||
# executors and already have live PASS evidence; sweeping them
|
||||
# nightly produces fresh evidence for their acceptance review.
|
||||
return status == "planned" and not case["id"].startswith("heal.")
|
||||
|
||||
selected = [(case["id"], module["id"]) for module in catalog["modules"]
|
||||
for case in module["cases"]
|
||||
if sweepable(case)]
|
||||
if not selected:
|
||||
raise SystemExit("catalog has no sweepable cases")
|
||||
|
||||
one_case = os.environ.get("SELECTED_CASE", "").strip()
|
||||
module_filter = os.environ.get("SELECTED_MODULE", "").strip()
|
||||
known = {case_id for case_id, _ in selected}
|
||||
modules = {module_id for _, module_id in selected}
|
||||
if one_case:
|
||||
if one_case not in known:
|
||||
raise SystemExit(f"unknown case or no lane executor for it: {one_case}")
|
||||
selected = [(one_case, dict(selected)[one_case])]
|
||||
elif module_filter:
|
||||
if module_filter not in modules:
|
||||
raise SystemExit(f"unknown module or no implemented cases: {module_filter}")
|
||||
selected = [pair for pair in selected if pair[1] == module_filter]
|
||||
elif os.environ.get("SWEEP_EVENT") == "schedule":
|
||||
# Stateless rotation: every night take one window of the catalog
|
||||
# so a full sweep needs len(selected)/limit nights and no state
|
||||
# has to be carried between runs.
|
||||
limit = max(1, int(os.environ.get("CASE_LIMIT") or 12))
|
||||
day = datetime.datetime.now(datetime.timezone.utc).timetuple().tm_yday
|
||||
start = (day * limit) % len(selected)
|
||||
selected = (selected[start:] + selected[:start])[:limit]
|
||||
limit = int(os.environ.get("CASE_LIMIT") or 0)
|
||||
if one_case or module_filter or (not os.environ.get("SWEEP_EVENT") == "schedule"):
|
||||
if limit > 0:
|
||||
selected = selected[:limit]
|
||||
|
||||
(sweep_root / "sweep-cases.txt").write_text("".join(f"{case_id}\n" for case_id, _ in selected))
|
||||
with open(os.environ["GITHUB_OUTPUT"], "a") as output:
|
||||
output.write(f"count={len(selected)}\n")
|
||||
print(f"selected {len(selected)} case(s):", ", ".join(c for c, _ in selected))
|
||||
PY
|
||||
|
||||
- name: Admit clean shared environment
|
||||
id: environment
|
||||
timeout-minutes: 5
|
||||
env:
|
||||
FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-validation-sweep.json
|
||||
run: python3 scripts/functional_environment.py enter --suite heal
|
||||
|
||||
- name: Run validation cases
|
||||
id: run_cases
|
||||
timeout-minutes: 120
|
||||
env:
|
||||
AUTO_TESTING_SHA: ${{ steps.candidate.outputs.testing_sha }}
|
||||
SWEEP_ROOT: ${{ runner.temp }}/validation-sweep
|
||||
run: |
|
||||
set -uo pipefail
|
||||
mkdir -p "${SWEEP_ROOT}/results"
|
||||
: > "${SWEEP_ROOT}/results.txt"
|
||||
while read -r CASE_ID; do
|
||||
[ -n "${CASE_ID}" ] || continue
|
||||
echo "::group::case ${CASE_ID}"
|
||||
rm -rf "${SWEEP_ROOT}/run"
|
||||
if ./auto-testing/scripts/run_validation_case.sh \
|
||||
"${CASE_ID}" "${SWEEP_ROOT}/candidate.json" \
|
||||
"${AUTO_TESTING_SHA}" "${SWEEP_ROOT}/run" \
|
||||
> "${SWEEP_ROOT}/results/${CASE_ID//\//_}.log" 2>&1; then
|
||||
printf '%s\tPASS\n' "${CASE_ID}" >> "${SWEEP_ROOT}/results.txt"
|
||||
echo "case ${CASE_ID}: PASS"
|
||||
else
|
||||
printf '%s\tFAIL\n' "${CASE_ID}" >> "${SWEEP_ROOT}/results.txt"
|
||||
echo "case ${CASE_ID}: FAIL (log retained in the sweep artifact)"
|
||||
fi
|
||||
if [ -f "${SWEEP_ROOT}/run/report.md" ]; then
|
||||
cp "${SWEEP_ROOT}/run/report.md" "${SWEEP_ROOT}/results/${CASE_ID//\//_}.report.md" 2>/dev/null || true
|
||||
fi
|
||||
echo "::endgroup::"
|
||||
# Fleet hygiene between cases: the case runners set up their own
|
||||
# topology, this purge guarantees the next case starts clean even
|
||||
# when the previous one died mid-run.
|
||||
for node in ${RUSTFS_NODES}; do
|
||||
# -n keeps ssh from consuming the case list this loop reads from stdin
|
||||
ssh -n -o BatchMode=yes -o ConnectTimeout=10 "${RUSTFS_SSH_USER}@${node}" '
|
||||
SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
|
||||
${SUDO} systemctl stop rustfs 2>/dev/null || true
|
||||
if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
|
||||
${SUDO} dpkg -P rustfs
|
||||
fi
|
||||
for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
|
||||
${SUDO} rm -f /etc/default/rustfs
|
||||
' || echo "inter-case cleanup failed on ${node} (continuing)"
|
||||
done
|
||||
done < "${SWEEP_ROOT}/sweep-cases.txt"
|
||||
echo "sweep finished: $(wc -l < "${SWEEP_ROOT}/results.txt") case(s) executed"
|
||||
|
||||
- name: Generate sweep report
|
||||
if: ${{ always() && steps.environment.outcome == 'success' }}
|
||||
env:
|
||||
SWEEP_ROOT: ${{ runner.temp }}/validation-sweep
|
||||
PACKAGE_URL: ${{ steps.candidate.outputs.package_url }}
|
||||
SOURCE_SHA: ${{ steps.candidate.outputs.source_sha }}
|
||||
AUTO_TESTING_SHA: ${{ steps.candidate.outputs.testing_sha }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
python3 - <<'PY'
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
root = Path(os.environ["SWEEP_ROOT"])
|
||||
catalog = json.loads(Path("auto-testing/validation-cases.json").read_text())
|
||||
module_of = {case["id"]: module["id"] for module in catalog["modules"] for case in module["cases"]}
|
||||
rows = []
|
||||
for line in (root / "results.txt").read_text().splitlines():
|
||||
if not line:
|
||||
continue
|
||||
case_id, status = line.split("\t")
|
||||
rows.append((case_id, module_of.get(case_id, "?"), status))
|
||||
passed = sum(1 for _, _, s in rows if s == "PASS")
|
||||
failed = len(rows) - passed
|
||||
run_url = (f"{os.environ.get('GITHUB_SERVER_URL', 'https://github.com')}/"
|
||||
f"{os.environ.get('GITHUB_REPOSITORY', 'rustfs/rustfs')}/actions/runs/"
|
||||
f"{os.environ.get('GITHUB_RUN_ID', '')}")
|
||||
|
||||
report = [
|
||||
"# RustFS validation sweep report", "",
|
||||
f"- Run: {run_url}",
|
||||
f"- Attempt: {os.environ.get('GITHUB_RUN_ATTEMPT', '')}",
|
||||
f"- Trigger: {os.environ.get('GITHUB_EVENT_NAME', '')}",
|
||||
f"- Package: {os.environ.get('PACKAGE_URL', 'nightly (R2 latest)')}",
|
||||
f"- Source SHA: {os.environ.get('SOURCE_SHA', '')}",
|
||||
f"- auto-testing: {os.environ.get('AUTO_TESTING_SHA', '')}", "",
|
||||
"Canonical validation cases (rustfs/auto-testing#113) executed by the sweep, grouped by catalog module.", "",
|
||||
]
|
||||
# Group the executed cases by catalog module (the major category:
|
||||
# pool lifecycle, heal, ILM expiration, bucket/site replication, kms,
|
||||
# data services, identity, s3 compatibility). Per-module subtotals
|
||||
# deliberately use "- Subtotal:" so the dashboard overview regexes
|
||||
# (which anchor on "- Total:") keep reading the grand totals below.
|
||||
by_module = {}
|
||||
module_order = []
|
||||
for case_id, module, status in rows:
|
||||
if module not in by_module:
|
||||
by_module[module] = []
|
||||
module_order.append(module)
|
||||
by_module[module].append((case_id, status))
|
||||
report += ["## Case Summary", ""]
|
||||
for module in module_order:
|
||||
cases = by_module[module]
|
||||
mod_pass = sum(1 for _, st in cases if st == "PASS")
|
||||
mod_fail = len(cases) - mod_pass
|
||||
report += [f"### {module} ({len(cases)})", "",
|
||||
"| Case | Status |", "| --- | --- |"]
|
||||
report += [f"| {case_id} | {status} |" for case_id, status in cases]
|
||||
report += ["", f"- Subtotal: {mod_pass} passed, {mod_fail} failed", ""]
|
||||
report += [
|
||||
"## Totals", "",
|
||||
f"- Total: {len(rows)}",
|
||||
f"- PASS: {passed}",
|
||||
f"- FAIL: {failed}",
|
||||
"- UNSUPPORTED: 0", "",
|
||||
"Failing cases keep this workflow red only as a report; the sweep artifact",
|
||||
"retains every case log and per-case report for diagnosis.", "",
|
||||
]
|
||||
(root / "report.md").write_text("\n".join(report))
|
||||
(root / "cases.md").write_text("\n".join(f"| {c} | {m} | {s} |" for c, m, s in rows) + "\n")
|
||||
PY
|
||||
cat "${SWEEP_ROOT}/report.md" >> "${GITHUB_STEP_SUMMARY}"
|
||||
|
||||
- name: Upload functional report to dashboard
|
||||
if: ${{ always() && steps.environment.outcome == 'success' }}
|
||||
timeout-minutes: 2
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
|
||||
REPORT_FILE: ${{ runner.temp }}/validation-sweep/report.md
|
||||
SUITE: validation
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; skipping dashboard upload"
|
||||
exit 0
|
||||
fi
|
||||
DATE="$(date -u +%Y-%m-%d)"
|
||||
REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
|
||||
B64_FILE="$(mktemp)"
|
||||
python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
|
||||
SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
|
||||
if [ -n "${SHA}" ]; then
|
||||
jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
|
||||
'{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
|
||||
else
|
||||
jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
|
||||
'{message:$msg, content:($content|rtrimstr("\n"))}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
|
||||
fi
|
||||
echo "dashboard updated: ${REPORT_PATH}"
|
||||
|
||||
- name: Retain sweep evidence
|
||||
if: ${{ always() }}
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: rustfs-validation-sweep-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: |
|
||||
${{ runner.temp }}/validation-sweep/report.md
|
||||
${{ runner.temp }}/validation-sweep/results.txt
|
||||
${{ runner.temp }}/validation-sweep/sweep-cases.txt
|
||||
${{ runner.temp }}/validation-sweep/candidate.json
|
||||
${{ runner.temp }}/validation-sweep/results/
|
||||
${{ runner.temp }}/validation-sweep/run/
|
||||
# warn (not error): an early infra failure legitimately leaves no
|
||||
# per-case evidence, and the failing step is already red.
|
||||
if-no-files-found: warn
|
||||
retention-days: 30
|
||||
|
||||
- name: Cleanup environment (after)
|
||||
id: environment_cleanup
|
||||
timeout-minutes: 5
|
||||
if: ${{ always() && steps.environment.outcome == 'success' }}
|
||||
run: |
|
||||
set -uo pipefail
|
||||
cleanup_status=0
|
||||
for node in ${RUSTFS_NODES}; do
|
||||
ssh -o BatchMode=yes -o ConnectTimeout=10 "${RUSTFS_SSH_USER}@${node}" '
|
||||
set -euo pipefail
|
||||
SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
|
||||
${SUDO} systemctl stop rustfs 2>/dev/null || true
|
||||
if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
|
||||
${SUDO} dpkg -P rustfs
|
||||
fi
|
||||
for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
|
||||
${SUDO} rm -f /etc/default/rustfs
|
||||
' || cleanup_status=$?
|
||||
done
|
||||
exit "${cleanup_status}"
|
||||
|
||||
- name: Release environment only after successful cleanup
|
||||
if: ${{ always() && steps.environment.outcome == 'success' }}
|
||||
timeout-minutes: 5
|
||||
env:
|
||||
FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-validation-sweep.json
|
||||
CLEANUP_OUTCOME: ${{ steps.environment_cleanup.outcome }}
|
||||
run: python3 scripts/functional_environment.py finish --suite heal --cleanup-outcome "$CLEANUP_OUTCOME"
|
||||
|
||||
- name: Notify on failure
|
||||
if: failure()
|
||||
run: echo "RustFS validation sweep failed (harness/environment breakdown); see the sweep artifact for case logs."
|
||||
|
||||
- name: Enforce sweep result
|
||||
if: ${{ always() && steps.environment.outcome == 'success' }}
|
||||
env:
|
||||
SWEEP_ROOT: ${{ runner.temp }}/validation-sweep
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if grep -q $'\tFAIL$' "${SWEEP_ROOT}/results.txt"; then
|
||||
echo "validation sweep has failing cases:"
|
||||
grep $'\tFAIL$' "${SWEEP_ROOT}/results.txt"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
alert-on-failure:
|
||||
name: Alert on scheduled failure
|
||||
# A scheduled sweep failure usually means a canonical validation case
|
||||
# regressed or the fleet environment broke; surface it as a tracking
|
||||
# issue instead of leaving it in the Actions tab.
|
||||
needs: [sweep]
|
||||
if: github.repository == 'rustfs/rustfs' && (always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure'))
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
contents: read
|
||||
issues: write
|
||||
steps:
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v5.0.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Open or update failure-tracking issue
|
||||
uses: ./.github/actions/schedule-failure-issue
|
||||
with:
|
||||
github-token: ${{ secrets.BACKLOG_ISSUE_TOKEN }}
|
||||
@@ -34,7 +34,7 @@ jobs:
|
||||
# self-hosted sm-standard-* pool is down, which is exactly when the
|
||||
# scheduled validations it watches stall.
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
@@ -47,7 +47,6 @@ jobs:
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
RUSTFS_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
|
||||
NIGHTLY_SOURCE_REF: ${{ vars.NIGHTLY_BRANCH || 'main' }}
|
||||
run: |
|
||||
set +e
|
||||
python3 scripts/check_scheduled_validation_freshness.py \
|
||||
|
||||
@@ -16,8 +16,19 @@ name: Scheduled Validation Watchdog
|
||||
|
||||
on:
|
||||
workflow_run:
|
||||
# The registry filters this broad subscription; names are not duplicated here.
|
||||
workflows: ["*"]
|
||||
workflows:
|
||||
- "Security Audit"
|
||||
- "Build and Release"
|
||||
- "Continuous Integration"
|
||||
- "coverage"
|
||||
- "e2e-nightly"
|
||||
- "e2e-distributed"
|
||||
- "e2e-s3tests"
|
||||
- "Fuzz"
|
||||
- "mint"
|
||||
- "minio-interop"
|
||||
- "Nightly GNU Build"
|
||||
- "Runner Hygiene"
|
||||
types: [completed]
|
||||
|
||||
permissions:
|
||||
@@ -26,7 +37,11 @@ permissions:
|
||||
jobs:
|
||||
alert-on-incomplete-run:
|
||||
name: Alert on incomplete scheduled run
|
||||
if: github.repository == 'rustfs/rustfs'
|
||||
if: >-
|
||||
github.repository == 'rustfs/rustfs' &&
|
||||
github.event.workflow_run.event == 'schedule' &&
|
||||
github.event.workflow_run.conclusion != 'success' &&
|
||||
github.event.workflow_run.conclusion != 'failure'
|
||||
# Stay on GitHub-hosted runners: this monitor must keep running when the
|
||||
# self-hosted sm-standard-* pool is down, which is exactly when the
|
||||
# scheduled validations it watches stall.
|
||||
@@ -40,11 +55,7 @@ jobs:
|
||||
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Select expected completion from registry
|
||||
id: registry
|
||||
run: python3 scripts/validation_registry.py watchdog
|
||||
- name: Open or update incomplete-run issue
|
||||
if: steps.registry.outputs.selected == 'true'
|
||||
uses: ./.github/actions/schedule-failure-issue
|
||||
with:
|
||||
github-token: ${{ secrets.BACKLOG_ISSUE_TOKEN }}
|
||||
|
||||
+1
-1
@@ -155,7 +155,7 @@ default build (lifecycle:
|
||||
- ⚠️ PARTIALLY VIOLATED: serving-side `s3s` references remain in ecstore
|
||||
(bucket metadata/replication/lifecycle DTOs and error mapping). The
|
||||
count is ratcheted shrink-only by `scripts/check_s3s_footprint.sh`
|
||||
(`S3S_ECSTORE_FILES_BASELINE`; the `object_lock` module was converted to
|
||||
(the `ecstore_files` line of `.config/s3s-footprint-baseline.txt`; the `object_lock` module was converted to
|
||||
storage-level types as the first ratchet step). Target state: the
|
||||
baseline reaches zero and ecstore's `Cargo.toml` drops `s3s`.
|
||||
|
||||
|
||||
@@ -26,7 +26,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
- IAM snapshot/deletion replay, target-assigned delete-marker purges, timestamp ordering, and best-effort peer broadcast now close the control-plane gaps found by the R6 review ([#7195](https://github.com/rustfs/rustfs/pull/7195)).
|
||||
- A `set-remote-target?update=true&creds=true` request on a site-replication peer's bucket target is now rejected with `400 InvalidRequest`, naming the target and why it counts as peer-owned. It used to return success while keeping the stored endpoint, credentials, and TLS settings, so an endpoint edit from the Console's bucket replication form appeared to apply but did not. Change a peer's address with a site-replication edit instead; see "Changing a site's address" in `docs/operations/site-replication-operations.md`. The `sync`, `bandwidth`, `proxy`, and `path` update groups are unchanged.
|
||||
- Site-replication edits that omit the site name, such as `mc admin replicate update --deployment-id <ID> --endpoint <URL>`, now keep the stored name on the editing site and on every receiver instead of renaming the site after the new endpoint host. No persisted format changes.
|
||||
- The persisted replication MRF journal is now replayed every 5 minutes while the server runs, not only at startup. Delete-marker purges whose targets were unreachable, and replications that overflowed the worker queues, now converge within one replay interval after the target returns instead of waiting for a restart. A target that stays down is re-attempted once per retained entry per interval across the cluster: the start time of the last pass is shared through a small `config/replication/mrf.bin.replay` object, so nodes do not each replay once per interval. Older versions ignore that object; existing persisted formats are unchanged.
|
||||
- Upgrade and rollback: upgrade every node in one site consecutively and verify reconciliation before moving to the next site; do not intentionally run a site mixed-version. Target-version ledger keys are harmless on rollback, although old code cannot use their routing. Before rolling back past [#7307](https://github.com/rustfs/rustfs/pull/7307), drain or repair every pending version purge: older code can free a retained version's data directory before its remote purge is acknowledged. See `docs/operations/site-replication-operations.md`.
|
||||
|
||||
### Security
|
||||
|
||||
Generated
+435
-443
File diff suppressed because it is too large
Load Diff
+39
-18
@@ -26,6 +26,7 @@ members = [
|
||||
"crates/ecstore", # Erasure coding storage implementation
|
||||
"crates/e2e_test", # End-to-end test suite
|
||||
"crates/filemeta", # File metadata management
|
||||
"crates/gateway-adapter", # RustFS implementations of the gateway extension traits
|
||||
"crates/heal", # Erasure set and object healing
|
||||
"crates/heal-contracts", # Heal request/response channel contracts
|
||||
"crates/iam", # Identity and Access Management
|
||||
@@ -48,7 +49,7 @@ members = [
|
||||
"crates/replication", # Replication contracts and wire formats
|
||||
"crates/concurrency", # Concurrency management for RustFS - timeout, locking, backpressure, and I/O scheduling
|
||||
"crates/s3-client", # S3 client for engine-side consumption of remote S3 endpoints (tiering, transition targets)
|
||||
"crates/s3-types", # S3 event type definitions
|
||||
"crates/s3-types", # S3 event and error type definitions
|
||||
"crates/s3-ops", # S3 operation definitions and mapping
|
||||
"crates/s3select-api", # S3 Select API interface
|
||||
"crates/s3select-query", # S3 Select query engine
|
||||
@@ -106,6 +107,7 @@ rustfs-credentials = { path = "crates/credentials", version = "1.0.2" }
|
||||
rustfs-crypto = { path = "crates/crypto", version = "1.0.2" }
|
||||
rustfs-ecstore = { path = "crates/ecstore", version = "1.0.2" }
|
||||
rustfs-filemeta = { path = "crates/filemeta", version = "1.0.2" }
|
||||
rustfs-gateway-adapter = { path = "crates/gateway-adapter", version = "1.0.2" }
|
||||
rustfs-iam = { path = "crates/iam", version = "1.0.2" }
|
||||
rustfs-keystone = { path = "crates/keystone", version = "1.0.2" }
|
||||
rustfs-license = { path = "crates/license", version = "1.0.2" }
|
||||
@@ -143,11 +145,22 @@ rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.2" }
|
||||
rustfs-utils = { path = "crates/utils", version = "1.0.2" }
|
||||
rustfs-zip = { path = "./crates/zip", version = "1.0.2" }
|
||||
|
||||
# The gateway's S3 types: the DTOs and persisted bucket-configuration codecs the
|
||||
# engine crates move onto from s3s (rustfs/backlog#2745). The one pin of the
|
||||
# gateway in this workspace; no crate may enable a `compat-s3s*` feature except
|
||||
# the s3s-facing edge of the rustfs binary.
|
||||
rustfs-gateway-types = { git = "https://github.com/rustfs/gateway.git", rev = "21283e874c2381bab611e9347acbc5b92e9e2b3c", version = "0.32.0", default-features = false }
|
||||
rustfs-gateway-stream = { git = "https://github.com/rustfs/gateway.git", rev = "21283e874c2381bab611e9347acbc5b92e9e2b3c", version = "0.2.0", default-features = false }
|
||||
# The gateway framework itself, consumed for its extension traits by the ring-2
|
||||
# adapter. `default-features = false` keeps the `server` feature, and with it the
|
||||
# server crate, out of the build.
|
||||
rustfs-gateway = { git = "https://github.com/rustfs/gateway.git", rev = "21283e874c2381bab611e9347acbc5b92e9e2b3c", version = "0.57.0", default-features = false }
|
||||
|
||||
# Async Runtime and Networking
|
||||
async-channel = "2.5.0"
|
||||
async_zip = { default-features = false, version = "0.0.19" }
|
||||
mysql_async = { default-features = false, version = "0.37.1" }
|
||||
async-compression = { version = "0.4.52" }
|
||||
async-compression = { version = "0.4.51" }
|
||||
async-trait = "0.1.92"
|
||||
async-nats = { version = "0.51.0", default-features = false }
|
||||
axum = "0.8.9"
|
||||
@@ -175,9 +188,9 @@ tokio-rustls = { default-features = false, version = "0.26.6" }
|
||||
tokio-stream = { version = "0.1.19" }
|
||||
tokio-test = "0.4.6"
|
||||
tokio-util = { version = "0.7.20" }
|
||||
tonic = { version = "0.14.9" }
|
||||
tonic-prost = { version = "0.14.9" }
|
||||
tonic-prost-build = { version = "0.14.9" }
|
||||
tonic = { version = "0.14.8" }
|
||||
tonic-prost = { version = "0.14.8" }
|
||||
tonic-prost-build = { version = "0.14.8" }
|
||||
tower = { version = "0.5.3" }
|
||||
tower-http = { version = "0.7.1" }
|
||||
h3 = "0.0.8"
|
||||
@@ -212,13 +225,12 @@ serde_urlencoded = "0.7.1"
|
||||
# releases.
|
||||
aes-gcm = { version = "0.11.1" }
|
||||
argon2 = { version = "0.6.0" }
|
||||
aws-lc-rs = { version = "1.18.0", default-features = false }
|
||||
blake2 = "0.11.0"
|
||||
chacha20poly1305 = { version = "0.11.0" }
|
||||
crc-fast = "1.10.0"
|
||||
hmac = { version = "0.13.0" }
|
||||
jsonwebtoken = { version = "11.1.0" }
|
||||
openidconnect = { default-features = false, version = "4.1.0" }
|
||||
openidconnect = { default-features = false, version = "4.0" }
|
||||
pbkdf2 = "0.13.0"
|
||||
p256 = { version = "0.14.0", features = ["ecdsa", "pkcs8"] }
|
||||
rsa = { version = "=0.10.0-rc.18" }
|
||||
@@ -235,7 +247,7 @@ proptest = "1"
|
||||
# Time and Date
|
||||
chrono = { version = "0.4.45" }
|
||||
humantime = "2.4.0"
|
||||
jiff = { version = "0.2.39" }
|
||||
jiff = { version = "0.2.38" }
|
||||
time = { version = "0.3.55" }
|
||||
|
||||
# Database
|
||||
@@ -254,9 +266,9 @@ atoi = "3.1.1"
|
||||
atomic_enum = "0.3.0"
|
||||
aws-config = { version = "1.13.0" }
|
||||
aws-credential-types = { version = "1.3.1" }
|
||||
aws-sdk-kms = { default-features = false, version = "1.129.0" }
|
||||
aws-sdk-s3 = { default-features = false, version = "1.157.0" }
|
||||
aws-sdk-sts = { default-features = false, version = "1.124.0" }
|
||||
aws-sdk-kms = { default-features = false, version = "1.128.0" }
|
||||
aws-sdk-s3 = { default-features = false, version = "1.156.0" }
|
||||
aws-sdk-sts = { default-features = false, version = "1.123.0" }
|
||||
aws-smithy-async = { version = "1.3.1" }
|
||||
aws-smithy-http-client = { default-features = false, version = "1.5.2" }
|
||||
aws-smithy-runtime-api = { version = "1.19.2" }
|
||||
@@ -320,13 +332,17 @@ rustfs-erasure-codec = { version = "9.0.0" }
|
||||
reed-solomon-simd = "3.1.0"
|
||||
regex = { version = "1.13.1" }
|
||||
rumqttc = { package = "rumqttc-next", version = "0.34.0" }
|
||||
redis = { version = "1.8.1" }
|
||||
redis = { version = "1.8.0" }
|
||||
rustify = { version = "0.7", default-features = false }
|
||||
rustix = { version = "1.1.6" }
|
||||
rust-embed = { version = "8.13.0" }
|
||||
rustc-hash = { version = "2.1.3" }
|
||||
s3s = { git = "https://github.com/s3s-project/s3s", rev = "5761ddfe4c509fcac87e3b99fd5bd9e176b442c0", features = ["minio"] }
|
||||
s3s-sigv4 = { git = "https://github.com/s3s-project/s3s", rev = "5761ddfe4c509fcac87e3b99fd5bd9e176b442c0" }
|
||||
# The same s3s pin under the name the legacy edge of the gateway migration uses
|
||||
# (crates/s3-types, feature `compat-s3s`). Keep `rev` identical to `s3s` above.
|
||||
# DELETE BY T4.2 (rustfs/backlog#2784).
|
||||
legacy-s3s = { package = "s3s", git = "https://github.com/s3s-project/s3s", rev = "5761ddfe4c509fcac87e3b99fd5bd9e176b442c0", features = ["minio"] }
|
||||
serial_test = "4.0.1"
|
||||
shadow-rs = { default-features = false, version = "2.0.0" }
|
||||
siphasher = "1.0.4"
|
||||
@@ -335,10 +351,11 @@ compact_str = "0.10.0"
|
||||
snap = "1.1.2"
|
||||
starshard = { version = "2.4.0" }
|
||||
strum = { version = "0.28.0" }
|
||||
sysinfo = "0.40.1"
|
||||
sync_wrapper = "1.0.2"
|
||||
sysinfo = "0.40.0"
|
||||
temp-env = "0.3.6"
|
||||
tempfile = "3.27.0"
|
||||
test-case = "3.4.1"
|
||||
test-case = "3.4.0"
|
||||
thiserror = "2.0.21"
|
||||
tracing = { version = "0.1.44" }
|
||||
tracing-appender = "0.2.5"
|
||||
@@ -357,13 +374,13 @@ winapi-util = "0.1.11"
|
||||
windows = { version = "0.62.2" }
|
||||
windows-sys = "0.61.2"
|
||||
xxhash-rust = { version = "0.8.19" }
|
||||
zip = "9.3.0"
|
||||
zip = "9.1.3"
|
||||
zstd = "0.14.3"
|
||||
|
||||
# Observability and Metrics
|
||||
metrics = "0.24.6"
|
||||
metrics-util = "0.20"
|
||||
dial9 = { version = "0.5.7", default-features = false }
|
||||
dial9 = { version = "0.5.6", default-features = false }
|
||||
opentelemetry = { version = "0.33.1" }
|
||||
opentelemetry-appender-tracing = { version = "0.33.1" }
|
||||
opentelemetry-otlp = { version = "0.33.1" }
|
||||
@@ -385,9 +402,9 @@ russh-sftp = "3.0.1"
|
||||
dav-server = "0.11.0"
|
||||
|
||||
# Performance Analysis and Memory Profiling
|
||||
rustfs-mimalloc = { version = "0.7.3" }
|
||||
rustfs-mimalloc = { version = "0.7.2" }
|
||||
# Preserve Unicode focus filters until rustfs/backlog#2302 is resolved.
|
||||
hotpath = { version = "0.30.3", default-features = false }
|
||||
hotpath = { version = "0.30.1", default-features = false }
|
||||
# Snapshot testing for output format regression detection
|
||||
insta = { version = "1.49" }
|
||||
|
||||
@@ -428,3 +445,7 @@ strip = "none"
|
||||
# Pin upstream fixes until compatible crates.io releases are available.
|
||||
[patch.crates-io]
|
||||
zstd = { path = "crates/zstd-compat" }
|
||||
# Preserve RustFS OIDC flows while migrating the old elliptic-curve dependency chain.
|
||||
# Upstream: openidconnect-rs#244 and oauth2-rs#352.
|
||||
openidconnect = { git = "https://github.com/ramosbugs/openidconnect-rs", rev = "2f77ee73bc4554e1cb6279adb386005429fb4e41" }
|
||||
oauth2 = { git = "https://github.com/ramosbugs/oauth2-rs", rev = "89585f4a280afd7b285eaf1c9be0068874e03d22" }
|
||||
|
||||
@@ -10,10 +10,6 @@ use std::time::Duration;
|
||||
use tokio::sync::{Notify, Semaphore};
|
||||
use tokio::time::Instant;
|
||||
|
||||
pub mod permit_set;
|
||||
|
||||
const MAX_PERMIT_REFERENCES: usize = usize::MAX / 2;
|
||||
|
||||
/// A resource owner publishes a capacity; running units retain their permits
|
||||
/// after a reduction. Lease slots are reserved once, with no per-unit Arc
|
||||
/// allocation. Tokio wait queues may lazily initialize platform mutex storage
|
||||
@@ -40,7 +36,6 @@ pub struct AdaptiveAdmission {
|
||||
maximum: u32,
|
||||
slots: Box<[PermitSlots]>,
|
||||
next_chunk: AtomicUsize,
|
||||
completed: AtomicU64,
|
||||
}
|
||||
|
||||
struct PermitSlots {
|
||||
@@ -66,12 +61,10 @@ pub struct AdaptivePermit {
|
||||
|
||||
impl Clone for AdaptivePermit {
|
||||
fn clone(&self) -> Self {
|
||||
// Subset carriers can retain slot references through a shared owner,
|
||||
// so the slot count needs its own bound in addition to Arc's guard.
|
||||
// Every slot reference owns a strong owner reference. Clone the Arc
|
||||
// first, so its overflow protection also bounds the slot refcount.
|
||||
let owner = Arc::clone(&self.owner);
|
||||
if owner.slots[self.slot / 64].references[self.slot % 64].fetch_add(1, Ordering::Relaxed) >= MAX_PERMIT_REFERENCES {
|
||||
std::process::abort();
|
||||
}
|
||||
owner.slots[self.slot / 64].references[self.slot % 64].fetch_add(1, Ordering::Relaxed);
|
||||
Self { owner, slot: self.slot }
|
||||
}
|
||||
}
|
||||
@@ -121,22 +114,11 @@ impl NodeAdmission {
|
||||
|
||||
/// Publish feedback with this owner's actual completed-work counter.
|
||||
pub fn update(&self, controller: &mut FeedbackController, observation: Option<FeedbackObservation>) -> FeedbackDecision {
|
||||
self.update_with_probe(controller, observation, true)
|
||||
}
|
||||
|
||||
/// Coordinate growth with other resource controllers in the same window.
|
||||
/// Protective reductions never depend on permission to probe.
|
||||
pub fn update_with_probe(
|
||||
&self,
|
||||
controller: &mut FeedbackController,
|
||||
observation: Option<FeedbackObservation>,
|
||||
allow_probe: bool,
|
||||
) -> FeedbackDecision {
|
||||
let observation = observation.map(|mut observation| {
|
||||
observation.completed_units = self.completed();
|
||||
observation
|
||||
});
|
||||
controller.update_with_probe(&self.capacity, observation, allow_probe)
|
||||
controller.update(&self.capacity, observation)
|
||||
}
|
||||
|
||||
/// Current leased scheduling cap and held async units.
|
||||
@@ -267,31 +249,16 @@ impl FeedbackController {
|
||||
/// Consume one sample and publish a three-second lease to the same gate.
|
||||
/// If this function stops being called, admission expires independently.
|
||||
pub fn update(&mut self, gate: &AdaptiveAdmission, observation: Option<FeedbackObservation>) -> FeedbackDecision {
|
||||
self.update_with_probe(gate, observation, true)
|
||||
}
|
||||
|
||||
/// Allow at most the sampling owner's selected controller to probe upward.
|
||||
pub fn update_with_probe(
|
||||
&mut self,
|
||||
gate: &AdaptiveAdmission,
|
||||
observation: Option<FeedbackObservation>,
|
||||
allow_probe: bool,
|
||||
) -> FeedbackDecision {
|
||||
let observation = observation.map(|mut observation| {
|
||||
observation.max_concurrency = observation.max_concurrency.min(gate.maximum);
|
||||
observation
|
||||
});
|
||||
let decision = self.decide_with_probe(Instant::now(), observation, allow_probe);
|
||||
let decision = self.decide(Instant::now(), observation);
|
||||
gate.set_sampled_limit(decision.limit, Duration::from_secs(3));
|
||||
decision
|
||||
}
|
||||
|
||||
fn decide_with_probe(
|
||||
&mut self,
|
||||
now: Instant,
|
||||
observation: Option<FeedbackObservation>,
|
||||
allow_probe: bool,
|
||||
) -> FeedbackDecision {
|
||||
fn decide(&mut self, now: Instant, observation: Option<FeedbackObservation>) -> FeedbackDecision {
|
||||
let Some(observation) = observation else {
|
||||
self.previous = None;
|
||||
self.stable_since = None;
|
||||
@@ -336,8 +303,7 @@ impl FeedbackController {
|
||||
FeedbackReason::NewBaseline
|
||||
} else {
|
||||
let since = *self.stable_since.get_or_insert(now);
|
||||
if allow_probe
|
||||
&& advanced
|
||||
if advanced
|
||||
&& now.saturating_duration_since(since) >= Duration::from_secs(8)
|
||||
&& now.saturating_duration_since(self.last_change) >= Duration::from_secs(8)
|
||||
&& self.limit < observation.max_concurrency
|
||||
@@ -359,13 +325,6 @@ impl FeedbackController {
|
||||
}
|
||||
|
||||
impl AdaptiveAdmission {
|
||||
/// Successful async operations using this physical owner. Native clones
|
||||
/// retain capacity but never count another completion.
|
||||
pub fn completed(&self) -> Option<u64> {
|
||||
let value = self.completed.load(Ordering::Acquire);
|
||||
(value != u64::MAX).then_some(value)
|
||||
}
|
||||
|
||||
/// Construct a bounded owner, with zero initial capacity allowed for unknown
|
||||
/// input. The immutable maximum reserves at most 65,535 lease slots once;
|
||||
/// acquiring or cloning a permit does not allocate a per-unit Arc.
|
||||
@@ -392,7 +351,6 @@ impl AdaptiveAdmission {
|
||||
maximum: maximum_u32,
|
||||
slots,
|
||||
next_chunk: AtomicUsize::new(0),
|
||||
completed: AtomicU64::new(0),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -508,21 +466,6 @@ impl AdaptiveAdmission {
|
||||
self.slots[slot / 64].free.fetch_or(1_u64 << (slot % 64), Ordering::Release);
|
||||
}
|
||||
|
||||
fn release_reference(&self, slot: usize) {
|
||||
self.release_reference_with(slot, || {});
|
||||
}
|
||||
|
||||
fn release_reference_with(&self, slot: usize, before_release: impl FnOnce()) {
|
||||
if self.slots[slot / 64].references[slot % 64].fetch_sub(1, Ordering::AcqRel) != 1 {
|
||||
return;
|
||||
}
|
||||
before_release();
|
||||
// Publish the free slot before exposing another admission capacity.
|
||||
self.release_slot(slot);
|
||||
self.state.fetch_sub(1, Ordering::AcqRel);
|
||||
self.changed.notify_one();
|
||||
}
|
||||
|
||||
/// Wait for capacity. The future owns no permit until it returns; dropping
|
||||
/// a pending future leaves capacity untouched. Wrap in the caller's existing
|
||||
/// cancellation/deadline select before acquiring any storage commit locks.
|
||||
@@ -550,7 +493,13 @@ impl AdaptiveAdmission {
|
||||
|
||||
impl Drop for AdaptivePermit {
|
||||
fn drop(&mut self) {
|
||||
self.owner.release_reference(self.slot);
|
||||
if self.owner.slots[self.slot / 64].references[self.slot % 64].fetch_sub(1, Ordering::AcqRel) != 1 {
|
||||
return;
|
||||
}
|
||||
// Publish the free slot before exposing another admission capacity.
|
||||
self.owner.release_slot(self.slot);
|
||||
self.owner.state.fetch_sub(1, Ordering::AcqRel);
|
||||
self.owner.changed.notify_one();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -912,24 +861,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn shared_probe_permission_blocks_growth_but_never_protective_cuts() {
|
||||
let gate = AdaptiveAdmission::new(NonZeroU16::new(8).expect("bound"), 0);
|
||||
let mut controller = FeedbackController::new();
|
||||
controller.update_with_probe(&gate, Some(healthy(0)), false);
|
||||
for completed in 1..=20 {
|
||||
tokio::time::advance(Duration::from_secs(1)).await;
|
||||
assert_eq!(controller.update_with_probe(&gate, Some(healthy(completed)), false).limit, 1);
|
||||
}
|
||||
tokio::time::advance(Duration::from_secs(1)).await;
|
||||
assert_eq!(controller.update_with_probe(&gate, Some(healthy(21)), true).reason, FeedbackReason::Probe);
|
||||
assert_eq!(gate.snapshot().limit, 2);
|
||||
let mut busy = healthy(22);
|
||||
busy.pressure = ResourcePressure::Busy;
|
||||
assert_eq!(controller.update_with_probe(&gate, Some(busy), false).limit, 1);
|
||||
assert_eq!(controller.update_with_probe(&gate, None, false).limit, 0);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn controller_grows_one_unit_per_cooldown_and_cuts_pressure_immediately() {
|
||||
let gate = AdaptiveAdmission::new(NonZeroU16::new(8).expect("bounded test owner"), 0);
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -30,16 +30,6 @@ pub const COMMENT_KEY: &str = "comment";
|
||||
pub const ENV_AUDIT_ENABLE: &str = "RUSTFS_AUDIT_ENABLE";
|
||||
/// Global switch for enabling the notify module.
|
||||
pub const ENV_NOTIFY_ENABLE: &str = "RUSTFS_NOTIFY_ENABLE";
|
||||
/// Maximum audit entries per second for authorization denials of authenticated callers.
|
||||
/// `0` disables denial auditing for that class of caller.
|
||||
pub const ENV_AUDIT_DENIAL_RATE: &str = "RUSTFS_AUDIT_DENIAL_RATE";
|
||||
/// Maximum audit entries per second for authorization denials of anonymous callers.
|
||||
/// `0` disables denial auditing for that class of caller.
|
||||
pub const ENV_AUDIT_DENIAL_ANONYMOUS_RATE: &str = "RUSTFS_AUDIT_DENIAL_ANONYMOUS_RATE";
|
||||
/// Default denial audit rate for authenticated callers (entries per second).
|
||||
pub const DEFAULT_AUDIT_DENIAL_RATE: u64 = 50;
|
||||
/// Default denial audit rate for anonymous callers (entries per second).
|
||||
pub const DEFAULT_AUDIT_DENIAL_ANONYMOUS_RATE: u64 = 5;
|
||||
/// Default global audit switch (disabled by default).
|
||||
pub const DEFAULT_AUDIT_ENABLE: bool = false;
|
||||
/// Default global notify switch (disabled by default).
|
||||
@@ -338,7 +328,5 @@ mod tests {
|
||||
fn test_global_audit_notify_switch_constants() {
|
||||
assert_eq!(ENV_AUDIT_ENABLE, "RUSTFS_AUDIT_ENABLE");
|
||||
assert_eq!(ENV_NOTIFY_ENABLE, "RUSTFS_NOTIFY_ENABLE");
|
||||
assert_eq!(ENV_AUDIT_DENIAL_RATE, "RUSTFS_AUDIT_DENIAL_RATE");
|
||||
assert_eq!(ENV_AUDIT_DENIAL_ANONYMOUS_RATE, "RUSTFS_AUDIT_DENIAL_ANONYMOUS_RATE");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -24,14 +24,6 @@ use std::{
|
||||
time::{Duration, SystemTime},
|
||||
};
|
||||
|
||||
/// Copy an existing cache-model value without including surrounding work.
|
||||
#[inline]
|
||||
pub fn clone_cache_value<T: ToOwned + ?Sized>(source: &T) -> T::Owned {
|
||||
#[cfg(feature = "hotpath-alloc")]
|
||||
let _copy_scope = hotpath::MeasurementGuardSync::new("cache_model.owned_copy", false, false);
|
||||
source.to_owned()
|
||||
}
|
||||
|
||||
/// Maximum amount a persisted `last_update` may lead the local wall clock before the
|
||||
/// persisted timestamp is treated as untrustworthy.
|
||||
///
|
||||
@@ -350,7 +342,7 @@ impl UnknownTierStats {
|
||||
self.diagnostics_dropped = checked_saturating_add(1, self.diagnostics_dropped, &mut self.counter_overflowed);
|
||||
continue;
|
||||
}
|
||||
self.diagnostics.push(clone_cache_value(diagnostic));
|
||||
self.diagnostics.push(diagnostic.clone());
|
||||
}
|
||||
}
|
||||
if !self.diagnostics.is_empty() {
|
||||
@@ -375,7 +367,7 @@ impl UnknownTierStats {
|
||||
if !self.fits_add(other) {
|
||||
return None;
|
||||
}
|
||||
let mut merged = clone_cache_value(self);
|
||||
let mut merged = self.clone();
|
||||
merged.merge(other);
|
||||
Some(merged)
|
||||
}
|
||||
@@ -415,7 +407,7 @@ impl AllTierStats {
|
||||
if st.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let entry = self.tiers.entry(clone_cache_value(tier)).or_default();
|
||||
let entry = self.tiers.entry(tier.clone()).or_default();
|
||||
*entry = entry.add(st);
|
||||
}
|
||||
}
|
||||
@@ -445,7 +437,7 @@ impl AllTierStats {
|
||||
.tiers
|
||||
.keys()
|
||||
.filter(|tier| tier.as_str() != "STANDARD" && tier.as_str() != "REDUCED_REDUNDANCY" && !known.contains(tier.as_str()))
|
||||
.map(clone_cache_value)
|
||||
.cloned()
|
||||
.collect();
|
||||
for tier in retired_tiers {
|
||||
if let Some(stats) = self.tiers.remove(&tier) {
|
||||
@@ -1245,10 +1237,10 @@ impl DataUsageEntry {
|
||||
}
|
||||
|
||||
pub fn add_child(&mut self, hash: &DataUsageHash) {
|
||||
if self.children.contains(&clone_cache_value(&hash.0)) {
|
||||
if self.children.contains(&hash.key()) {
|
||||
return;
|
||||
}
|
||||
self.children.insert(clone_cache_value(&hash.0));
|
||||
self.children.insert(hash.key());
|
||||
}
|
||||
|
||||
pub fn merge(&mut self, other: &DataUsageEntry) {
|
||||
@@ -1265,7 +1257,7 @@ impl DataUsageEntry {
|
||||
s_rep.replica_size += o_rep.replica_size;
|
||||
s_rep.replica_count += o_rep.replica_count;
|
||||
for (arn, stat) in o_rep.targets.iter() {
|
||||
let st = s_rep.targets.entry(clone_cache_value(arn)).or_default();
|
||||
let st = s_rep.targets.entry(arn.clone()).or_default();
|
||||
st.pending_size += stat.pending_size;
|
||||
st.replicated_size += stat.replicated_size;
|
||||
st.failed_size += stat.failed_size;
|
||||
@@ -1459,7 +1451,7 @@ pub fn prefix_usage_in_cache(
|
||||
} else {
|
||||
format!("{bucket}/{prefix}")
|
||||
};
|
||||
let entry = clone_cache_value(cache.get(&clone_cache_value(&hash_path(&root).0))?);
|
||||
let entry = cache.get(&hash_path(&root).key())?.clone();
|
||||
|
||||
let usage = PrefixUsageSummary::from_entry(&flatten_entry(cache, &entry, 0)?);
|
||||
|
||||
@@ -1602,38 +1594,38 @@ pub struct DataUsageCache {
|
||||
impl DataUsageCache {
|
||||
pub fn replace(&mut self, path: &str, parent: &str, e: DataUsageEntry) {
|
||||
let hash = hash_path(path);
|
||||
self.cache.insert(clone_cache_value(&hash.0), e);
|
||||
self.cache.insert(hash.key(), e);
|
||||
if !parent.is_empty() {
|
||||
let phash = hash_path(parent);
|
||||
let p = {
|
||||
let p = self.cache.entry(clone_cache_value(&phash.0)).or_default();
|
||||
let p = self.cache.entry(phash.key()).or_default();
|
||||
p.add_child(&hash);
|
||||
clone_cache_value(p)
|
||||
p.clone()
|
||||
};
|
||||
self.cache.insert(clone_cache_value(&phash.0), p);
|
||||
self.cache.insert(phash.key(), p);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn replace_hashed(&mut self, hash: &DataUsageHash, parent: &Option<DataUsageHash>, e: &DataUsageEntry) {
|
||||
self.cache.insert(clone_cache_value(&hash.0), clone_cache_value(e));
|
||||
self.cache.insert(hash.key(), e.clone());
|
||||
if let Some(parent) = parent {
|
||||
self.cache.entry(clone_cache_value(&parent.0)).or_default().add_child(hash);
|
||||
self.cache.entry(parent.key()).or_default().add_child(hash);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn find(&self, path: &str) -> Option<DataUsageEntry> {
|
||||
self.cache.get(&clone_cache_value(&hash_path(path).0)).map(clone_cache_value)
|
||||
self.cache.get(&hash_path(path).key()).cloned()
|
||||
}
|
||||
|
||||
pub fn find_children_copy(&mut self, h: DataUsageHash) -> DataUsageHashMap {
|
||||
clone_cache_value(&self.cache.entry(clone_cache_value(&h.0)).or_default().children)
|
||||
self.cache.entry(h.string()).or_default().children.clone()
|
||||
}
|
||||
|
||||
pub fn flatten(&self, root: &DataUsageEntry) -> DataUsageEntry {
|
||||
let mut root = clone_cache_value(root);
|
||||
for id in clone_cache_value(&root.children).iter() {
|
||||
let mut root = root.clone();
|
||||
for id in root.children.clone().iter() {
|
||||
if let Some(e) = self.cache.get(id) {
|
||||
let mut e = clone_cache_value(e);
|
||||
let mut e = e.clone();
|
||||
if !e.children.is_empty() {
|
||||
e = self.flatten(&e);
|
||||
}
|
||||
@@ -1645,16 +1637,16 @@ impl DataUsageCache {
|
||||
}
|
||||
|
||||
pub fn copy_with_children(&mut self, src: &DataUsageCache, hash: &DataUsageHash, parent: &Option<DataUsageHash>) {
|
||||
if let Some(e) = src.cache.get(&clone_cache_value(&hash.0)) {
|
||||
self.cache.insert(clone_cache_value(&hash.0), clone_cache_value(e));
|
||||
if let Some(e) = src.cache.get(&hash.string()) {
|
||||
self.cache.insert(hash.key(), e.clone());
|
||||
for ch in e.children.iter() {
|
||||
if *ch == clone_cache_value(&hash.0) {
|
||||
if *ch == hash.key() {
|
||||
return;
|
||||
}
|
||||
self.copy_with_children(src, &DataUsageHash(clone_cache_value(ch)), &Some(clone_cache_value(hash)));
|
||||
self.copy_with_children(src, &DataUsageHash(ch.to_string()), &Some(hash.clone()));
|
||||
}
|
||||
if let Some(parent) = parent {
|
||||
let p = self.cache.entry(clone_cache_value(&parent.0)).or_default();
|
||||
let p = self.cache.entry(parent.key()).or_default();
|
||||
p.add_child(hash);
|
||||
}
|
||||
}
|
||||
@@ -1662,14 +1654,14 @@ impl DataUsageCache {
|
||||
|
||||
pub fn delete_recursive(&mut self, hash: &DataUsageHash) {
|
||||
let mut need_remove = Vec::new();
|
||||
if let Some(v) = self.cache.get(&clone_cache_value(&hash.0)) {
|
||||
if let Some(v) = self.cache.get(&hash.string()) {
|
||||
for child in v.children.iter() {
|
||||
need_remove.push(clone_cache_value(child));
|
||||
need_remove.push(child.clone());
|
||||
}
|
||||
}
|
||||
self.cache.remove(&clone_cache_value(&hash.0));
|
||||
self.cache.remove(&hash.string());
|
||||
need_remove.iter().for_each(|child| {
|
||||
self.delete_recursive(&DataUsageHash(clone_cache_value(child)));
|
||||
self.delete_recursive(&DataUsageHash(child.to_string()));
|
||||
});
|
||||
}
|
||||
|
||||
@@ -1690,7 +1682,7 @@ impl DataUsageCache {
|
||||
}
|
||||
|
||||
pub fn search_parent(&self, hash: &DataUsageHash) -> Option<DataUsageHash> {
|
||||
let want = clone_cache_value(&hash.0);
|
||||
let want = hash.key();
|
||||
if let Some(last_index) = want.rfind('/')
|
||||
&& let Some(v) = self.find(&want[0..last_index])
|
||||
&& v.children.contains(&want)
|
||||
@@ -1701,7 +1693,7 @@ impl DataUsageCache {
|
||||
|
||||
for (k, v) in self.cache.iter() {
|
||||
if v.children.contains(&want) {
|
||||
let found = DataUsageHash(clone_cache_value(k));
|
||||
let found = DataUsageHash(k.clone());
|
||||
return Some(found);
|
||||
}
|
||||
}
|
||||
@@ -1709,7 +1701,7 @@ impl DataUsageCache {
|
||||
}
|
||||
|
||||
pub fn is_compacted(&self, hash: &DataUsageHash) -> bool {
|
||||
match self.cache.get(&clone_cache_value(&hash.0)) {
|
||||
match self.cache.get(&hash.key()) {
|
||||
Some(due) => due.compacted,
|
||||
None => false,
|
||||
}
|
||||
@@ -1734,7 +1726,7 @@ impl DataUsageCache {
|
||||
if self.cache.len() < limit {
|
||||
return;
|
||||
}
|
||||
let top = clone_cache_value(&hash_path(&self.info.name).0);
|
||||
let top = hash_path(&self.info.name).key();
|
||||
let top_e = match self.find(&top) {
|
||||
Some(e) => e,
|
||||
None => return,
|
||||
@@ -1761,7 +1753,7 @@ impl DataUsageCache {
|
||||
}
|
||||
|
||||
pub fn reduce_children_of(&mut self, path: &DataUsageHash, limit: usize, compact_self: bool) {
|
||||
let e = match self.cache.get(&clone_cache_value(&path.0)) {
|
||||
let e = match self.cache.get(&path.key()) {
|
||||
Some(e) => e,
|
||||
None => return,
|
||||
};
|
||||
@@ -1771,13 +1763,13 @@ impl DataUsageCache {
|
||||
}
|
||||
|
||||
if e.children.len() > limit && compact_self {
|
||||
let mut flat = self.size_recursive(&clone_cache_value(&path.0)).unwrap_or_default();
|
||||
let mut flat = self.size_recursive(&path.key()).unwrap_or_default();
|
||||
flat.compacted = true;
|
||||
self.delete_recursive(path);
|
||||
self.replace_hashed(path, &None, &flat);
|
||||
return;
|
||||
}
|
||||
let total = self.total_children_rec(&clone_cache_value(&path.0));
|
||||
let total = self.total_children_rec(&path.key());
|
||||
if total < limit {
|
||||
return;
|
||||
}
|
||||
@@ -1790,12 +1782,12 @@ impl DataUsageCache {
|
||||
let mut candidate_index = 0;
|
||||
while remove > 0 && candidate_index < candidates.len() {
|
||||
let e = &candidates[candidate_index];
|
||||
let candidate = clone_cache_value(&e.path);
|
||||
let candidate = e.path.clone();
|
||||
if candidate == *path && !compact_self {
|
||||
break;
|
||||
}
|
||||
let removing = self.total_children_rec(&clone_cache_value(&candidate.0));
|
||||
let mut flat = match self.size_recursive(&clone_cache_value(&candidate.0)) {
|
||||
let removing = self.total_children_rec(&candidate.key());
|
||||
let mut flat = match self.size_recursive(&candidate.key()) {
|
||||
Some(flat) => flat,
|
||||
None => {
|
||||
candidate_index += 1;
|
||||
@@ -1832,7 +1824,7 @@ impl DataUsageCache {
|
||||
if o.root().is_none() {
|
||||
return;
|
||||
}
|
||||
*self = clone_cache_value(o);
|
||||
*self = o.clone();
|
||||
return;
|
||||
};
|
||||
|
||||
@@ -1845,8 +1837,7 @@ impl DataUsageCache {
|
||||
}
|
||||
|
||||
existing_root.merge(&other_root);
|
||||
self.cache
|
||||
.insert(clone_cache_value(&hash_path(&self.info.name).0), existing_root);
|
||||
self.cache.insert(hash_path(&self.info.name).key(), existing_root);
|
||||
|
||||
let root_hash = self.root_hash();
|
||||
for key in other_root.children.iter() {
|
||||
@@ -1857,7 +1848,7 @@ impl DataUsageCache {
|
||||
if let Some(existing) = self.cache.get_mut(key) {
|
||||
existing.merge(&flat);
|
||||
} else {
|
||||
self.replace_hashed(&DataUsageHash(clone_cache_value(key)), &Some(clone_cache_value(&root_hash)), &flat);
|
||||
self.replace_hashed(&DataUsageHash(key.clone()), &Some(root_hash.clone()), &flat);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1901,7 +1892,7 @@ impl DataUsageCache {
|
||||
|
||||
for (arn, stat) in rs.targets.iter() {
|
||||
bui.replication_info.insert(
|
||||
clone_cache_value(arn),
|
||||
arn.clone(),
|
||||
BucketTargetUsageInfo {
|
||||
replication_pending_size: stat.pending_size,
|
||||
replicated_size: stat.replicated_size,
|
||||
@@ -1914,7 +1905,7 @@ impl DataUsageCache {
|
||||
);
|
||||
}
|
||||
}
|
||||
buckets_usage.insert(clone_cache_value(bucket_name), bui);
|
||||
buckets_usage.insert(bucket_name.clone(), bui);
|
||||
}
|
||||
|
||||
DataUsageInfo {
|
||||
@@ -1946,20 +1937,20 @@ struct Inner {
|
||||
}
|
||||
|
||||
fn add(data_usage_cache: &DataUsageCache, path: &DataUsageHash, candidates: &mut Vec<Inner>) -> usize {
|
||||
let e = match data_usage_cache.cache.get(&clone_cache_value(&path.0)) {
|
||||
let e = match data_usage_cache.cache.get(&path.key()) {
|
||||
Some(e) => e,
|
||||
None => return 0,
|
||||
};
|
||||
let mut objects = e.objects;
|
||||
for ch in e.children.iter() {
|
||||
objects += add(data_usage_cache, &DataUsageHash(clone_cache_value(ch)), candidates);
|
||||
objects += add(data_usage_cache, &DataUsageHash(ch.clone()), candidates);
|
||||
}
|
||||
// Collect internal nodes (with children) as compaction candidates.
|
||||
// Leaf nodes have no children to remove, so compacting them is a no-op.
|
||||
if !e.children.is_empty() {
|
||||
candidates.push(Inner {
|
||||
objects,
|
||||
path: clone_cache_value(path),
|
||||
path: path.clone(),
|
||||
});
|
||||
}
|
||||
objects
|
||||
@@ -1967,7 +1958,7 @@ fn add(data_usage_cache: &DataUsageCache, path: &DataUsageHash, candidates: &mut
|
||||
|
||||
fn mark(duc: &DataUsageCache, entry: &DataUsageEntry, found: &mut HashSet<String>) {
|
||||
for k in entry.children.iter() {
|
||||
found.insert(clone_cache_value(k));
|
||||
found.insert(k.to_string());
|
||||
if let Some(ch) = duc.cache.get(k) {
|
||||
mark(duc, ch, found);
|
||||
}
|
||||
@@ -2417,11 +2408,11 @@ impl SizeSummary {
|
||||
}
|
||||
|
||||
for entry in &other.size_reconciliation {
|
||||
self.record_size_reconciliation(clone_cache_value(entry));
|
||||
self.record_size_reconciliation(entry.clone());
|
||||
}
|
||||
self.size_reconciliation_truncated |= other.size_reconciliation_truncated;
|
||||
for scope in &other.reconciliation_scopes {
|
||||
self.record_reconciliation_scope_with(&scope.bucket, &scope.object, || clone_cache_value(scope));
|
||||
self.record_reconciliation_scope(&scope.bucket, &scope.object);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2446,13 +2437,6 @@ impl SizeSummary {
|
||||
/// Mark one object scope as refreshed. Duplicate scopes are suppressed so
|
||||
/// merging summaries remains bounded and deterministic.
|
||||
pub fn record_reconciliation_scope(&mut self, bucket: &str, object: &str) {
|
||||
self.record_reconciliation_scope_with(bucket, object, || SizeReconciliationScope {
|
||||
bucket: bucket.to_string(),
|
||||
object: object.to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
fn record_reconciliation_scope_with(&mut self, bucket: &str, object: &str, build: impl FnOnce() -> SizeReconciliationScope) {
|
||||
if !self
|
||||
.reconciliation_scopes
|
||||
.iter()
|
||||
@@ -2462,7 +2446,10 @@ impl SizeSummary {
|
||||
self.size_reconciliation_truncated = true;
|
||||
return;
|
||||
}
|
||||
self.reconciliation_scopes.push(build());
|
||||
self.reconciliation_scopes.push(SizeReconciliationScope {
|
||||
bucket: bucket.to_string(),
|
||||
object: object.to_string(),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -3417,42 +3404,6 @@ mod tests {
|
||||
assert_eq!(summary1.versions, 15);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_reconciliation_scope_builds_only_after_admission() {
|
||||
let mut summary = SizeSummary::new();
|
||||
let builds = std::cell::Cell::new(0);
|
||||
let build = || {
|
||||
builds.set(builds.get() + 1);
|
||||
SizeReconciliationScope {
|
||||
bucket: "bucket".to_string(),
|
||||
object: "object".to_string(),
|
||||
}
|
||||
};
|
||||
|
||||
summary.record_reconciliation_scope_with("bucket", "object", build);
|
||||
assert_eq!(builds.get(), 1, "an admitted scope is copied once");
|
||||
assert_eq!(summary.reconciliation_scopes.len(), 1);
|
||||
assert_eq!(summary.reconciliation_scopes[0].bucket, "bucket");
|
||||
assert_eq!(summary.reconciliation_scopes[0].object, "object");
|
||||
|
||||
summary.record_reconciliation_scope_with("bucket", "object", build);
|
||||
assert_eq!(builds.get(), 1, "duplicate scopes must not be copied");
|
||||
assert_eq!(summary.reconciliation_scopes.len(), 1);
|
||||
assert!(!summary.size_reconciliation_truncated);
|
||||
|
||||
for index in 1..1024 {
|
||||
summary.record_reconciliation_scope("bucket", &index.to_string());
|
||||
}
|
||||
summary.record_reconciliation_scope_with("bucket", "object", build);
|
||||
assert_eq!(builds.get(), 1, "a duplicate at capacity must not be copied");
|
||||
assert!(!summary.size_reconciliation_truncated);
|
||||
|
||||
summary.record_reconciliation_scope_with("bucket", "overflow", build);
|
||||
assert_eq!(builds.get(), 1, "a rejected scope must not be copied");
|
||||
assert_eq!(summary.reconciliation_scopes.len(), 1024);
|
||||
assert!(summary.size_reconciliation_truncated);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_size_histogram_compat_rollup_sums_all_sub_buckets() {
|
||||
let mut hist = SizeHistogram::default();
|
||||
|
||||
@@ -25,7 +25,6 @@ workspace = true
|
||||
|
||||
[features]
|
||||
default = []
|
||||
scanner-peer-mixed-version = []
|
||||
hotpath = [
|
||||
"hotpath/hotpath",
|
||||
"hotpath/tokio",
|
||||
|
||||
@@ -26,7 +26,7 @@ Registered in [`src/lib.rs`](src/lib.rs). Grouped by concern:
|
||||
| **protocols** | [`src/protocols/`](src/protocols) | FTPS, WebDAV, SFTP compliance. Fixed ports, own guide: [`src/protocols/README.md`](src/protocols/README.md) |
|
||||
| **reliant** | [`src/reliant/`](src/reliant) | Tests that reuse an **externally started** server (SQL/select, conditional writes, lifecycle, deleted-object reads, node-interact). Run via [`scripts/run_e2e_tests.sh`](../../scripts/run_e2e_tests.sh); see [`src/reliant/README.md`](src/reliant/README.md) |
|
||||
| **cluster** | `cluster_concurrency_test`, `stale_multipart_cleanup_cluster_test`, `namespace_lock_quorum_test`, `admin_timeout_regression_test`, `object_lambda_test`, `replication_extension_test`, `tier_stats_cluster_test` | Multi-node scenarios via `RustFSTestClusterEnvironment` |
|
||||
| **distributed 4×4** | [`src/distributed/`](src/distributed) | Nightly `e2e-distributed` lane: S3, object lock/WORM, versioning, bucket/site replication, quota, expand/decommission/rebalance, concurrency, chaos, 4-node upgrade of historical data and IAM AK/SK. Map: [`docs/testing/distributed-e2e.md`](../../docs/testing/distributed-e2e.md) |
|
||||
| **distributed 4×4** | [`src/distributed/`](src/distributed) | Storage-sensitive PR and nightly `e2e-distributed` lane: S3, object lock/WORM, versioning, bucket/site replication, quota, expand/decommission/rebalance, concurrency, chaos, 4-node upgrade of historical data and IAM AK/SK. Map: [`docs/testing/distributed-e2e.md`](../../docs/testing/distributed-e2e.md) |
|
||||
| **chaos / reliability** | [`src/chaos.rs`](src/chaos.rs), `reliability_disk_fault_test`, `heal_erasure_disk_rebuild_test`, `server_startup_failfast_test` | Disk offline/replace/corrupt, EC rebuild, heal, fail-fast startup |
|
||||
| **upgrade compatibility** | `upgrade_compatibility_test` | Pinned previous-release writes followed by current-build reads on the same data directory |
|
||||
|
||||
@@ -64,25 +64,6 @@ Each build/run holds an exclusive `rustfs.e2e.lock` marker beside the binary; co
|
||||
|
||||
The protocols suite has its own fixed-port and single-worker contract in [`src/protocols/README.md`](src/protocols/README.md). Use its command under [Troubleshooting](#troubleshooting).
|
||||
|
||||
### Scanner mixed-version gate
|
||||
|
||||
The Linux N/O scanner gate needs a separately qualified `33482bb` server, which supports scoped ACKs. Set `RUSTFS_SCANNER_COMPAT_OLD_BINARY` to that executable and `RUSTFS_SCANNER_COMPAT_OLD_SHA256` to its independently verified digest. The release binary used by ordinary upgrade CI is a different baseline. Missing or mismatched inputs fail the test.
|
||||
|
||||
Enable the harness-only `scanner-peer-mixed-version` feature explicitly; keep the server wrapper features identical between build and run:
|
||||
|
||||
```bash
|
||||
python3 scripts/e2e_binary.py build --features e2e-test-hooks
|
||||
python3 scripts/e2e_binary.py run --features e2e-test-hooks -- \
|
||||
cargo nextest run --profile e2e-distributed -p e2e_test \
|
||||
--features scanner-peer-mixed-version --ignore-default-filter \
|
||||
-E 'test(=distributed::scanner_peer_bucket_test::scanner_peer_bucket_new_coordinator_certifies_old_peers_and_scopes_next_cycle)' \
|
||||
--test-threads 1 --retries 0 --no-tests=fail
|
||||
```
|
||||
|
||||
The separate `scanner_peer_bucket_new_coordinator_certifies_old_worker_peers_and_scopes_next_cycle` case uses the same feature and qualified old binary inputs. Select its exact name in the command above. It keeps the original N process enabled and starts the three official O replacements with `RUSTFS_SCANNER_ENABLED=false`: their autonomous scanner loops are disabled, while real S3 dirty tracking, authenticated snapshots, publication leases, scoped ACKs, and N-dispatched RemoteScanner workers remain active. It records each node's actual scanner status and retains the same identity, durability, root, ACK, data-conservation, and physical-walk requirements. This fixed-role case does not validate autonomous O scanning or all-enabled election behavior. The original all-enabled N/O test remains separate and has not passed acceptance; a fixed-role result cannot satisfy that gate.
|
||||
|
||||
For this fixed-role case, the `e2e-test-hooks` server also records the N client's complete bounded worker inventory. Requests are observed before opening the normal stream and paired only after authenticated source/plan validation and terminal EOF. Closed manifests and final health receipts must bind both cycles to their actual candidate/root; the next cycle needs exactly four sets of hot-only execution and at least one positive-progress worker mapped to a measured official O process. A zero-progress Current response, missing/extra request, unhealthy recorder, or missing physical coverage fails the gate. The recorder does not add O hooks or change any protocol or timeout. Final health reads check immutable closed-cycle evidence, not current N liveness.
|
||||
|
||||
### `#[ignore]` semantics
|
||||
|
||||
Ignored tests are excluded from the default `cargo nextest run` pass because
|
||||
@@ -194,9 +175,9 @@ the same profile for membership and execution with one nightly worker.
|
||||
| `s3s-e2e` black-box | `e2e-tests` + `e2e-tests-rio-v2` jobs | **Active** (external conformance tool) |
|
||||
| ILM / lifecycle (ignored) | `test-ilm-integration-serial` lane, `-j1` | **Active** (backlog#1148 ilm-1) |
|
||||
| KMS suite | `e2e-full` job, merge queue + main | **Active** |
|
||||
| Direct and mixed-version rolling upgrades from pinned previous release | `e2e-upgrade.yml`, release tags + weekly schedule + manual dispatch | **Active** |
|
||||
| Direct and mixed-version rolling upgrades from pinned previous release | `e2e-upgrade.yml`, storage-sensitive PRs + release tags + weekly | **Active** |
|
||||
| Cluster faults (`e2e-nightly` profile) | consolidated nightly workflow | **Active** (backlog#1149 ci-7) |
|
||||
| Distributed 4-node 4-disk (`e2e-distributed` profile) | `.github/workflows/e2e-distributed.yml` | **Active** (nightly / dispatch) |
|
||||
| Distributed 4-node 4-disk (`e2e-distributed` profile) | `.github/workflows/e2e-distributed.yml` | **Active** (storage-sensitive PR / nightly / dispatch) |
|
||||
| Protocols (FTPS/WebDAV/SFTP) | consolidated nightly workflow, serial | **Active** (backlog#1149 ci-7) |
|
||||
| Replication (fast subset) | `e2e-smoke` profile, `e2e-tests` job, every PR | **Active** (backlog#1147 repl-1) |
|
||||
| Replication (slow + multi-node) | `e2e-repl-nightly` profile, consolidated nightly workflow | **Active** (backlog#1147 repl-1) |
|
||||
|
||||
@@ -17,7 +17,8 @@
|
||||
//! when PublicAccessBlock configuration is missing or explicitly set.
|
||||
|
||||
use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
|
||||
use aws_sdk_s3::types::PublicAccessBlockConfiguration;
|
||||
use crate::sts_query_compat_test::{OPA_AUTH_TOKEN, OpaMock};
|
||||
use aws_sdk_s3::types::{PublicAccessBlockConfiguration, Tag, Tagging};
|
||||
use tracing::info;
|
||||
|
||||
async fn setup_public_bucket(
|
||||
@@ -69,6 +70,98 @@ async fn anonymous_get_object(
|
||||
local_http_client().get(&url).send().await
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_plain_http_matches_secure_transport_false_bucket_policy() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
assert!(env.url.starts_with("http://"), "fixture must exercise plaintext HTTP: {}", env.url);
|
||||
|
||||
let bucket = "secure-transport-policy";
|
||||
let key = "plain-http.txt";
|
||||
let admin = env.create_s3_client();
|
||||
admin.create_bucket().bucket(bucket).send().await?;
|
||||
admin
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.body(aws_sdk_s3::primitives::ByteStream::from_static(b"plain http"))
|
||||
.send()
|
||||
.await?;
|
||||
admin
|
||||
.put_bucket_policy()
|
||||
.bucket(bucket)
|
||||
.policy(
|
||||
serde_json::json!({
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{
|
||||
"Effect": "Allow",
|
||||
"Principal": "*",
|
||||
"Action": "s3:GetObject",
|
||||
"Resource": format!("arn:aws:s3:::{bucket}/*"),
|
||||
"Condition": {"Bool": {"aws:SecureTransport": "false"}}
|
||||
}]
|
||||
})
|
||||
.to_string(),
|
||||
)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let response = anonymous_get_object(&env, bucket, key).await?;
|
||||
assert_eq!(response.status().as_u16(), 200);
|
||||
assert_eq!(response.bytes().await?.as_ref(), b"plain http");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Root credentials have no attached IAM policy, so these requests depend on
|
||||
/// the owner flag reaching policy evaluation unchanged.
|
||||
#[tokio::test]
|
||||
async fn test_root_requests_keep_policy_owner_flag() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
let opa = OpaMock::start().await?;
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server_with_env(
|
||||
vec![],
|
||||
&[
|
||||
("RUSTFS_POLICY_PLUGIN_URL", opa.url.as_str()),
|
||||
("RUSTFS_POLICY_PLUGIN_AUTH_TOKEN", OPA_AUTH_TOKEN),
|
||||
],
|
||||
)
|
||||
.await?;
|
||||
|
||||
let bucket = "root-owner-flag";
|
||||
let key = "owner.txt";
|
||||
let client = env.create_s3_client();
|
||||
client.create_bucket().bucket(bucket).send().await?;
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.body(aws_sdk_s3::primitives::ByteStream::from_static(b"owner access"))
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
// Duplicate stored tags are ambiguous. An owner request must not enter the
|
||||
// OPA tag-loading path, which rejects this metadata before evaluation.
|
||||
client
|
||||
.put_bucket_tagging()
|
||||
.bucket(bucket)
|
||||
.tagging(
|
||||
Tagging::builder()
|
||||
.tag_set(Tag::builder().key("department").value("finance").build()?)
|
||||
.tag_set(Tag::builder().key("department").value("engineering").build()?)
|
||||
.build()?,
|
||||
)
|
||||
.send()
|
||||
.await?;
|
||||
let object = client.get_object().bucket(bucket).key(key).send().await?;
|
||||
|
||||
assert_eq!(object.body.collect().await?.into_bytes().as_ref(), b"owner access");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Issue #2036: Anonymous GetObject should succeed when bucket policy allows it
|
||||
/// and no PublicAccessBlock configuration exists (ConfigNotFound).
|
||||
#[tokio::test]
|
||||
|
||||
@@ -502,163 +502,10 @@ mod tests {
|
||||
assert!(response.contains(if expected == reqwest::StatusCode::FORBIDDEN {
|
||||
"<Code>AccessDenied</Code>"
|
||||
} else {
|
||||
"<Code>BucketAlreadyOwnedByYou</Code>"
|
||||
"<Code>BucketAlreadyExists</Code>"
|
||||
}));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// A deployment owns a single bucket namespace, so an IAM user that recreates
|
||||
/// an existing bucket is the owner, not another account. In the default region
|
||||
/// every such request returns the legacy idempotent `200 OK` (AWS CreateBucket
|
||||
/// semantics), including buckets created by root and by another IAM user.
|
||||
#[tokio::test]
|
||||
async fn test_create_bucket_duplicate_in_default_region_is_idempotent_for_iam_user()
|
||||
-> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec!["--region", "us-east-1"]).await?;
|
||||
let root = env.create_s3_client();
|
||||
|
||||
admin_create_user(&env, "duplicate-iam", "duplicate-iam-secret").await?;
|
||||
admin_attach_user_policy_via(
|
||||
AdminTransport::Signed,
|
||||
&env.url,
|
||||
&env.access_key,
|
||||
&env.secret_key,
|
||||
"readwrite",
|
||||
"duplicate-iam",
|
||||
)
|
||||
.await?;
|
||||
let iam = Client::from_conf(build_test_s3_config(
|
||||
&env.url,
|
||||
"duplicate-iam",
|
||||
"duplicate-iam-secret",
|
||||
None,
|
||||
"duplicate-create",
|
||||
));
|
||||
|
||||
root.create_bucket()
|
||||
.bucket("duplicate-root-owned")
|
||||
.send()
|
||||
.await
|
||||
.expect("root creates the first bucket");
|
||||
iam.create_bucket()
|
||||
.bucket("duplicate-root-owned")
|
||||
.send()
|
||||
.await
|
||||
.expect("an IAM user recreating a root-owned bucket in us-east-1 gets 200 OK");
|
||||
|
||||
iam.create_bucket()
|
||||
.bucket("duplicate-iam-owned")
|
||||
.send()
|
||||
.await
|
||||
.expect("the IAM user creates its own bucket");
|
||||
iam.create_bucket()
|
||||
.bucket("duplicate-iam-owned")
|
||||
.send()
|
||||
.await
|
||||
.expect("the same IAM user recreating its bucket gets 200 OK");
|
||||
root.create_bucket()
|
||||
.bucket("duplicate-iam-owned")
|
||||
.send()
|
||||
.await
|
||||
.expect("root recreating an IAM-created bucket gets 200 OK");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Outside the default region the same ownership answer becomes
|
||||
/// `409 BucketAlreadyOwnedByYou` for every authorized requester. Requests that
|
||||
/// lack `s3:CreateBucket` still fail authorization before the duplicate check.
|
||||
#[tokio::test]
|
||||
async fn test_create_bucket_duplicate_outside_default_region_reports_owned_by_you() -> Result<(), Box<dyn Error + Send + Sync>>
|
||||
{
|
||||
init_logging();
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec!["--region", "eu-west-1"]).await?;
|
||||
let region = "eu-west-1";
|
||||
let root = Client::from_conf(
|
||||
env.create_s3_client()
|
||||
.config()
|
||||
.to_builder()
|
||||
.region(Region::new(region))
|
||||
.build(),
|
||||
);
|
||||
|
||||
admin_create_user(&env, "duplicate-eu-iam", "duplicate-eu-iam-secret").await?;
|
||||
admin_attach_user_policy_via(
|
||||
AdminTransport::Signed,
|
||||
&env.url,
|
||||
&env.access_key,
|
||||
&env.secret_key,
|
||||
"readwrite",
|
||||
"duplicate-eu-iam",
|
||||
)
|
||||
.await?;
|
||||
let iam = Client::from_conf(
|
||||
build_test_s3_config(&env.url, "duplicate-eu-iam", "duplicate-eu-iam-secret", None, "duplicate-create")
|
||||
.to_builder()
|
||||
.region(Region::new(region))
|
||||
.build(),
|
||||
);
|
||||
|
||||
admin_create_user(&env, "duplicate-eu-readonly", "duplicate-eu-readonly-secret").await?;
|
||||
admin_attach_user_policy_via(
|
||||
AdminTransport::Signed,
|
||||
&env.url,
|
||||
&env.access_key,
|
||||
&env.secret_key,
|
||||
"readonly",
|
||||
"duplicate-eu-readonly",
|
||||
)
|
||||
.await?;
|
||||
let readonly = Client::from_conf(
|
||||
build_test_s3_config(
|
||||
&env.url,
|
||||
"duplicate-eu-readonly",
|
||||
"duplicate-eu-readonly-secret",
|
||||
None,
|
||||
"duplicate-create",
|
||||
)
|
||||
.to_builder()
|
||||
.region(Region::new(region))
|
||||
.build(),
|
||||
);
|
||||
|
||||
for (client, bucket) in [(&root, "duplicate-eu-root"), (&iam, "duplicate-eu-iam")] {
|
||||
client
|
||||
.create_bucket()
|
||||
.bucket(bucket)
|
||||
.send()
|
||||
.await
|
||||
.unwrap_or_else(|error| panic!("{bucket} must be created: {error:?}"));
|
||||
// Both the creator and the other authenticated identity own the namespace.
|
||||
for creator in [&root, &iam] {
|
||||
let error = creator
|
||||
.create_bucket()
|
||||
.bucket(bucket)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("a duplicate create outside us-east-1 must report ownership");
|
||||
assert_eq!(
|
||||
error.as_service_error().and_then(ProvideErrorMetadata::code),
|
||||
Some("BucketAlreadyOwnedByYou"),
|
||||
"{bucket}"
|
||||
);
|
||||
assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(409), "{bucket}");
|
||||
}
|
||||
}
|
||||
|
||||
let denied = readonly
|
||||
.create_bucket()
|
||||
.bucket("duplicate-eu-root")
|
||||
.send()
|
||||
.await
|
||||
.expect_err("a requester without s3:CreateBucket must be denied before the duplicate check");
|
||||
assert_eq!(denied.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,193 +16,8 @@
|
||||
mod tests {
|
||||
use crate::common::{RustFSTestEnvironment, init_logging};
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::config::http::HttpResponse;
|
||||
use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError};
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, VersioningConfiguration};
|
||||
|
||||
fn assert_not_found<E: ProvideErrorMetadata + std::fmt::Debug>(error: &SdkError<E, HttpResponse>) {
|
||||
assert_eq!(error.raw_response().map(|r| r.status().as_u16()), Some(404), "{error:?}");
|
||||
assert_eq!(
|
||||
error.as_service_error().and_then(ProvideErrorMetadata::code),
|
||||
Some("NoSuchKey"),
|
||||
"{error:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn conditional_writes_cannot_replace_delete_marker_with_if_match() {
|
||||
init_logging();
|
||||
let mut env = RustFSTestEnvironment::new().await.expect("create test environment");
|
||||
env.start_rustfs_server(vec![]).await.expect("start RustFS");
|
||||
let client = env.create_s3_client();
|
||||
let bucket = "conditional-delete-marker-proof";
|
||||
create_versioned_bucket(&client, bucket).await;
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key("source")
|
||||
.body(ByteStream::from_static(b"source"))
|
||||
.send()
|
||||
.await
|
||||
.expect("write copy source");
|
||||
|
||||
for operation in ["put", "copy", "multipart"] {
|
||||
let old = client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(operation)
|
||||
.body(ByteStream::from_static(b"old"))
|
||||
.send()
|
||||
.await
|
||||
.expect("write historical version");
|
||||
let old_etag = old.e_tag().expect("historical ETag");
|
||||
let marker = client
|
||||
.delete_object()
|
||||
.bucket(bucket)
|
||||
.key(operation)
|
||||
.send()
|
||||
.await
|
||||
.expect("create delete marker");
|
||||
let marker_id = marker.version_id().expect("delete marker version");
|
||||
|
||||
let (upload_id, completed) = if operation == "multipart" {
|
||||
let upload = client
|
||||
.create_multipart_upload()
|
||||
.bucket(bucket)
|
||||
.key(operation)
|
||||
.send()
|
||||
.await
|
||||
.expect("initiate multipart upload");
|
||||
let upload_id = upload.upload_id().expect("upload ID").to_owned();
|
||||
let part = client
|
||||
.upload_part()
|
||||
.bucket(bucket)
|
||||
.key(operation)
|
||||
.upload_id(&upload_id)
|
||||
.part_number(1)
|
||||
.body(ByteStream::from_static(b"new"))
|
||||
.send()
|
||||
.await
|
||||
.expect("upload part");
|
||||
let completed = CompletedMultipartUpload::builder()
|
||||
.parts(
|
||||
CompletedPart::builder()
|
||||
.part_number(1)
|
||||
.set_e_tag(part.e_tag().map(str::to_owned))
|
||||
.build(),
|
||||
)
|
||||
.build();
|
||||
(Some(upload_id), Some(completed))
|
||||
} else {
|
||||
(None, None)
|
||||
};
|
||||
|
||||
for etag in [old_etag, "\"00000000000000000000000000000000\""] {
|
||||
match operation {
|
||||
"put" => {
|
||||
let err = client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(operation)
|
||||
.if_match(etag)
|
||||
.body(ByteStream::from_static(b"new"))
|
||||
.send()
|
||||
.await
|
||||
.expect_err("If-Match must reject marker");
|
||||
assert_not_found(&err);
|
||||
}
|
||||
"copy" => {
|
||||
let err = client
|
||||
.copy_object()
|
||||
.bucket(bucket)
|
||||
.key(operation)
|
||||
.copy_source(format!("{bucket}/source"))
|
||||
.if_match(etag)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("If-Match copy must reject marker");
|
||||
assert_not_found(&err);
|
||||
}
|
||||
_ => {
|
||||
let err = client
|
||||
.complete_multipart_upload()
|
||||
.bucket(bucket)
|
||||
.key(operation)
|
||||
.upload_id(upload_id.as_deref().expect("upload ID"))
|
||||
.multipart_upload(completed.clone().expect("completed parts"))
|
||||
.if_match(etag)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("If-Match completion must reject marker");
|
||||
assert_not_found(&err);
|
||||
}
|
||||
}
|
||||
let versions = client
|
||||
.list_object_versions()
|
||||
.bucket(bucket)
|
||||
.prefix(operation)
|
||||
.send()
|
||||
.await
|
||||
.expect("list versions after rejected write");
|
||||
assert_eq!(versions.versions().len(), 1, "{operation}: failed write created a version");
|
||||
assert_eq!(versions.delete_markers().len(), 1);
|
||||
assert_eq!(versions.delete_markers()[0].version_id(), Some(marker_id));
|
||||
assert_eq!(versions.delete_markers()[0].is_latest(), Some(true));
|
||||
}
|
||||
|
||||
match operation {
|
||||
"put" => {
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(operation)
|
||||
.if_none_match("*")
|
||||
.body(ByteStream::from_static(b"new"))
|
||||
.send()
|
||||
.await
|
||||
.expect("If-None-Match PUT over marker");
|
||||
}
|
||||
"copy" => {
|
||||
client
|
||||
.copy_object()
|
||||
.bucket(bucket)
|
||||
.key(operation)
|
||||
.copy_source(format!("{bucket}/source"))
|
||||
.if_none_match("*")
|
||||
.send()
|
||||
.await
|
||||
.expect("If-None-Match copy over marker");
|
||||
}
|
||||
_ => {
|
||||
client
|
||||
.complete_multipart_upload()
|
||||
.bucket(bucket)
|
||||
.key(operation)
|
||||
.upload_id(upload_id.as_deref().expect("upload ID"))
|
||||
.multipart_upload(completed.expect("completed parts"))
|
||||
.if_none_match("*")
|
||||
.send()
|
||||
.await
|
||||
.expect("If-None-Match completion over marker");
|
||||
}
|
||||
}
|
||||
let current = client
|
||||
.get_object()
|
||||
.bucket(bucket)
|
||||
.key(operation)
|
||||
.send()
|
||||
.await
|
||||
.expect("read successful conditional write")
|
||||
.body
|
||||
.collect()
|
||||
.await
|
||||
.expect("read object body")
|
||||
.into_bytes();
|
||||
let expected: &[u8] = if operation == "copy" { b"source" } else { b"new" };
|
||||
assert_eq!(current.as_ref(), expected, "{operation}: unexpected current object");
|
||||
}
|
||||
}
|
||||
use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
|
||||
|
||||
async fn create_versioned_bucket(client: &Client, bucket: &str) {
|
||||
client
|
||||
|
||||
@@ -12,249 +12,11 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::harness::{DistCluster, DistLayout, TestResult, payload_for, s3_probe_client, sha256_hex, unique_bucket};
|
||||
use super::harness::{DistCluster, DistLayout, TestResult, assert_object_bytes, payload_for, put_object, unique_bucket};
|
||||
use crate::common::init_logging;
|
||||
use aws_sdk_s3::{
|
||||
Client,
|
||||
error::{ProvideErrorMetadata, SdkError},
|
||||
primitives::ByteStream,
|
||||
};
|
||||
use std::any::TypeId;
|
||||
use std::collections::BTreeSet;
|
||||
use std::fmt::Debug;
|
||||
use std::future::Future;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use tokio::sync::Barrier;
|
||||
use tokio::time::Instant;
|
||||
|
||||
const LOCK_ADMISSION_BUDGET: Duration = Duration::from_secs(45);
|
||||
|
||||
async fn after_lock_contention<T, E, F, Fut>(operation: &str, mut call: F) -> TestResult<Result<T, SdkError<E>>>
|
||||
where
|
||||
E: Debug + ProvideErrorMetadata + 'static,
|
||||
F: FnMut() -> Fut,
|
||||
Fut: Future<Output = Result<T, SdkError<E>>>,
|
||||
{
|
||||
let started = Instant::now();
|
||||
let mut attempts = 0;
|
||||
// The workload uses unique unversioned keys. Repeat only explicitly
|
||||
// advertised lock backpressure; every step still needs its real result.
|
||||
let outcome = tokio::time::timeout(LOCK_ADMISSION_BUDGET, async {
|
||||
loop {
|
||||
attempts += 1;
|
||||
let result = call().await;
|
||||
let retry = result.as_ref().err().is_some_and(|error| {
|
||||
let Some(response) = error.raw_response() else {
|
||||
return false;
|
||||
};
|
||||
let Some(service_error) = error.as_service_error() else {
|
||||
return false;
|
||||
};
|
||||
// HEAD has no XML error body, so its SDK service error can
|
||||
// lack a code even when the server supplies retry headers.
|
||||
let namespace_code = service_error.code() == Some("ServiceUnavailable")
|
||||
|| (service_error.code().is_none()
|
||||
&& TypeId::of::<E>() == TypeId::of::<aws_sdk_s3::operation::head_object::HeadObjectError>());
|
||||
response.status().as_u16() == 503
|
||||
&& namespace_code
|
||||
&& response.headers().get("x-rustfs-error-cause") == Some("namespace_lock_timeout")
|
||||
&& response.headers().get("retry-after") == Some("1")
|
||||
});
|
||||
if !retry {
|
||||
return result;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_secs(1)).await;
|
||||
}
|
||||
})
|
||||
.await;
|
||||
if attempts > 1 || !matches!(&outcome, Ok(Ok(_))) {
|
||||
eprintln!(
|
||||
"mixed workload request: operation={operation}, attempts={attempts}, elapsed={:?}",
|
||||
started.elapsed()
|
||||
);
|
||||
}
|
||||
outcome.map_err(|_| {
|
||||
format!("{operation} exceeded its {LOCK_ADMISSION_BUDGET:?} request budget after {attempts} attempts").into()
|
||||
})
|
||||
}
|
||||
|
||||
async fn assert_workload_bytes(client: &Client, bucket: &str, key: &str, expected: &[u8]) -> TestResult {
|
||||
let output = after_lock_contention(&format!("GET {key}"), || client.get_object().bucket(bucket).key(key).send()).await??;
|
||||
let actual = output.body.collect().await?.into_bytes();
|
||||
if actual.as_ref() != expected {
|
||||
return Err(format!(
|
||||
"object {key} bytes mismatch: expected {} bytes sha256={} got {} bytes sha256={}",
|
||||
expected.len(),
|
||||
sha256_hex(expected),
|
||||
actual.len(),
|
||||
sha256_hex(&actual)
|
||||
)
|
||||
.into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use aws_smithy_http_client::test_util::infallible_client_fn;
|
||||
use aws_smithy_types::body::SdkBody;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
||||
type Reply = (u16, &'static str, Option<&'static str>, Option<&'static str>, &'static str);
|
||||
|
||||
fn client(respond: impl Fn(usize) -> Reply + Send + Sync + 'static) -> (Client, Arc<AtomicUsize>) {
|
||||
let requests = Arc::new(AtomicUsize::new(0));
|
||||
let seen = Arc::clone(&requests);
|
||||
let transport = infallible_client_fn(move |_| {
|
||||
let (status, code, cause, retry_after, body) = respond(seen.fetch_add(1, Ordering::SeqCst));
|
||||
let body = if code.is_empty() {
|
||||
body.to_string()
|
||||
} else {
|
||||
format!("<Error><Code>{code}</Code></Error>")
|
||||
};
|
||||
let mut response = http::Response::builder()
|
||||
.status(status)
|
||||
.header("content-type", "application/xml")
|
||||
.header("content-length", body.len());
|
||||
if let Some(cause) = cause {
|
||||
response = response.header("x-rustfs-error-cause", cause);
|
||||
}
|
||||
if let Some(retry_after) = retry_after {
|
||||
response = response.header("retry-after", retry_after);
|
||||
}
|
||||
response.body(SdkBody::from(body)).expect("scripted S3 response")
|
||||
});
|
||||
let config = crate::common::build_test_s3_config(
|
||||
"http://contention.example.invalid",
|
||||
"synthetic-access",
|
||||
"synthetic-secret",
|
||||
None,
|
||||
"contention-test",
|
||||
);
|
||||
let client = Client::from_conf(config.to_builder().http_client(transport).build());
|
||||
(s3_probe_client(&client), requests)
|
||||
}
|
||||
|
||||
fn contention() -> Reply {
|
||||
(503, "ServiceUnavailable", Some("namespace_lock_timeout"), Some("1"), "")
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn advertised_contention_recovers_after_standard_sdk_attempts_are_exhausted() -> TestResult {
|
||||
let (client, requests) = client(|attempt| {
|
||||
if attempt < 3 {
|
||||
contention()
|
||||
} else {
|
||||
(200, "", None, None, "")
|
||||
}
|
||||
});
|
||||
let standard = Client::from_conf(
|
||||
client
|
||||
.config()
|
||||
.to_builder()
|
||||
.retry_config(aws_sdk_s3::config::retry::RetryConfig::standard().with_max_attempts(3))
|
||||
.build(),
|
||||
);
|
||||
standard
|
||||
.put_object()
|
||||
.bucket("bucket")
|
||||
.key("key")
|
||||
.body(ByteStream::from_static(b"body"))
|
||||
.send()
|
||||
.await
|
||||
.expect_err("three advertised lock responses exhaust the standard SDK policy");
|
||||
assert_eq!(requests.load(Ordering::SeqCst), 3);
|
||||
requests.store(0, Ordering::SeqCst);
|
||||
let started = Instant::now();
|
||||
after_lock_contention("PUT recovery", || {
|
||||
client
|
||||
.put_object()
|
||||
.bucket("bucket")
|
||||
.key("key")
|
||||
.body(ByteStream::from_static(b"body"))
|
||||
.send()
|
||||
})
|
||||
.await??;
|
||||
assert_eq!(requests.load(Ordering::SeqCst), 4);
|
||||
assert!(started.elapsed() >= Duration::from_secs(3), "honor each Retry-After response");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn bodyless_head_contention_uses_explicit_retry_headers() -> TestResult {
|
||||
let (client, requests) = client(|attempt| {
|
||||
if attempt == 0 {
|
||||
(503, "", Some("namespace_lock_timeout"), Some("1"), "")
|
||||
} else {
|
||||
(200, "", None, None, "")
|
||||
}
|
||||
});
|
||||
after_lock_contention("HEAD recovery", || client.head_object().bucket("bucket").key("key").send()).await??;
|
||||
assert_eq!(requests.load(Ordering::SeqCst), 2);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn unrelated_or_malformed_responses_are_not_retried() -> TestResult {
|
||||
for reply in [
|
||||
(500, "InternalError", Some("namespace_lock_timeout"), Some("1"), ""),
|
||||
(403, "AccessDenied", None, None, ""),
|
||||
(404, "NoSuchKey", None, None, ""),
|
||||
(503, "ServiceUnavailable", Some("storage_write_quorum"), None, ""),
|
||||
(503, "ServiceUnavailable", Some("namespace_lock_unavailable"), Some("1"), ""),
|
||||
(503, "InternalError", Some("namespace_lock_timeout"), Some("1"), ""),
|
||||
(503, "", Some("namespace_lock_timeout"), Some("1"), ""),
|
||||
(503, "ServiceUnavailable", None, Some("1"), ""),
|
||||
(503, "ServiceUnavailable", Some("namespace_lock_timeout"), None, ""),
|
||||
(503, "ServiceUnavailable", Some("namespace_lock_timeout"), Some("2"), ""),
|
||||
] {
|
||||
let (client, requests) = client(move |_| reply);
|
||||
let result = after_lock_contention("PUT fatal response", || {
|
||||
client
|
||||
.put_object()
|
||||
.bucket("bucket")
|
||||
.key("key")
|
||||
.body(ByteStream::from_static(b"body"))
|
||||
.send()
|
||||
})
|
||||
.await?;
|
||||
assert!(result.is_err(), "fatal response cannot become success: {reply:?}");
|
||||
assert_eq!(requests.load(Ordering::SeqCst), 1, "fatal response must remain visible: {reply:?}");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn persistent_lock_contention_exhausts_the_request_budget() {
|
||||
let (client, requests) = client(|_| contention());
|
||||
let started = Instant::now();
|
||||
let result = after_lock_contention("PUT unavailable", || {
|
||||
client
|
||||
.put_object()
|
||||
.bucket("bucket")
|
||||
.key("key")
|
||||
.body(ByteStream::from_static(b"body"))
|
||||
.send()
|
||||
})
|
||||
.await;
|
||||
assert!(result.is_err(), "persistent contention cannot acknowledge an operation");
|
||||
assert!(started.elapsed() >= LOCK_ADMISSION_BUDGET);
|
||||
assert!(requests.load(Ordering::SeqCst) > 1);
|
||||
assert!(requests.load(Ordering::SeqCst) <= LOCK_ADMISSION_BUDGET.as_secs() as usize + 1);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn acknowledged_wrong_bytes_fail_without_retry() {
|
||||
let (client, requests) = client(|_| (200, "", None, None, "wrong bytes"));
|
||||
let error = assert_workload_bytes(&client, "bucket", "key", b"expected bytes")
|
||||
.await
|
||||
.expect_err("a successful but incorrect read is a consistency failure");
|
||||
assert!(error.to_string().contains("bytes mismatch"));
|
||||
assert_eq!(requests.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node() -> TestResult {
|
||||
@@ -262,9 +24,7 @@ async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node()
|
||||
let dist = DistCluster::start(DistLayout::FourByFour).await?;
|
||||
let bucket = unique_bucket("conc");
|
||||
dist.create_bucket(&bucket).await?;
|
||||
// This consistency probe owns its admission policy; unrelated responses
|
||||
// must not be hidden by the SDK's general-purpose retries.
|
||||
let clients = Arc::new(dist.clients()?.iter().map(s3_probe_client).collect::<Vec<_>>());
|
||||
let clients = Arc::new(dist.clients()?);
|
||||
let worker_count = 24;
|
||||
let rounds = 4;
|
||||
let barrier = Arc::new(Barrier::new(worker_count));
|
||||
@@ -284,46 +44,31 @@ async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node()
|
||||
let key = format!("source/worker-{idx:02}-round-{round}.bin");
|
||||
let copy_key = format!("retained/worker-{idx:02}-round-{round}.bin");
|
||||
let body = payload_for(&key, 64 * 1024);
|
||||
let context = format!("worker={idx} round={round}");
|
||||
after_lock_contention(&format!("{context} PUT source"), || {
|
||||
writer
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key(&key)
|
||||
.body(ByteStream::from(body.clone()))
|
||||
.send()
|
||||
})
|
||||
.await??;
|
||||
put_object(writer, &bucket, &key, body.clone()).await?;
|
||||
|
||||
let head = after_lock_contention(&format!("{context} HEAD source"), || {
|
||||
reader.head_object().bucket(&bucket).key(&key).send()
|
||||
})
|
||||
.await??;
|
||||
let head = reader.head_object().bucket(&bucket).key(&key).send().await?;
|
||||
if head.content_length() != Some(body.len() as i64) {
|
||||
return Err(format!("HEAD returned the wrong size for {key}: {head:?}").into());
|
||||
}
|
||||
assert_workload_bytes(reader, &bucket, &key, &body).await?;
|
||||
assert_object_bytes(reader, &bucket, &key, &body).await?;
|
||||
|
||||
after_lock_contention(&format!("{context} COPY retained"), || {
|
||||
copier
|
||||
.copy_object()
|
||||
.bucket(&bucket)
|
||||
.key(©_key)
|
||||
.copy_source(format!("{bucket}/{key}"))
|
||||
.send()
|
||||
})
|
||||
.await??;
|
||||
assert_workload_bytes(writer, &bucket, ©_key, &body).await?;
|
||||
copier
|
||||
.copy_object()
|
||||
.bucket(&bucket)
|
||||
.key(©_key)
|
||||
.copy_source(format!("{bucket}/{key}"))
|
||||
.send()
|
||||
.await?;
|
||||
assert_object_bytes(writer, &bucket, ©_key, &body).await?;
|
||||
|
||||
after_lock_contention(&format!("{context} DELETE source"), || {
|
||||
writer.delete_object().bucket(&bucket).key(&key).send()
|
||||
})
|
||||
.await??;
|
||||
let missing = after_lock_contention(&format!("{context} HEAD deleted source"), || {
|
||||
reader.head_object().bucket(&bucket).key(&key).send()
|
||||
})
|
||||
.await?
|
||||
.expect_err("deleted source key must not remain visible");
|
||||
writer.delete_object().bucket(&bucket).key(&key).send().await?;
|
||||
let missing = reader
|
||||
.head_object()
|
||||
.bucket(&bucket)
|
||||
.key(&key)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("deleted source key must not remain visible");
|
||||
if missing.raw_response().map(|response| response.status().as_u16()) != Some(404) {
|
||||
return Err(format!("deleted source {key} returned an unexpected result: {missing:?}").into());
|
||||
}
|
||||
@@ -337,22 +82,14 @@ async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node()
|
||||
for handle in handles {
|
||||
inventory.extend(handle.await??);
|
||||
}
|
||||
assert_eq!(
|
||||
inventory.len(),
|
||||
worker_count * rounds,
|
||||
"every worker round must retain its acknowledged copy"
|
||||
);
|
||||
|
||||
let expected_keys: BTreeSet<_> = inventory.iter().map(|(key, _)| key.as_str()).collect();
|
||||
for (node_idx, client) in clients.iter().enumerate() {
|
||||
let listed = after_lock_contention(&format!("node={node_idx} LIST retained"), || {
|
||||
client.list_objects_v2().bucket(&bucket).prefix("retained/").send()
|
||||
})
|
||||
.await??;
|
||||
let listed = client.list_objects_v2().bucket(&bucket).prefix("retained/").send().await?;
|
||||
let listed_keys: BTreeSet<_> = listed.contents().iter().filter_map(|object| object.key()).collect();
|
||||
assert_eq!(listed_keys, expected_keys, "node {node_idx} returned a divergent retained-key listing");
|
||||
for (key, body) in &inventory {
|
||||
assert_workload_bytes(client, &bucket, key, body)
|
||||
assert_object_bytes(client, &bucket, key, body)
|
||||
.await
|
||||
.map_err(|error| format!("node {node_idx} failed to read {key}: {error}"))?;
|
||||
}
|
||||
|
||||
@@ -39,8 +39,6 @@ mod s3_during_data_movement_test;
|
||||
mod scanner_heal_topology_fixture;
|
||||
mod scanner_heal_topology_test;
|
||||
mod scanner_heal_topology_tier_test;
|
||||
mod scanner_peer_bucket_fixture;
|
||||
mod scanner_peer_bucket_test;
|
||||
mod site_replication_test;
|
||||
mod table_catalog_test;
|
||||
mod upgrade_test;
|
||||
|
||||
@@ -18,7 +18,7 @@ use super::harness::{
|
||||
};
|
||||
use crate::common::{ClusterTopology, RustFSTestClusterEnvironment};
|
||||
use crate::rpc_fault_proxy::{RpcFaultProxy, RpcPause};
|
||||
use crate::storage_api::multipart_fence_acceptance::{LOCK_DIAGNOSTICS_FILTER, RUSTFS_META_MULTIPART_BUCKET};
|
||||
use crate::storage_api::multipart_fence_acceptance::RUSTFS_META_MULTIPART_BUCKET;
|
||||
use aws_sdk_s3::{
|
||||
Client,
|
||||
config::retry::RetryConfig,
|
||||
@@ -38,16 +38,14 @@ const DEADLINE: Duration = Duration::from_secs(45);
|
||||
async fn fixture(
|
||||
drives_per_node: usize,
|
||||
storage_class: &str,
|
||||
lock_diagnostics: bool,
|
||||
) -> TestResult<(RustFSTestClusterEnvironment, Vec<RpcFaultProxy>, Vec<Client>, String)> {
|
||||
fixture_from_binary(drives_per_node, storage_class, None, lock_diagnostics).await
|
||||
fixture_from_binary(drives_per_node, storage_class, None).await
|
||||
}
|
||||
|
||||
async fn fixture_from_binary(
|
||||
drives_per_node: usize,
|
||||
storage_class: &str,
|
||||
binary: Option<&std::path::Path>,
|
||||
lock_diagnostics: bool,
|
||||
) -> TestResult<(RustFSTestClusterEnvironment, Vec<RpcFaultProxy>, Vec<Client>, String)> {
|
||||
let mut cluster =
|
||||
RustFSTestClusterEnvironment::with_topology(ClusterTopology::single_pool_multidrive(4, drives_per_node)).await?;
|
||||
@@ -92,10 +90,6 @@ async fn fixture_from_binary(
|
||||
] {
|
||||
cluster.set_env(name, value);
|
||||
}
|
||||
if lock_diagnostics {
|
||||
cluster.set_env("RUST_LOG", LOCK_DIAGNOSTICS_FILTER);
|
||||
cluster.set_env("RUSTFS_OBJECT_LOCK_DIAG_ENABLE", "true");
|
||||
}
|
||||
let mut proxies = Vec::new();
|
||||
if binary.is_none() {
|
||||
for node in 0..4 {
|
||||
@@ -369,120 +363,26 @@ fn upload_path_scope(bucket: &str, key: &str, upload_id: &str) -> TestResult<Str
|
||||
))
|
||||
}
|
||||
|
||||
async fn with_paused_peers<T, E>(
|
||||
async fn with_paused_peers<T: Send + 'static>(
|
||||
proxies: &[RpcFaultProxy],
|
||||
methods: &[&'static str],
|
||||
limits: &[usize],
|
||||
scope: &str,
|
||||
request: impl std::future::Future<Output = Result<T, SdkError<E>>> + Send + 'static,
|
||||
) -> TestResult<Result<T, SdkError<E>>>
|
||||
where
|
||||
T: Send + 'static,
|
||||
E: std::error::Error + ProvideErrorMetadata + Send + Sync + 'static,
|
||||
{
|
||||
let deadline = tokio::time::Instant::now() + DEADLINE;
|
||||
request: impl std::future::Future<Output = T> + Send + 'static,
|
||||
) -> TestResult<T> {
|
||||
let pauses: Vec<_> = proxies[1..=limits.len()]
|
||||
.iter()
|
||||
.zip(limits)
|
||||
.map(|(proxy, limit)| proxy.pause(methods, *limit, Some(scope)))
|
||||
.collect::<Result<_, _>>()?;
|
||||
let mut task = tokio::spawn(request);
|
||||
let mut joined = None;
|
||||
let mut deadline_expired = false;
|
||||
let mut barrier_result = {
|
||||
let faults = wait_for_faults(&pauses, methods);
|
||||
tokio::pin!(faults);
|
||||
match tokio::time::timeout_at(deadline, async {
|
||||
tokio::select! {
|
||||
result = &mut faults => result,
|
||||
result = &mut task => {
|
||||
joined = Some(result);
|
||||
// A ready request is not evidence that every selected fault
|
||||
// was reached. Poll the original barrier once for that proof.
|
||||
match futures::poll!(faults.as_mut()) {
|
||||
std::task::Poll::Ready(result) => result,
|
||||
std::task::Poll::Pending => Err("SDK request completed before the selected RPC fault barrier".into()),
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
{
|
||||
Ok(result) => result,
|
||||
Err(error) => {
|
||||
deadline_expired = true;
|
||||
Err(std::io::Error::new(std::io::ErrorKind::TimedOut, error).into())
|
||||
}
|
||||
}
|
||||
};
|
||||
// Tokio's timeout polls an already-ready inner future before its timer.
|
||||
// A late observation cannot certify the original absolute deadline.
|
||||
deadline_expired |= tokio::time::Instant::now() >= deadline;
|
||||
if deadline_expired {
|
||||
barrier_result = Err(std::io::Error::new(
|
||||
std::io::ErrorKind::TimedOut,
|
||||
"selected RPC fault proof observed after the absolute MPU deadline",
|
||||
)
|
||||
.into());
|
||||
}
|
||||
if barrier_result.is_err() {
|
||||
for pause in &pauses {
|
||||
pause.discard_pending();
|
||||
}
|
||||
}
|
||||
let mut client_task_aborted = false;
|
||||
let request_result = match joined {
|
||||
Some(result) => result,
|
||||
None => match tokio::time::timeout_at(deadline, &mut task).await {
|
||||
Ok(result) => result,
|
||||
Err(error) => {
|
||||
deadline_expired = true;
|
||||
for pause in &pauses {
|
||||
pause.discard_pending();
|
||||
}
|
||||
barrier_result = Err(std::io::Error::new(std::io::ErrorKind::TimedOut, error).into());
|
||||
// This owns only the test's SDK request, not server/native work.
|
||||
// Retire the client task without granting another operation budget.
|
||||
client_task_aborted = true;
|
||||
task.abort();
|
||||
task.await
|
||||
}
|
||||
},
|
||||
};
|
||||
deadline_expired |= tokio::time::Instant::now() >= deadline;
|
||||
if deadline_expired {
|
||||
barrier_result = Err(std::io::Error::new(
|
||||
std::io::ErrorKind::TimedOut,
|
||||
"SDK request result observed after the absolute MPU deadline",
|
||||
)
|
||||
.into());
|
||||
}
|
||||
let hits: Vec<_> = pauses.iter().map(RpcPause::hits).collect();
|
||||
let task = tokio::spawn(request);
|
||||
wait_for_faults(&pauses, methods).await?;
|
||||
let result = tokio::time::timeout(DEADLINE, task).await??;
|
||||
for pause in &pauses {
|
||||
pause.discard_pending();
|
||||
}
|
||||
drop(pauses);
|
||||
if barrier_result.is_err() {
|
||||
let sdk_error = request_result.as_ref().ok().and_then(|result| result.as_ref().err());
|
||||
let response = sdk_error.and_then(SdkError::raw_response);
|
||||
eprintln!(
|
||||
"MPU fault barrier failed: methods={methods:?}, hits={hits:?}, client_task_aborted={client_task_aborted}, deadline_expired={deadline_expired}, sdk_code={:?}, cause={:?}, request_id={:?}",
|
||||
sdk_error
|
||||
.and_then(|error| error.as_service_error())
|
||||
.and_then(|error| error.meta().code()),
|
||||
response.and_then(|response| response.headers().get("x-rustfs-error-cause")),
|
||||
response.and_then(|response| response.headers().get("x-amz-request-id")),
|
||||
);
|
||||
}
|
||||
match (barrier_result, request_result) {
|
||||
// Preserve the actual SDK source instead of replacing an early service
|
||||
// failure with an unrelated fault-wait timeout. The outer Err stays red.
|
||||
(Err(_), Ok(Err(error))) => Err(Box::new(error)),
|
||||
(Err(_), Err(error)) if !client_task_aborted => Err(Box::new(error)),
|
||||
(Err(error), _) => Err(error),
|
||||
(Ok(()), Ok(result)) => Ok(result),
|
||||
(Ok(()), Err(error)) => Err(Box::new(error)),
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
fn quorum_outcome<T: std::fmt::Debug, E: std::fmt::Debug + ProvideErrorMetadata>(
|
||||
@@ -1418,7 +1318,7 @@ async fn concurrent_capped_parts_respect_limit_on_distinct_nodes(clients: &[Clie
|
||||
|
||||
#[tokio::test]
|
||||
async fn metadata_delay_preserves_write_quorum_and_rejects_quorum_minus_one() -> TestResult {
|
||||
let (cluster, proxies, clients, bucket) = fixture(1, "EC:2", true).await?;
|
||||
let (cluster, proxies, clients, bucket) = fixture(1, "EC:2").await?;
|
||||
list_parts_disk_loss_after_metadata_read(&cluster, &proxies, &clients, &bucket).await?;
|
||||
concurrent_capped_parts_respect_limit_on_distinct_nodes(&clients, &bucket).await?;
|
||||
phase_quorum_matrix(&cluster, &proxies, &clients, &bucket, false, 1, 3).await
|
||||
@@ -1426,13 +1326,13 @@ async fn metadata_delay_preserves_write_quorum_and_rejects_quorum_minus_one() ->
|
||||
|
||||
#[tokio::test]
|
||||
async fn lock_delay_preserves_admission_quorum_and_reports_timeout() -> TestResult {
|
||||
let (cluster, proxies, clients, bucket) = fixture(1, "EC:2", false).await?;
|
||||
let (cluster, proxies, clients, bucket) = fixture(1, "EC:2").await?;
|
||||
phase_quorum_matrix(&cluster, &proxies, &clients, &bucket, true, 1, 3).await
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn sixteen_shard_multipart_metadata_requires_twelve_votes() -> TestResult {
|
||||
let (cluster, proxies, clients, bucket) = fixture(4, "EC:4", false).await?;
|
||||
let (cluster, proxies, clients, bucket) = fixture(4, "EC:4").await?;
|
||||
let storage = cluster_admin_ok(&cluster, http::Method::GET, "/rustfs/admin/v3/storageinfo", None).await?;
|
||||
let storage: serde_json::Value = serde_json::from_str(&storage)?;
|
||||
let storage: rustfs_madmin::StorageInfo = serde_json::from_value(storage["info"].clone())?;
|
||||
@@ -1446,7 +1346,7 @@ async fn sixteen_shard_multipart_metadata_requires_twelve_votes() -> TestResult
|
||||
#[tokio::test]
|
||||
async fn mixed_version_failed_complete_requires_current_coordinator_for_abort() -> TestResult {
|
||||
let previous = super::upgrade_test::source_binary()?;
|
||||
let (mut cluster, mut proxies, clients, bucket) = fixture_from_binary(4, "EC:4", Some(&previous), false).await?;
|
||||
let (mut cluster, mut proxies, clients, bucket) = fixture_from_binary(4, "EC:4", Some(&previous)).await?;
|
||||
assert!(
|
||||
proxies.is_empty(),
|
||||
"the genuine previous fleet boots and seeds through its normal direct transport"
|
||||
@@ -1649,7 +1549,7 @@ async fn native_certified_partial_abort_retry_reclaims_survivors_below_metadata_
|
||||
capability_sha256: Option<[u8; 32]>,
|
||||
abort: Option<AbortRef>,
|
||||
}
|
||||
let (mut cluster, proxies, clients, bucket) = fixture(4, "EC:4", false).await?;
|
||||
let (mut cluster, proxies, clients, bucket) = fixture(4, "EC:4").await?;
|
||||
let key = "partial-abort";
|
||||
let committed = payload_for("committed-before-abort", 4096);
|
||||
clients[0]
|
||||
@@ -1865,7 +1765,7 @@ async fn native_certified_partial_abort_retry_reclaims_survivors_below_metadata_
|
||||
#[tokio::test]
|
||||
async fn late_complete_packets_cannot_publish_after_abort_and_bucket_recreation() -> TestResult {
|
||||
for size in [0, 8 * 1024 * 1024] {
|
||||
let (cluster, proxies, clients, bucket) = fixture(1, "EC:2", false).await?;
|
||||
let (cluster, proxies, clients, bucket) = fixture(1, "EC:2").await?;
|
||||
let key = "late-complete";
|
||||
let upload = clients[0].create_multipart_upload().bucket(&bucket).key(key).send().await?;
|
||||
let upload_id = upload.upload_id().ok_or("missing upload ID")?;
|
||||
@@ -1992,7 +1892,7 @@ async fn late_complete_packets_cannot_publish_after_abort_and_bucket_recreation(
|
||||
async fn late_part_packets_preserve_retired_and_fresh_uploads() -> TestResult {
|
||||
for (method, settles) in [("PreparePartTransaction", false), ("SettlePartTransaction", true)] {
|
||||
for recreate_bucket in [false, true] {
|
||||
let (cluster, proxies, clients, bucket) = fixture(1, "EC:2", false).await?;
|
||||
let (cluster, proxies, clients, bucket) = fixture(1, "EC:2").await?;
|
||||
let key = "late-part-transaction";
|
||||
let previous = payload_for("committed-before-late-part", 4096);
|
||||
clients[0]
|
||||
@@ -2175,7 +2075,7 @@ async fn late_part_packets_preserve_retired_and_fresh_uploads() -> TestResult {
|
||||
|
||||
#[tokio::test]
|
||||
async fn cancelled_metadata_wait_releases_fence_for_bucket_recreation() -> TestResult {
|
||||
let (_cluster, proxies, clients, bucket) = fixture(1, "EC:2", false).await?;
|
||||
let (_cluster, proxies, clients, bucket) = fixture(1, "EC:2").await?;
|
||||
let pauses: Vec<_> = proxies[1..]
|
||||
.iter()
|
||||
.map(|proxy| proxy.pause(&["WriteMetadata"], usize::MAX, Some(&bucket)))
|
||||
@@ -2198,7 +2098,7 @@ async fn cancelled_metadata_wait_releases_fence_for_bucket_recreation() -> TestR
|
||||
async fn part_resend_owner_serializes_complete_and_abort_after_client_cancellation() -> TestResult {
|
||||
use md5::{Digest as _, Md5};
|
||||
|
||||
let (cluster, proxies, clients, bucket) = fixture(1, "EC:2", false).await?;
|
||||
let (cluster, proxies, clients, bucket) = fixture(1, "EC:2").await?;
|
||||
for cancel_waiter in [false, true] {
|
||||
for complete in [false, true] {
|
||||
let key = format!("resend-{cancel_waiter}-{complete}");
|
||||
@@ -2686,7 +2586,7 @@ async fn resume_at_rpc<T: Send + 'static>(
|
||||
|
||||
#[tokio::test]
|
||||
async fn each_multipart_phase_resumes_after_metadata_and_lock_delays() -> TestResult {
|
||||
let (_cluster, proxies, clients, bucket) = fixture(1, "EC:2", false).await?;
|
||||
let (_cluster, proxies, clients, bucket) = fixture(1, "EC:2").await?;
|
||||
for locks in [false, true] {
|
||||
let key = format!("phase-{locks}");
|
||||
let client = clients[0].clone();
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,512 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! What one stack answered, and how two answers are compared.
|
||||
//!
|
||||
//! An [`Observation`] keeps only what a client acts on: the HTTP status, the S3 error code, the
|
||||
//! allowlisted response headers and a few typed fields decoded from the response. The error
|
||||
//! message is kept for diagnostics and never compared: message wording is judged by the wire
|
||||
//! corpus, not here. The comparator is pure so it can be tested without a server.
|
||||
|
||||
use serde::Serialize;
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
/// Response headers compared exactly (after [`normalized_header_value`]).
|
||||
const HEADER_ALLOWLIST: [&str; 7] = [
|
||||
"content-type",
|
||||
"x-amz-version-id",
|
||||
"etag",
|
||||
"x-amz-delete-marker",
|
||||
"content-range",
|
||||
"accept-ranges",
|
||||
"x-amz-request-charged",
|
||||
];
|
||||
|
||||
/// Every response header with this prefix is compared as well.
|
||||
const HEADER_ALLOWLIST_PREFIX: &str = "x-amz-server-side-encryption";
|
||||
|
||||
/// Placeholder for a field one side did not send.
|
||||
const ABSENT: &str = "<absent>";
|
||||
|
||||
/// The lowercase header name if the comparator looks at this header.
|
||||
pub(crate) fn compared_header(name: &str) -> Option<String> {
|
||||
let name = name.to_ascii_lowercase();
|
||||
(HEADER_ALLOWLIST.contains(&name.as_str()) || name.starts_with(HEADER_ALLOWLIST_PREFIX)).then_some(name)
|
||||
}
|
||||
|
||||
/// A version ID is minted per process, so its value never matches across two servers; only
|
||||
/// whether it is the literal `null` (an unversioned write) carries meaning.
|
||||
fn normalized_header_value(name: &str, value: &str) -> String {
|
||||
if name == "x-amz-version-id" && value != "null" {
|
||||
"<version-id>".to_owned()
|
||||
} else {
|
||||
value.to_owned()
|
||||
}
|
||||
}
|
||||
|
||||
/// One request's outcome as a client sees it.
|
||||
#[derive(Clone, Debug, Serialize)]
|
||||
pub(crate) struct Observation {
|
||||
/// The S3 operation, named as in the `s3s::S3` trait (`GetObject`).
|
||||
pub op: &'static str,
|
||||
/// Which request of the script this was (`normal`, `negative`, or a setup label).
|
||||
pub case: &'static str,
|
||||
/// HTTP status; 0 when no response arrived.
|
||||
pub status: u16,
|
||||
/// The S3 error code, or the SDK's error kind when there was no service error.
|
||||
pub error_code: Option<String>,
|
||||
/// Allowlisted headers only, lowercase names, normalized values.
|
||||
pub headers: BTreeMap<String, String>,
|
||||
/// Typed values decoded from the response body, never free text.
|
||||
pub fields: BTreeMap<String, String>,
|
||||
/// Diagnostic only; never compared.
|
||||
#[serde(skip)]
|
||||
pub message: Option<String>,
|
||||
}
|
||||
|
||||
impl Observation {
|
||||
/// Builds an observation, keeping only the allowlisted headers.
|
||||
pub(crate) fn new<'h>(
|
||||
op: &'static str,
|
||||
case: &'static str,
|
||||
status: u16,
|
||||
error_code: Option<String>,
|
||||
headers: impl IntoIterator<Item = (&'h str, &'h str)>,
|
||||
) -> Self {
|
||||
let headers = headers
|
||||
.into_iter()
|
||||
.filter_map(|(name, value)| compared_header(name).map(|name| (name, value)))
|
||||
.map(|(name, value)| {
|
||||
let value = normalized_header_value(&name, value);
|
||||
(name, value)
|
||||
})
|
||||
.collect();
|
||||
Self {
|
||||
op,
|
||||
case,
|
||||
status,
|
||||
error_code,
|
||||
headers,
|
||||
fields: BTreeMap::new(),
|
||||
message: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn with_field(mut self, name: &str, value: impl Into<String>) -> Self {
|
||||
self.fields.insert(name.to_owned(), value.into());
|
||||
self
|
||||
}
|
||||
|
||||
pub(crate) fn is_success(&self) -> bool {
|
||||
(200..300).contains(&self.status)
|
||||
}
|
||||
}
|
||||
|
||||
/// One field on which the two stacks answered differently.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
|
||||
pub(crate) struct Difference {
|
||||
pub op: String,
|
||||
pub case: String,
|
||||
pub field: String,
|
||||
pub a: String,
|
||||
pub b: String,
|
||||
}
|
||||
|
||||
/// Every difference between two transcripts of the same script, matched by `(op, case)`.
|
||||
pub(crate) fn compare(a: &[Observation], b: &[Observation]) -> Vec<Difference> {
|
||||
let mut differences = Vec::new();
|
||||
let (index_a, duplicates_a) = index(a);
|
||||
let (index_b, duplicates_b) = index(b);
|
||||
for (op, case) in duplicates_a.union(&duplicates_b) {
|
||||
let count = |side: &[Observation]| {
|
||||
side.iter()
|
||||
.filter(|observation| (observation.op, observation.case) == (*op, *case))
|
||||
.count()
|
||||
.to_string()
|
||||
};
|
||||
differences.push(difference(op, case, "duplicate_case", count(a), count(b)));
|
||||
}
|
||||
let keys: BTreeSet<_> = index_a.keys().chain(index_b.keys()).copied().collect();
|
||||
for key @ (op, case) in keys {
|
||||
match (index_a.get(&key), index_b.get(&key)) {
|
||||
(Some(a), Some(b)) => compare_one(a, b, &mut differences),
|
||||
(a, b) => {
|
||||
let presence = |side: Option<&&Observation>| if side.is_some() { "present" } else { ABSENT }.to_owned();
|
||||
differences.push(difference(op, case, "observation", presence(a), presence(b)));
|
||||
}
|
||||
}
|
||||
}
|
||||
differences
|
||||
}
|
||||
|
||||
type Key = (&'static str, &'static str);
|
||||
|
||||
fn index(side: &[Observation]) -> (BTreeMap<Key, &Observation>, BTreeSet<Key>) {
|
||||
let mut index = BTreeMap::new();
|
||||
let mut duplicates = BTreeSet::new();
|
||||
for observation in side {
|
||||
let key = (observation.op, observation.case);
|
||||
if index.insert(key, observation).is_some() {
|
||||
duplicates.insert(key);
|
||||
}
|
||||
}
|
||||
(index, duplicates)
|
||||
}
|
||||
|
||||
fn difference(op: &str, case: &str, field: &str, a: String, b: String) -> Difference {
|
||||
Difference {
|
||||
op: op.to_owned(),
|
||||
case: case.to_owned(),
|
||||
field: field.to_owned(),
|
||||
a,
|
||||
b,
|
||||
}
|
||||
}
|
||||
|
||||
fn compare_one(a: &Observation, b: &Observation, differences: &mut Vec<Difference>) {
|
||||
let mut field = |name: String, value_a: Option<&str>, value_b: Option<&str>| {
|
||||
if value_a != value_b {
|
||||
let show = |value: Option<&str>| value.unwrap_or(ABSENT).to_owned();
|
||||
differences.push(difference(a.op, a.case, &name, show(value_a), show(value_b)));
|
||||
}
|
||||
};
|
||||
field(
|
||||
"status".to_owned(),
|
||||
Some(a.status.to_string().as_str()),
|
||||
Some(b.status.to_string().as_str()),
|
||||
);
|
||||
field("error_code".to_owned(), a.error_code.as_deref(), b.error_code.as_deref());
|
||||
let headers: BTreeSet<_> = a.headers.keys().chain(b.headers.keys()).collect();
|
||||
for name in headers {
|
||||
field(
|
||||
format!("header.{name}"),
|
||||
a.headers.get(name).map(String::as_str),
|
||||
b.headers.get(name).map(String::as_str),
|
||||
);
|
||||
}
|
||||
let fields: BTreeSet<_> = a.fields.keys().chain(b.fields.keys()).collect();
|
||||
for name in fields {
|
||||
field(
|
||||
format!("body.{name}"),
|
||||
a.fields.get(name).map(String::as_str),
|
||||
b.fields.get(name).map(String::as_str),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Operations observed on both sides.
|
||||
pub(crate) fn compared_ops(a: &[Observation], b: &[Observation]) -> BTreeSet<&'static str> {
|
||||
let ops_b: BTreeSet<_> = b.iter().map(|observation| observation.op).collect();
|
||||
a.iter()
|
||||
.map(|observation| observation.op)
|
||||
.filter(|op| ops_b.contains(op))
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// `(op, case)` pairs observed on both sides.
|
||||
pub(crate) fn compared_cases(a: &[Observation], b: &[Observation]) -> usize {
|
||||
let cases_b: BTreeSet<_> = b.iter().map(|observation| (observation.op, observation.case)).collect();
|
||||
a.iter()
|
||||
.map(|observation| (observation.op, observation.case))
|
||||
.collect::<BTreeSet<_>>()
|
||||
.intersection(&cases_b)
|
||||
.count()
|
||||
}
|
||||
|
||||
/// The JSON report written for every run.
|
||||
#[derive(Debug, Serialize)]
|
||||
pub(crate) struct Report {
|
||||
pub schema: u32,
|
||||
pub run: String,
|
||||
pub stack_a: String,
|
||||
pub stack_b: String,
|
||||
pub region_a: Option<String>,
|
||||
pub region_b: Option<String>,
|
||||
pub compared_ops: usize,
|
||||
pub compared_cases: usize,
|
||||
pub differences: Vec<Difference>,
|
||||
}
|
||||
|
||||
impl Report {
|
||||
/// Writes the report as `<directory>/<run>.json` and returns its path.
|
||||
pub(crate) fn write_to(&self, directory: &Path) -> std::io::Result<PathBuf> {
|
||||
std::fs::create_dir_all(directory)?;
|
||||
let path = directory.join(format!("{}.json", self.run));
|
||||
let json = serde_json::to_vec_pretty(self).map_err(std::io::Error::other)?;
|
||||
std::fs::write(&path, json)?;
|
||||
Ok(path)
|
||||
}
|
||||
}
|
||||
|
||||
/// One line per difference, for a failing assertion.
|
||||
pub(crate) fn render(differences: &[Difference]) -> String {
|
||||
differences
|
||||
.iter()
|
||||
.map(|d| format!("({}, {}, {}, a={:?}, b={:?})", d.op, d.case, d.field, d.a, d.b))
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n")
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn ok(op: &'static str) -> Observation {
|
||||
Observation::new(op, "normal", 200, None, [("content-type", "application/xml")])
|
||||
}
|
||||
|
||||
fn fields(differences: &[Difference]) -> Vec<&str> {
|
||||
differences.iter().map(|d| d.field.as_str()).collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn identical_transcripts_have_no_difference() {
|
||||
let a = vec![
|
||||
ok("HeadBucket"),
|
||||
ok("GetBucketLocation").with_field("location_constraint", "us-east-1"),
|
||||
];
|
||||
let b = a.clone();
|
||||
assert_eq!(compare(&a, &b), Vec::new());
|
||||
assert_eq!(compared_ops(&a, &b).len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_status_difference_is_reported() {
|
||||
let a = vec![ok("PutBucketAcl")];
|
||||
let b = vec![Observation::new(
|
||||
"PutBucketAcl",
|
||||
"normal",
|
||||
403,
|
||||
None,
|
||||
[("content-type", "application/xml")],
|
||||
)];
|
||||
assert_eq!(
|
||||
compare(&a, &b),
|
||||
vec![Difference {
|
||||
op: "PutBucketAcl".into(),
|
||||
case: "normal".into(),
|
||||
field: "status".into(),
|
||||
a: "200".into(),
|
||||
b: "403".into(),
|
||||
}]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_error_code_difference_is_reported() {
|
||||
let a = vec![Observation::new("GetObject", "negative", 404, Some("NoSuchKey".into()), [])];
|
||||
let b = vec![Observation::new(
|
||||
"GetObject",
|
||||
"negative",
|
||||
404,
|
||||
Some("NoSuchBucket".into()),
|
||||
[],
|
||||
)];
|
||||
let differences = compare(&a, &b);
|
||||
assert_eq!(fields(&differences), ["error_code"]);
|
||||
assert_eq!((differences[0].a.as_str(), differences[0].b.as_str()), ("NoSuchKey", "NoSuchBucket"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_error_code_on_one_side_only_is_reported() {
|
||||
let a = vec![Observation::new("HeadBucket", "negative", 404, Some("NotFound".into()), [])];
|
||||
let b = vec![Observation::new("HeadBucket", "negative", 404, None, [])];
|
||||
let differences = compare(&a, &b);
|
||||
assert_eq!(fields(&differences), ["error_code"]);
|
||||
assert_eq!(differences[0].b, ABSENT);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_allowlisted_header_difference_is_reported() {
|
||||
let a = vec![Observation::new("HeadObject", "normal", 200, None, [("ETag", "\"aa\"")])];
|
||||
let b = vec![Observation::new("HeadObject", "normal", 200, None, [("etag", "\"bb\"")])];
|
||||
assert_eq!(fields(&compare(&a, &b)), ["header.etag"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_header_sent_by_one_side_only_is_reported() {
|
||||
let a = vec![Observation::new(
|
||||
"DeleteObject",
|
||||
"normal",
|
||||
204,
|
||||
None,
|
||||
[("x-amz-delete-marker", "true")],
|
||||
)];
|
||||
let b = vec![Observation::new("DeleteObject", "normal", 204, None, [])];
|
||||
let differences = compare(&a, &b);
|
||||
assert_eq!(fields(&differences), ["header.x-amz-delete-marker"]);
|
||||
assert_eq!((differences[0].a.as_str(), differences[0].b.as_str()), ("true", ABSENT));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_server_side_encryption_header_is_compared() {
|
||||
let header = "x-amz-server-side-encryption-customer-key-md5";
|
||||
let a = vec![Observation::new("PutObject", "sse-c", 200, None, [(header, "one")])];
|
||||
let b = vec![Observation::new("PutObject", "sse-c", 200, None, [(header, "two")])];
|
||||
assert_eq!(fields(&compare(&a, &b)), [format!("header.{header}")]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn headers_outside_the_allowlist_are_not_compared() {
|
||||
let a = vec![Observation::new(
|
||||
"ListBuckets",
|
||||
"normal",
|
||||
200,
|
||||
None,
|
||||
[
|
||||
("date", "Mon"),
|
||||
("x-amz-request-id", "1"),
|
||||
("server", "a"),
|
||||
("content-length", "10"),
|
||||
],
|
||||
)];
|
||||
let b = vec![Observation::new(
|
||||
"ListBuckets",
|
||||
"normal",
|
||||
200,
|
||||
None,
|
||||
[
|
||||
("date", "Tue"),
|
||||
("x-amz-request-id", "2"),
|
||||
("server", "b"),
|
||||
("content-length", "11"),
|
||||
],
|
||||
)];
|
||||
assert_eq!(compare(&a, &b), Vec::new());
|
||||
assert!(a[0].headers.is_empty(), "non-allowlisted headers were kept: {:?}", a[0].headers);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn distinct_version_ids_are_not_a_difference() {
|
||||
let a = vec![Observation::new(
|
||||
"PutObject",
|
||||
"normal",
|
||||
200,
|
||||
None,
|
||||
[("x-amz-version-id", "0b1c")],
|
||||
)];
|
||||
let b = vec![Observation::new(
|
||||
"PutObject",
|
||||
"normal",
|
||||
200,
|
||||
None,
|
||||
[("x-amz-version-id", "9f8e")],
|
||||
)];
|
||||
assert_eq!(compare(&a, &b), Vec::new());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_null_version_id_against_a_real_one_is_reported() {
|
||||
let a = vec![Observation::new(
|
||||
"PutObject",
|
||||
"normal",
|
||||
200,
|
||||
None,
|
||||
[("x-amz-version-id", "null")],
|
||||
)];
|
||||
let b = vec![Observation::new(
|
||||
"PutObject",
|
||||
"normal",
|
||||
200,
|
||||
None,
|
||||
[("x-amz-version-id", "9f8e")],
|
||||
)];
|
||||
assert_eq!(fields(&compare(&a, &b)), ["header.x-amz-version-id"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_body_field_difference_is_reported() {
|
||||
let a = vec![ok("GetBucketLocation").with_field("location_constraint", "us-east-1")];
|
||||
let b = vec![ok("GetBucketLocation").with_field("location_constraint", "eu-west-1")];
|
||||
let differences = compare(&a, &b);
|
||||
assert_eq!(fields(&differences), ["body.location_constraint"]);
|
||||
assert_eq!((differences[0].a.as_str(), differences[0].b.as_str()), ("us-east-1", "eu-west-1"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_body_field_on_one_side_only_is_reported() {
|
||||
let a = vec![ok("ListObjectsV2").with_field("keys", "[]")];
|
||||
let b = vec![ok("ListObjectsV2")];
|
||||
assert_eq!(fields(&compare(&a, &b)), ["body.keys"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn error_messages_are_never_compared() {
|
||||
let mut a = Observation::new("GetObject", "negative", 404, Some("NoSuchKey".into()), []);
|
||||
let mut b = a.clone();
|
||||
a.message = Some("The specified key does not exist.".into());
|
||||
b.message = Some("Object not found".into());
|
||||
assert_eq!(compare(&[a], &[b]), Vec::new());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_case_observed_on_one_side_only_is_reported() {
|
||||
let a = vec![ok("HeadBucket"), ok("ListBuckets")];
|
||||
let b = vec![ok("HeadBucket")];
|
||||
let differences = compare(&a, &b);
|
||||
assert_eq!(fields(&differences), ["observation"]);
|
||||
assert_eq!(differences[0].op, "ListBuckets");
|
||||
assert_eq!(compared_ops(&a, &b).into_iter().collect::<Vec<_>>(), ["HeadBucket"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_case_recorded_twice_is_reported() {
|
||||
let a = vec![ok("HeadBucket"), ok("HeadBucket")];
|
||||
let b = vec![ok("HeadBucket")];
|
||||
assert_eq!(fields(&compare(&a, &b)), ["duplicate_case"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compared_ops_counts_operations_not_cases() {
|
||||
let negative = Observation::new("HeadBucket", "negative", 404, Some("NotFound".into()), []);
|
||||
let a = vec![ok("HeadBucket"), negative, ok("ListBuckets")];
|
||||
let b = a.clone();
|
||||
assert_eq!(compared_ops(&a, &b).len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_report_lists_each_difference_as_op_field_a_b() {
|
||||
let directory = std::env::temp_dir().join(format!("dual-stack-report-{}", uuid::Uuid::new_v4()));
|
||||
let report = Report {
|
||||
schema: 1,
|
||||
run: "unit".into(),
|
||||
stack_a: "legacy".into(),
|
||||
stack_b: "legacy".into(),
|
||||
region_a: None,
|
||||
region_b: Some("eu-west-1".into()),
|
||||
compared_ops: 1,
|
||||
compared_cases: 1,
|
||||
differences: compare(
|
||||
&[ok("GetBucketLocation").with_field("location_constraint", "us-east-1")],
|
||||
&[ok("GetBucketLocation").with_field("location_constraint", "eu-west-1")],
|
||||
),
|
||||
};
|
||||
let path = report.write_to(&directory).expect("write the report");
|
||||
let json: serde_json::Value = serde_json::from_slice(&std::fs::read(&path).expect("read the report")).expect("parse");
|
||||
let _ = std::fs::remove_dir_all(&directory);
|
||||
assert_eq!(path.file_name().and_then(|name| name.to_str()), Some("unit.json"));
|
||||
assert_eq!(json["compared_ops"], 1);
|
||||
assert_eq!(
|
||||
json["differences"],
|
||||
serde_json::json!([{
|
||||
"op": "GetBucketLocation",
|
||||
"case": "normal",
|
||||
"field": "body.location_constraint",
|
||||
"a": "us-east-1",
|
||||
"b": "eu-west-1",
|
||||
}])
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,199 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Dual-stack differential: one `rustfs` binary, two processes, one SDK script (rustfs/backlog#2740).
|
||||
//!
|
||||
//! Stack A always runs `RUSTFS_S3_STACK=legacy`. Stack B runs the value of `RUSTFS_E2E_STACK_B`
|
||||
//! (default `legacy` until the gateway stack is assembled), and `RUSTFS_E2E_STACK_B_REGION`, when
|
||||
//! set, starts B in another region so that a difference can be injected by hand. Both servers
|
||||
//! answer the script in `scenarios`, which covers every operation of `impl S3 for FS`; `compare`
|
||||
//! reports each `(op, field, a, b)` on which the answers differ, and every run writes its report
|
||||
//! to `<target>/dual-stack/<run>.json`. Two processes, not two embedded servers: bucket metadata
|
||||
//! is process-global, so two stacks in one process would read each other's buckets.
|
||||
|
||||
mod compare;
|
||||
mod scenarios;
|
||||
|
||||
use crate::common::{RustFSTestEnvironment, init_logging, workspace_root};
|
||||
use compare::{Difference, Observation, Report, compare, compared_cases, compared_ops, render};
|
||||
use std::path::PathBuf;
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
type BoxError = Box<dyn std::error::Error + Send + Sync>;
|
||||
|
||||
const STACK_A: &str = "legacy";
|
||||
const ENV_STACK_B: &str = "RUSTFS_E2E_STACK_B";
|
||||
const ENV_STACK_B_REGION: &str = "RUSTFS_E2E_STACK_B_REGION";
|
||||
/// Becomes `gateway` once rustfs/backlog#2734 T2.5 assembles the gateway stack.
|
||||
const DEFAULT_STACK_B: &str = "legacy";
|
||||
/// The region the negative control gives stack A explicitly, so an inherited `RUSTFS_REGION`
|
||||
/// cannot erase the injected difference. Not `us-east-1`: that region answers GetBucketLocation
|
||||
/// with an empty LocationConstraint and re-creating an owned bucket with 200, so it would not
|
||||
/// report its own name and would add CreateBucket differences to the control.
|
||||
const CONTROL_REGION_A: &str = "us-west-2";
|
||||
const CONTROL_REGION_B: &str = "eu-west-1";
|
||||
/// `normal` cases the legacy stack must answer with a 2xx. The comparison alone cannot tell a
|
||||
/// script that exercises real behavior from one in which both stacks fail the same way. Five of
|
||||
/// the 73 fail on legacy by design of this script: PutBucketReplication (no registered remote
|
||||
/// target), GetBucketReplication (so no configuration), PutBucketNotificationConfiguration (no
|
||||
/// notification target configured), RestoreObject (the object is not archived) and
|
||||
/// GetObjectTorrent (legacy answers `NoSuchKey`).
|
||||
const MIN_NORMAL_SUCCESSES: usize = 68;
|
||||
|
||||
struct PairConfig {
|
||||
stack_b: String,
|
||||
region_a: Option<String>,
|
||||
region_b: Option<String>,
|
||||
}
|
||||
|
||||
impl PairConfig {
|
||||
fn from_env() -> Self {
|
||||
Self {
|
||||
stack_b: std::env::var(ENV_STACK_B).unwrap_or_else(|_| DEFAULT_STACK_B.to_owned()),
|
||||
region_a: None,
|
||||
region_b: std::env::var(ENV_STACK_B_REGION).ok(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Starts one server on `stack`, overriding its region only when asked.
|
||||
async fn start_stack(stack: &str, region: Option<&str>) -> Result<RustFSTestEnvironment, BoxError> {
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
let mut vars = vec![("RUSTFS_S3_STACK", stack)];
|
||||
if let Some(region) = region {
|
||||
vars.push(("RUSTFS_REGION", region));
|
||||
}
|
||||
env.start_rustfs_server_without_cleanup_with_env(&vars).await?;
|
||||
Ok(env)
|
||||
}
|
||||
|
||||
/// Starts stack A and stack B side by side, each with its own port and data directory.
|
||||
async fn spawn_pair(config: &PairConfig) -> Result<(RustFSTestEnvironment, RustFSTestEnvironment), BoxError> {
|
||||
tokio::try_join!(
|
||||
start_stack(STACK_A, config.region_a.as_deref()),
|
||||
start_stack(&config.stack_b, config.region_b.as_deref())
|
||||
)
|
||||
}
|
||||
|
||||
fn report_directory() -> PathBuf {
|
||||
let target = std::env::var_os("CARGO_TARGET_DIR").map_or_else(|| PathBuf::from("target"), PathBuf::from);
|
||||
let target = if target.is_absolute() {
|
||||
target
|
||||
} else {
|
||||
workspace_root().join(target)
|
||||
};
|
||||
target.join("dual-stack")
|
||||
}
|
||||
|
||||
struct Run {
|
||||
report: Report,
|
||||
path: PathBuf,
|
||||
a: Vec<Observation>,
|
||||
}
|
||||
|
||||
/// Runs the script against both stacks concurrently and writes the report.
|
||||
async fn run_differential(label: &str, config: &PairConfig) -> Run {
|
||||
let (server_a, server_b) = spawn_pair(config).await.expect("start stack A and stack B");
|
||||
let (a, b) = tokio::join!(Box::pin(scenarios::run(&server_a)), Box::pin(scenarios::run(&server_b)));
|
||||
let millis = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.map_or(0, |elapsed| elapsed.as_millis());
|
||||
let report = Report {
|
||||
schema: 1,
|
||||
run: format!("{label}-{millis}-{}", std::process::id()),
|
||||
stack_a: STACK_A.to_owned(),
|
||||
stack_b: config.stack_b.clone(),
|
||||
region_a: config.region_a.clone(),
|
||||
region_b: config.region_b.clone(),
|
||||
compared_ops: compared_ops(&a, &b).len(),
|
||||
compared_cases: compared_cases(&a, &b),
|
||||
differences: compare(&a, &b),
|
||||
};
|
||||
let path = report.write_to(&report_directory()).expect("write the dual-stack report");
|
||||
Run { report, path, a }
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn both_stacks_answer_every_operation_identically() {
|
||||
init_logging();
|
||||
let config = PairConfig::from_env();
|
||||
let run = run_differential("self-comparison", &config).await;
|
||||
let report = &run.report;
|
||||
|
||||
let mut observed: Vec<_> = run.a.iter().map(|observation| observation.op).collect();
|
||||
observed.sort_unstable();
|
||||
observed.dedup();
|
||||
assert_eq!(
|
||||
observed,
|
||||
scenarios::OPERATIONS,
|
||||
"the script must drive exactly the operations of impl S3 for FS"
|
||||
);
|
||||
assert_eq!(report.compared_ops, scenarios::OPERATIONS.len(), "report: {}", run.path.display());
|
||||
assert!(
|
||||
report.differences.is_empty(),
|
||||
"stack {} and stack {} differ on {} of {} cases (report: {}):\n{}",
|
||||
report.stack_a,
|
||||
report.stack_b,
|
||||
report.differences.len(),
|
||||
report.compared_cases,
|
||||
run.path.display(),
|
||||
render(&report.differences)
|
||||
);
|
||||
|
||||
let not_4xx: Vec<_> = run
|
||||
.a
|
||||
.iter()
|
||||
.filter(|observation| observation.case == "negative" && !(400..500).contains(&observation.status))
|
||||
.collect();
|
||||
assert!(not_4xx.is_empty(), "negative cases must fail with a 4xx on {STACK_A}: {not_4xx:#?}");
|
||||
let successes = run
|
||||
.a
|
||||
.iter()
|
||||
.filter(|observation| observation.case == "normal" && observation.is_success())
|
||||
.count();
|
||||
assert!(
|
||||
successes >= MIN_NORMAL_SUCCESSES,
|
||||
"only {successes} normal cases succeeded on {STACK_A}: {:#?}",
|
||||
run.a
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn differential_detects_an_injected_difference() {
|
||||
init_logging();
|
||||
let config = PairConfig {
|
||||
stack_b: STACK_A.to_owned(),
|
||||
region_a: Some(CONTROL_REGION_A.to_owned()),
|
||||
region_b: Some(CONTROL_REGION_B.to_owned()),
|
||||
};
|
||||
let run = run_differential("injected-region", &config).await;
|
||||
let expected = Difference {
|
||||
op: "GetBucketLocation".to_owned(),
|
||||
case: "normal".to_owned(),
|
||||
field: "body.location_constraint".to_owned(),
|
||||
a: CONTROL_REGION_A.to_owned(),
|
||||
b: CONTROL_REGION_B.to_owned(),
|
||||
};
|
||||
assert!(
|
||||
run.report.differences.contains(&expected),
|
||||
"B started in {CONTROL_REGION_B} but GetBucketLocation was not reported (report: {}):\n{}",
|
||||
run.path.display(),
|
||||
render(&run.report.differences)
|
||||
);
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1700,15 +1700,6 @@ mod tests {
|
||||
);
|
||||
|
||||
let mut cluster = RustFSTestClusterEnvironment::with_topology(topology.cluster_topology()).await?;
|
||||
let bucket = "heal-restart-during-rebuild";
|
||||
let mrf_fallback_scope = if !background_enabled {
|
||||
let path = PathBuf::from(&cluster.temp_dir).join("skip-outage-mrf-fallback");
|
||||
std::fs::write(&path, format!("{bucket}/cluster/written-while-node-down-"))?;
|
||||
cluster.set_env("RUSTFS_E2E_SKIP_MRF_FALLBACK_PATH", path.to_string_lossy().into_owned());
|
||||
Some(path)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
cluster.set_env("RUSTFS_UNSAFE_BYPASS_DISK_CHECK", "true");
|
||||
cluster.set_env("RUSTFS_HEAL_ENABLED", "true");
|
||||
if let Some(storage_class) = evidence_case.and_then(|case| case.storage_class_standard) {
|
||||
@@ -1961,21 +1952,6 @@ mod tests {
|
||||
}
|
||||
};
|
||||
|
||||
if let Some(path) = &mrf_fallback_scope {
|
||||
let observed = path.with_extension("observed");
|
||||
let expected = format!("{bucket}/{outage_key}");
|
||||
let deadline = Instant::now() + Duration::from_secs(30);
|
||||
loop {
|
||||
if tokio::fs::read_to_string(&observed).await.ok().as_deref() == Some(expected.as_str()) {
|
||||
break;
|
||||
}
|
||||
if Instant::now() >= deadline {
|
||||
return Err("outage repair did not reach the isolated best-effort fallback before target rejoin".into());
|
||||
}
|
||||
sleep(Duration::from_millis(50)).await;
|
||||
}
|
||||
}
|
||||
|
||||
assert!(
|
||||
!outage_target_manifest_required
|
||||
|| target_set_peer_drives
|
||||
@@ -2069,12 +2045,7 @@ mod tests {
|
||||
} else {
|
||||
cluster_heal_is_idle(&recovered)
|
||||
};
|
||||
let pool_ready = if ready && background_enabled && !outage_write_deferred_until_rejoin {
|
||||
all_nodes_pool_metadata_readiness(&cluster, recovery_deadline).await.0
|
||||
} else {
|
||||
true
|
||||
};
|
||||
if ready && pool_ready {
|
||||
if ready {
|
||||
break;
|
||||
}
|
||||
if Instant::now() >= recovery_deadline {
|
||||
@@ -2571,10 +2542,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
if let Some(path) = &mrf_fallback_scope {
|
||||
std::fs::remove_file(path)?;
|
||||
}
|
||||
|
||||
if let Some(cycle_end) = scanner_cycle_floor {
|
||||
wait_for_scanner_cycle_after(&cluster, cycle_end).await?;
|
||||
}
|
||||
|
||||
@@ -31,6 +31,12 @@ pub mod chaos;
|
||||
#[cfg(test)]
|
||||
pub mod fake_s3_target;
|
||||
|
||||
// Dual-stack differential (rustfs/backlog#2740): one binary, two processes on
|
||||
// different RUSTFS_S3_STACK values, the same SDK script for every S3 operation,
|
||||
// compared on status, error code, allowlisted headers and typed body fields.
|
||||
#[cfg(test)]
|
||||
mod dual_stack_differential;
|
||||
|
||||
// On-demand migration (backlog#2147): shared two-server environment, admin
|
||||
// wrappers, and the harness self-test (backlog#2151). Behavior scenarios are
|
||||
// added by later ODM tasks.
|
||||
|
||||
@@ -32,7 +32,9 @@
|
||||
//! Advisory: <https://github.com/rustfs/rustfs/security/advisories/GHSA-3p3x-734c-h5vx>
|
||||
|
||||
use crate::common::rustfs_binary_path_with_features;
|
||||
use crate::common::{AdminTransport, admin_add_canned_policy_via, admin_attach_user_policy_via, admin_create_user_via};
|
||||
use crate::common::{
|
||||
AdminTransport, admin_add_canned_policy_via, admin_attach_user_policy_via, admin_create_user_via, signed_s3_request,
|
||||
};
|
||||
use crate::common::{FAST_DATA_USAGE_SCANNER_ENV, admin_request};
|
||||
use crate::protocols::test_env::{DEFAULT_ACCESS_KEY, DEFAULT_SECRET_KEY, ProtocolTestEnvironment};
|
||||
use anyhow::Result;
|
||||
@@ -774,6 +776,81 @@ async fn test_webdav_core_operations_direct() -> Result<()> {
|
||||
test_webdav_core_operations().await
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_webdav_content_type_directory_marker() -> Result<()> {
|
||||
let env = ProtocolTestEnvironment::new().map_err(|e| anyhow::anyhow!("{}", e))?;
|
||||
let admin_base_url = format!("http://{}", S3_TEST_ADDRESS);
|
||||
let binary_path = rustfs_binary_path_with_features(Some("webdav"));
|
||||
let mut server_process = Command::new(&binary_path)
|
||||
.arg("--address")
|
||||
.arg(S3_TEST_ADDRESS)
|
||||
.env("RUSTFS_CONSOLE_ENABLE", "false")
|
||||
.env("RUSTFS_WEBDAV_ENABLE", "true")
|
||||
.env("RUSTFS_WEBDAV_ADDRESS", WEBDAV_ADDRESS)
|
||||
.env("RUSTFS_WEBDAV_TLS_ENABLED", "false")
|
||||
.arg(&env.temp_dir)
|
||||
.spawn()?;
|
||||
|
||||
let result = async {
|
||||
ProtocolTestEnvironment::wait_for_port_ready(WEBDAV_PORT, 30)
|
||||
.await
|
||||
.map_err(|e| anyhow::anyhow!("{}", e))?;
|
||||
|
||||
let client = create_client();
|
||||
let bucket_name = "webdav-content-type-bucket";
|
||||
let directory_name = "content-type-directory";
|
||||
let webdav_base_url = format!("http://{}", WEBDAV_ADDRESS);
|
||||
let mkcol_bucket = client
|
||||
.request(
|
||||
reqwest::Method::from_bytes(b"MKCOL").expect("MKCOL is a valid HTTP method"),
|
||||
format!("{}/{}", webdav_base_url, bucket_name),
|
||||
)
|
||||
.header("Authorization", basic_auth_header())
|
||||
.send()
|
||||
.await?;
|
||||
anyhow::ensure!(
|
||||
mkcol_bucket.status().is_success() || mkcol_bucket.status().as_u16() == 201,
|
||||
"MKCOL bucket should succeed, got: {}",
|
||||
mkcol_bucket.status()
|
||||
);
|
||||
|
||||
let mkcol_directory = client
|
||||
.request(
|
||||
reqwest::Method::from_bytes(b"MKCOL").expect("MKCOL is a valid HTTP method"),
|
||||
format!("{}/{}/{}", webdav_base_url, bucket_name, directory_name),
|
||||
)
|
||||
.header("Authorization", basic_auth_header())
|
||||
.send()
|
||||
.await?;
|
||||
anyhow::ensure!(
|
||||
mkcol_directory.status().is_success() || mkcol_directory.status().as_u16() == 201,
|
||||
"MKCOL directory should succeed, got: {}",
|
||||
mkcol_directory.status()
|
||||
);
|
||||
|
||||
let head_url = format!("{}/{}/{}/", admin_base_url, bucket_name, directory_name);
|
||||
let head = signed_s3_request(http::Method::HEAD, &head_url, None, None, DEFAULT_ACCESS_KEY, DEFAULT_SECRET_KEY)
|
||||
.await
|
||||
.map_err(|error| anyhow::anyhow!("{error}"))?;
|
||||
anyhow::ensure!(head.status().is_success(), "HEAD directory marker should succeed, got: {}", head.status());
|
||||
let content_type = head
|
||||
.headers()
|
||||
.get("content-type")
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.unwrap_or_default();
|
||||
anyhow::ensure!(
|
||||
content_type == "application/x-directory",
|
||||
"WebDAV directory marker must preserve its content type, got: {content_type}"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
.await;
|
||||
|
||||
let _ = server_process.kill().await;
|
||||
let _ = server_process.wait().await;
|
||||
result
|
||||
}
|
||||
|
||||
/// Extract the numeric value of the first occurrence of a DAV property in a
|
||||
/// PROPFIND multistatus body. Namespace-prefix agnostic; a self-closing
|
||||
/// (unreported) property yields None.
|
||||
|
||||
@@ -31,8 +31,8 @@ use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use aws_sdk_s3::operation::list_object_versions::ListObjectVersionsOutput;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{
|
||||
BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, Delete, DeleteMarkerEntry, ObjectIdentifier, ObjectVersion,
|
||||
ServerSideEncryption, VersioningConfiguration,
|
||||
BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, DeleteMarkerEntry, ObjectVersion, ServerSideEncryption,
|
||||
VersioningConfiguration,
|
||||
};
|
||||
use base64_simd::STANDARD as BASE64_STANDARD;
|
||||
use bytes::Bytes;
|
||||
@@ -3729,8 +3729,6 @@ async fn test_bucket_replication_converges_delete_marker_and_version_purge() ->
|
||||
init_logging();
|
||||
|
||||
let mut source_env = RustFSTestEnvironment::new().await?;
|
||||
let source_log = format!("{}/source.log", source_env.temp_dir);
|
||||
source_env.capture_log_path = Some(source_log.clone());
|
||||
let mut source_env_vars = replication_fast_env();
|
||||
source_env_vars.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
|
||||
source_env.start_rustfs_server_with_env(vec![], &source_env_vars).await?;
|
||||
@@ -3793,66 +3791,19 @@ async fn test_bucket_replication_converges_delete_marker_and_version_purge() ->
|
||||
"target did not preserve the source delete-marker version ID"
|
||||
);
|
||||
|
||||
let wait_for_source_writeback = || async {
|
||||
let metrics_url = format!("{}/{source_bucket}?replication-metrics", source_env.url);
|
||||
timeout(Duration::from_secs(30), async {
|
||||
loop {
|
||||
let response = signed_request(
|
||||
http::Method::GET,
|
||||
&metrics_url,
|
||||
&source_env.access_key,
|
||||
&source_env.secret_key,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
let metrics: serde_json::Value = response.error_for_status()?.json().await?;
|
||||
let queued = metrics["queued"]["curr"]["count"]
|
||||
.as_f64()
|
||||
.ok_or("missing replication queue count")?;
|
||||
if metrics["provider_available"] == true && queued == 0.0 {
|
||||
return Ok::<(), Box<dyn Error + Send + Sync>>(());
|
||||
}
|
||||
sleep(Duration::from_millis(50)).await;
|
||||
}
|
||||
})
|
||||
.await?
|
||||
};
|
||||
wait_for_source_writeback().await?;
|
||||
|
||||
let purged = source_client
|
||||
.delete_objects()
|
||||
source_client
|
||||
.delete_object()
|
||||
.bucket(source_bucket)
|
||||
.delete(
|
||||
Delete::builder()
|
||||
.objects(
|
||||
ObjectIdentifier::builder()
|
||||
.key(object_key)
|
||||
.version_id(&purged_version_id)
|
||||
.build()?,
|
||||
)
|
||||
.objects(
|
||||
ObjectIdentifier::builder()
|
||||
.key(object_key)
|
||||
.version_id(delete_marker_version_id)
|
||||
.build()?,
|
||||
)
|
||||
.build()?,
|
||||
)
|
||||
.key(object_key)
|
||||
.version_id(&purged_version_id)
|
||||
.send()
|
||||
.await?;
|
||||
assert!(purged.errors().is_empty(), "batch version purge failed: {:?}", purged.errors());
|
||||
assert_eq!(purged.deleted().len(), 2, "both the data version and marker must be purged");
|
||||
assert_replication_converged(&source_client, source_bucket, &target_client, target_bucket).await?;
|
||||
let target_state = list_replication_state(&target_client, target_bucket).await?;
|
||||
assert!(
|
||||
target_state.iter().all(|entry| entry.version_id != purged_version_id),
|
||||
"target retained the explicitly purged object version"
|
||||
);
|
||||
assert!(
|
||||
target_state.iter().all(|entry| entry.version_id != delete_marker_version_id),
|
||||
"target retained the explicitly purged delete marker"
|
||||
);
|
||||
assert!(
|
||||
target_state
|
||||
.iter()
|
||||
@@ -3868,19 +3819,6 @@ async fn test_bucket_replication_converges_delete_marker_and_version_purge() ->
|
||||
.await?;
|
||||
assert_eq!(retained.body.collect().await?.into_bytes().as_ref(), b"versioned replication payload v2");
|
||||
|
||||
// Wait for the source workers to finish their final writeback, which runs
|
||||
// after the remote purge has already become visible (rustfs#9121).
|
||||
wait_for_source_writeback().await?;
|
||||
let logs = fs::read_to_string(source_log).await?;
|
||||
let writeback_failures: Vec<_> = logs
|
||||
.lines()
|
||||
.filter(|line| line.contains("apply_replication_delete_state"))
|
||||
.collect();
|
||||
assert!(
|
||||
writeback_failures.is_empty(),
|
||||
"a completed batch purge must not report a source writeback failure: {writeback_failures:?}"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
@@ -28,8 +28,7 @@ use hyper_util::{
|
||||
use prost::Message as _;
|
||||
use rustfs_protos::proto_gen::node_service::{
|
||||
BatchGenerallyLockRequest, DeleteRequest, GenerallyLockRequest, PreparePartTransactionResponse, ReadVersionRequest,
|
||||
RecoverMultipartSourceRequest, RenameDataResponse, ScannerScopedDirtyUsageAckRequest, SettlePartTransactionRequest,
|
||||
SettlePartTransactionResponse,
|
||||
RecoverMultipartSourceRequest, RenameDataResponse, SettlePartTransactionRequest, SettlePartTransactionResponse,
|
||||
};
|
||||
use std::{
|
||||
collections::HashSet,
|
||||
@@ -50,9 +49,6 @@ use tokio::{
|
||||
type ProxyBody = BoxBody<Bytes, io::Error>;
|
||||
type ProxyClient = Client<HttpConnector, ProxyBody>;
|
||||
|
||||
mod scoped_ack;
|
||||
pub(crate) use scoped_ack::{ScopedAckFault, ScopedAckQueue, ScopedAckScope};
|
||||
|
||||
#[derive(Clone, Copy, PartialEq, Eq)]
|
||||
enum Disposition {
|
||||
Hold,
|
||||
@@ -125,8 +121,7 @@ impl RpcPause {
|
||||
self.state.blocked.load(Ordering::SeqCst)
|
||||
}
|
||||
|
||||
/// Selected RPCs with successful HTTP/gRPC response headers, excluding
|
||||
/// discarded packets. This does not prove a successful gRPC trailer or ACK apply.
|
||||
/// Selected RPCs with successful HTTP/gRPC headers, excluding discarded packets.
|
||||
pub(crate) fn forwarded(&self) -> usize {
|
||||
self.state.forwarded.load(Ordering::SeqCst)
|
||||
}
|
||||
@@ -238,7 +233,6 @@ pub(crate) struct RpcFaultProxy {
|
||||
address: SocketAddr,
|
||||
slot: PauseSlot,
|
||||
lock_slot: PauseSlot,
|
||||
scoped_ack: scoped_ack::QueueSlot,
|
||||
task: JoinHandle<()>,
|
||||
late_requests: LateRequests,
|
||||
}
|
||||
@@ -251,8 +245,6 @@ impl RpcFaultProxy {
|
||||
let active = slot.clone();
|
||||
let lock_slot: PauseSlot = Arc::new(Mutex::new(None));
|
||||
let active_lock = lock_slot.clone();
|
||||
let scoped_ack = scoped_ack::empty_slot();
|
||||
let active_ack = scoped_ack.clone();
|
||||
let late_requests: LateRequests = Arc::new(Mutex::new(Some(Vec::new())));
|
||||
let late = late_requests.clone();
|
||||
let task = tokio::spawn(async move {
|
||||
@@ -267,7 +259,6 @@ impl RpcFaultProxy {
|
||||
let h2 = h2.clone();
|
||||
let active = active.clone();
|
||||
let active_lock = active_lock.clone();
|
||||
let active_ack = active_ack.clone();
|
||||
let late = late.clone();
|
||||
connections.spawn(async move {
|
||||
let service = service_fn(move |request: Request<Incoming>| {
|
||||
@@ -281,7 +272,6 @@ impl RpcFaultProxy {
|
||||
None
|
||||
};
|
||||
let pause = owner_pause.or_else(|| active.lock().expect("RPC pause slot").clone());
|
||||
let ack = active_ack.lock().expect("scoped ACK queue slot").clone();
|
||||
let retain = pause.as_ref().is_some_and(|pause| {
|
||||
pause.retain_after_disconnect
|
||||
&& request.uri().path().strip_prefix("/node_service.NodeService/")
|
||||
@@ -290,11 +280,11 @@ impl RpcFaultProxy {
|
||||
let late = late.clone();
|
||||
async move {
|
||||
if !retain {
|
||||
return forward_request(request, client, target, pause, ack).await;
|
||||
return forward_request(request, client, target, pause).await;
|
||||
}
|
||||
// This opt-in fault owns the packet after client cancellation.
|
||||
// Proxy teardown still aborts every retained request.
|
||||
let task = tokio::spawn(forward_request(request, client, target, pause, ack));
|
||||
let task = tokio::spawn(forward_request(request, client, target, pause));
|
||||
{
|
||||
let mut requests = late.lock().expect("late RPC ownership");
|
||||
if let Some(requests) = requests.as_mut() {
|
||||
@@ -319,7 +309,6 @@ impl RpcFaultProxy {
|
||||
address,
|
||||
slot,
|
||||
lock_slot,
|
||||
scoped_ack,
|
||||
task,
|
||||
late_requests,
|
||||
})
|
||||
@@ -329,11 +318,6 @@ impl RpcFaultProxy {
|
||||
self.address
|
||||
}
|
||||
|
||||
/// Retain only genuine scoped ACKs for one receiver/session and bucket set.
|
||||
pub(crate) fn scoped_ack_queue(&self, scope: ScopedAckScope, capacity: usize) -> io::Result<ScopedAckQueue> {
|
||||
ScopedAckQueue::arm(self.scoped_ack.clone(), scope, capacity)
|
||||
}
|
||||
|
||||
/// Pause at most limit requests to the named unary methods; other streams
|
||||
/// on the same HTTP/2 connection continue forwarding.
|
||||
pub(crate) fn pause(&self, methods: &[&'static str], limit: usize, scope: Option<&str>) -> io::Result<RpcPause> {
|
||||
@@ -485,9 +469,6 @@ fn unary_payload(body: &[u8]) -> io::Result<&[u8]> {
|
||||
}
|
||||
|
||||
fn scope_matches(method: &str, body: &[u8], scope: &[u8]) -> io::Result<bool> {
|
||||
if method == "ScannerScopedDirtyUsageAck" {
|
||||
return scoped_ack_matches(body, Some(scope));
|
||||
}
|
||||
if method == "RecoverMultipartSource" {
|
||||
let request = RecoverMultipartSourceRequest::decode(unary_payload(body)?).map_err(io::Error::other)?;
|
||||
if request.protocol_version != u32::from(MULTIPART_SOURCE_PROTOCOL_VERSION) || request.path.as_bytes() != scope {
|
||||
@@ -544,9 +525,7 @@ fn scoped_lock_owners(method: &str, body: &[u8], scope: &[u8]) -> io::Result<Vec
|
||||
let args = match method {
|
||||
"Lock" => vec![GenerallyLockRequest::decode(payload).map_err(io::Error::other)?.args],
|
||||
"LockBatch" => BatchGenerallyLockRequest::decode(payload).map_err(io::Error::other)?.args,
|
||||
_ => {
|
||||
return Err(io::Error::new(io::ErrorKind::InvalidInput, "lock observer requires a lock method"));
|
||||
}
|
||||
_ => return Err(io::Error::new(io::ErrorKind::InvalidInput, "lock observer requires a lock method")),
|
||||
};
|
||||
args.into_iter()
|
||||
.filter(|args| args.as_bytes().windows(scope.len()).any(|window| window == scope))
|
||||
@@ -561,39 +540,18 @@ fn scoped_lock_owners(method: &str, body: &[u8], scope: &[u8]) -> io::Result<Vec
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn scoped_ack_matches(body: &[u8], scope: Option<&[u8]>) -> io::Result<bool> {
|
||||
let request = ScannerScopedDirtyUsageAckRequest::decode(unary_payload(body)?).map_err(io::Error::other)?;
|
||||
Ok(!request.probe_only && scope.is_none_or(|scope| request.entries.iter().any(|entry| entry.bucket.as_bytes() == scope)))
|
||||
}
|
||||
|
||||
async fn forward_request(
|
||||
request: Request<Incoming>,
|
||||
client: ProxyClient,
|
||||
target: SocketAddr,
|
||||
pause: Option<Arc<PauseState>>,
|
||||
ack: Option<Arc<scoped_ack::QueueState>>,
|
||||
) -> io::Result<hyper::Response<ProxyBody>> {
|
||||
let request = request.map(|body| body.map_err(io::Error::other).boxed());
|
||||
let request = if let Some(ack) = ack
|
||||
&& request.uri().path() == "/node_service.ScannerControlService/ScannerScopedDirtyUsageAck"
|
||||
{
|
||||
match scoped_ack::intercept(request, client.clone(), target, ack).await? {
|
||||
scoped_ack::Interception::Pass(request) => request,
|
||||
scoped_ack::Interception::Handled(response) => return Ok(response),
|
||||
}
|
||||
} else {
|
||||
request
|
||||
};
|
||||
let (parts, body) = request.into_parts();
|
||||
let mut body = body;
|
||||
let (mut parts, body) = request.into_parts();
|
||||
let mut body = body.map_err(io::Error::other).boxed();
|
||||
let mut observed = None;
|
||||
let mut inspection = None;
|
||||
if let Some(pause) = pause {
|
||||
let method = parts
|
||||
.uri
|
||||
.path()
|
||||
.strip_prefix("/node_service.NodeService/")
|
||||
.or_else(|| parts.uri.path().strip_prefix("/node_service.ScannerControlService/"));
|
||||
let method = parts.uri.path().strip_prefix("/node_service.NodeService/");
|
||||
if let Some(method) = method.filter(|name| pause.methods.contains(name)) {
|
||||
let mut route = None;
|
||||
let mut lock_owners = Vec::new();
|
||||
@@ -641,16 +599,6 @@ async fn forward_request(
|
||||
.with_trailers(async move { trailers.map(Ok::<_, io::Error>) })
|
||||
.boxed();
|
||||
matches
|
||||
} else if method == "ScannerScopedDirtyUsageAck" {
|
||||
let collected = Limited::new(body, 1024 * 1024).collect().await.map_err(io::Error::other)?;
|
||||
let trailers = collected.trailers().cloned();
|
||||
let bytes = collected.to_bytes();
|
||||
let matches = scoped_ack_matches(&bytes, None)?;
|
||||
body = Full::new(bytes)
|
||||
.map_err(|never| match never {})
|
||||
.with_trailers(async move { trailers.map(Ok::<_, io::Error>) })
|
||||
.boxed();
|
||||
matches
|
||||
} else {
|
||||
true
|
||||
};
|
||||
@@ -694,7 +642,24 @@ async fn forward_request(
|
||||
}
|
||||
}
|
||||
}
|
||||
let response = forward_to_backend(Request::from_parts(parts, body), client, target).await?;
|
||||
if !parts.headers.contains_key(http::header::HOST)
|
||||
&& let Some(authority) = parts.uri.authority()
|
||||
{
|
||||
parts.headers.insert(
|
||||
http::header::HOST,
|
||||
http::HeaderValue::from_str(authority.as_str()).map_err(io::Error::other)?,
|
||||
);
|
||||
}
|
||||
parts.uri = http::Uri::builder()
|
||||
.scheme("http")
|
||||
.authority(target.to_string())
|
||||
.path_and_query(parts.uri.path_and_query().map_or("/", |value| value.as_str()))
|
||||
.build()
|
||||
.map_err(io::Error::other)?;
|
||||
let response = client
|
||||
.request(Request::from_parts(parts, body))
|
||||
.await
|
||||
.map_err(io::Error::other)?;
|
||||
if let Some((pause, disk)) = inspection
|
||||
&& response.status().is_success()
|
||||
&& response.headers().get("grpc-status").is_some_and(|status| status == "12")
|
||||
@@ -747,32 +712,6 @@ async fn forward_request(
|
||||
Ok(response.map(|body| body.map_err(io::Error::other).boxed()))
|
||||
}
|
||||
|
||||
async fn forward_to_backend(
|
||||
request: Request<ProxyBody>,
|
||||
client: ProxyClient,
|
||||
target: SocketAddr,
|
||||
) -> io::Result<hyper::Response<Incoming>> {
|
||||
let (mut parts, body) = request.into_parts();
|
||||
if !parts.headers.contains_key(http::header::HOST)
|
||||
&& let Some(authority) = parts.uri.authority()
|
||||
{
|
||||
parts.headers.insert(
|
||||
http::header::HOST,
|
||||
http::HeaderValue::from_str(authority.as_str()).map_err(io::Error::other)?,
|
||||
);
|
||||
}
|
||||
parts.uri = http::Uri::builder()
|
||||
.scheme("http")
|
||||
.authority(target.to_string())
|
||||
.path_and_query(parts.uri.path_and_query().map_or("/", |value| value.as_str()))
|
||||
.build()
|
||||
.map_err(io::Error::other)?;
|
||||
client
|
||||
.request(Request::from_parts(parts, body))
|
||||
.await
|
||||
.map_err(io::Error::other)
|
||||
}
|
||||
|
||||
impl Drop for RpcFaultProxy {
|
||||
fn drop(&mut self) {
|
||||
if let Some(requests) = self.late_requests.lock().expect("late RPC ownership").take() {
|
||||
@@ -792,7 +731,6 @@ mod tests {
|
||||
|
||||
struct Backend {
|
||||
received: Arc<AtomicUsize>,
|
||||
auth_headers: Arc<Mutex<Vec<(String, String)>>>,
|
||||
address: SocketAddr,
|
||||
task: JoinHandle<()>,
|
||||
}
|
||||
@@ -816,8 +754,6 @@ mod tests {
|
||||
let address = listener.local_addr()?;
|
||||
let received = Arc::new(AtomicUsize::new(0));
|
||||
let received_task = received.clone();
|
||||
let auth_headers = Arc::new(Mutex::new(Vec::new()));
|
||||
let backend_auth = auth_headers.clone();
|
||||
let task = tokio::spawn(async move {
|
||||
let mut children = JoinSet::new();
|
||||
loop {
|
||||
@@ -825,12 +761,10 @@ mod tests {
|
||||
accepted = listener.accept() => {
|
||||
let Ok((socket, _)) = accepted else { break };
|
||||
let received = received_task.clone();
|
||||
let backend_auth = backend_auth.clone();
|
||||
let reply = reply.clone();
|
||||
children.spawn(async move {
|
||||
let service = service_fn(move |request: Request<Incoming>| {
|
||||
let received = received.clone();
|
||||
let backend_auth = backend_auth.clone();
|
||||
let reply = reply.clone();
|
||||
async move {
|
||||
received.fetch_add(1, Ordering::SeqCst);
|
||||
@@ -839,10 +773,6 @@ mod tests {
|
||||
http::HeaderValue::from_str(authority.as_str()).expect("valid HTTP authority")
|
||||
})
|
||||
});
|
||||
if let (Some(nonce), Some(signature)) = (request.headers().get("x-rustfs-rpc-nonce"), request.headers().get("x-rustfs-rpc-signature-v2")) {
|
||||
backend_auth.lock().expect("backend auth header observations").push((
|
||||
nonce.to_str().expect("test nonce").to_string(), signature.to_str().expect("test signature").to_string()));
|
||||
}
|
||||
let body = request.into_body().collect().await?.to_bytes();
|
||||
let body = reply.unwrap_or(body);
|
||||
let mut trailers = http::HeaderMap::new();
|
||||
@@ -865,12 +795,7 @@ mod tests {
|
||||
}
|
||||
}
|
||||
});
|
||||
Ok(Backend {
|
||||
received,
|
||||
auth_headers,
|
||||
address,
|
||||
task,
|
||||
})
|
||||
Ok(Backend { received, address, task })
|
||||
}
|
||||
|
||||
async fn call(client: &ProxyClient, address: SocketAddr, method: &str) -> io::Result<hyper::Response<Incoming>> {
|
||||
@@ -884,14 +809,9 @@ mod tests {
|
||||
bytes: Bytes,
|
||||
) -> io::Result<hyper::Response<Incoming>> {
|
||||
let body: ProxyBody = Full::new(bytes).map_err(|never| match never {}).boxed();
|
||||
let service = if method == "ScannerScopedDirtyUsageAck" {
|
||||
"ScannerControlService"
|
||||
} else {
|
||||
"NodeService"
|
||||
};
|
||||
let request = Request::builder()
|
||||
.method("POST")
|
||||
.uri(format!("http://{address}/node_service.{service}/{method}"))
|
||||
.uri(format!("http://{address}/node_service.NodeService/{method}"))
|
||||
.header(http::header::TE, "trailers")
|
||||
.body(body)
|
||||
.map_err(io::Error::other)?;
|
||||
@@ -906,353 +826,6 @@ mod tests {
|
||||
frame.into()
|
||||
}
|
||||
|
||||
const ACK_OWNER: &str = "11111111-1111-1111-1111-111111111111";
|
||||
const ACK_INSTANCE: &str = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
|
||||
|
||||
fn ack_scope() -> ScopedAckScope {
|
||||
ScopedAckScope {
|
||||
nonce: uuid::Uuid::new_v4().to_string(),
|
||||
owner_id: ACK_OWNER.to_string(),
|
||||
instance_id: ACK_INSTANCE.to_string(),
|
||||
buckets: std::collections::BTreeSet::from(["cold-bucket".to_string()]),
|
||||
}
|
||||
}
|
||||
|
||||
fn ack_request(bucket: &str, generation: u64) -> ScannerScopedDirtyUsageAckRequest {
|
||||
ScannerScopedDirtyUsageAckRequest {
|
||||
challenge: vec![7; 16].into(),
|
||||
protocol_version: 1,
|
||||
owner_id: ACK_OWNER.to_string(),
|
||||
instance_id: ACK_INSTANCE.to_string(),
|
||||
scope: 1,
|
||||
probe_only: false,
|
||||
entries: vec![rustfs_protos::proto_gen::node_service::ScannerScopedDirtyUsageEntry {
|
||||
bucket: bucket.to_string(),
|
||||
bucket_incarnation: vec![4; 16].into(),
|
||||
generation,
|
||||
}],
|
||||
}
|
||||
}
|
||||
|
||||
fn ack_frame(request: &ScannerScopedDirtyUsageAckRequest) -> Bytes {
|
||||
let payload = request.encode_to_vec();
|
||||
let mut frame = vec![0];
|
||||
frame.extend_from_slice(&u32::try_from(payload.len()).expect("bounded test frame").to_be_bytes());
|
||||
frame.extend_from_slice(&payload);
|
||||
Bytes::from(frame)
|
||||
}
|
||||
|
||||
fn ack_hash(bytes: &[u8]) -> String {
|
||||
use sha2::Digest as _;
|
||||
hex_simd::encode_to_string(&sha2::Sha256::digest(bytes)[..], hex_simd::AsciiCase::Lower)
|
||||
}
|
||||
|
||||
// Transport-only fixtures: the distributed tests supply production signatures.
|
||||
async fn call_ack_frame(
|
||||
client: &ProxyClient,
|
||||
address: SocketAddr,
|
||||
frame: Bytes,
|
||||
nonce: &str,
|
||||
) -> io::Result<hyper::Response<Incoming>> {
|
||||
let request = Request::builder()
|
||||
.method("POST")
|
||||
.uri(format!("http://{address}/node_service.ScannerControlService/ScannerScopedDirtyUsageAck"))
|
||||
.header(http::header::TE, "trailers")
|
||||
.header("x-rustfs-rpc-nonce", nonce)
|
||||
.header("x-rustfs-rpc-signature-v2", format!("opaque-signature-{nonce}"))
|
||||
.body(Full::new(frame).map_err(|never| match never {}).boxed())
|
||||
.map_err(io::Error::other)?;
|
||||
client.request(request).await.map_err(io::Error::other)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scoped_ack_queue_loss_preserves_backend_boundary() {
|
||||
tokio::time::timeout(Duration::from_secs(10), async {
|
||||
for fault in [ScopedAckFault::RequestLoss, ScopedAckFault::ResponseLoss] {
|
||||
let backend = backend().await.expect("backend");
|
||||
let proxy = RpcFaultProxy::start_on("127.0.0.1:0".parse().expect("loopback"), backend.address)
|
||||
.await
|
||||
.expect("proxy");
|
||||
let queue = proxy.scoped_ack_queue(ack_scope(), 1).expect("ACK queue");
|
||||
let client: ProxyClient = Client::builder(TokioExecutor::new()).http2_only(true).build_http();
|
||||
let frame = ack_frame(&ack_request("cold-bucket", 7));
|
||||
let address = proxy.local_addr();
|
||||
let body = frame.clone();
|
||||
let caller = tokio::spawn(async move { call_ack_frame(&client, address, body, "loss").await });
|
||||
queue.wait_for_packets(1).await.expect("captured ACK");
|
||||
assert_eq!(backend.received.load(Ordering::SeqCst), 0, "retention must precede backend dispatch");
|
||||
queue.settle(0, fault).expect("settle actual retained packet");
|
||||
assert!(
|
||||
caller.await.expect("caller task").is_err(),
|
||||
"the caller must observe the injected transport failure"
|
||||
);
|
||||
let receipt = queue.wait_for_completion(0).await.expect("complete fault receipt");
|
||||
let response_loss = fault == ScopedAckFault::ResponseLoss;
|
||||
assert_eq!(receipt["forwarded"], response_loss);
|
||||
assert_eq!(receipt["backend_complete"], response_loss);
|
||||
assert_eq!(backend.received.load(Ordering::SeqCst), usize::from(response_loss));
|
||||
if response_loss {
|
||||
assert_eq!(receipt["outcome"], "response_lost");
|
||||
assert_eq!(receipt["response"]["grpc_status"], "0", "status exists only in real backend trailers");
|
||||
assert_eq!(receipt["response"]["bytes"], frame.len());
|
||||
assert_eq!(receipt["response"]["sha256"], ack_hash(&frame));
|
||||
assert!(
|
||||
receipt["response"]["trailers_sha256"]
|
||||
.as_str()
|
||||
.is_some_and(|value| value.len() == 64)
|
||||
);
|
||||
} else {
|
||||
assert_eq!(receipt["outcome"], "request_lost");
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("bounded ACK loss checks");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scoped_ack_queue_keeps_cancelled_packets_and_releases_out_of_order() {
|
||||
tokio::time::timeout(Duration::from_secs(10), async {
|
||||
let backend = backend().await.expect("backend");
|
||||
let proxy = RpcFaultProxy::start_on("127.0.0.1:0".parse().expect("loopback"), backend.address)
|
||||
.await
|
||||
.expect("proxy");
|
||||
let queue = proxy.scoped_ack_queue(ack_scope(), 2).expect("two-packet ACK queue");
|
||||
let client: ProxyClient = Client::builder(TokioExecutor::new()).http2_only(true).build_http();
|
||||
let first = ack_frame(&ack_request("cold-bucket", 7));
|
||||
let second = ack_frame(&ack_request("cold-bucket", 9));
|
||||
let address = proxy.local_addr();
|
||||
let old_client = client.clone();
|
||||
let old_body = first.clone();
|
||||
let old_caller = tokio::spawn(async move { call_ack_frame(&old_client, address, old_body, "first").await });
|
||||
queue.wait_for_packets(1).await.expect("first retained packet");
|
||||
old_caller.abort();
|
||||
assert!(old_caller.await.expect_err("cancel original caller").is_cancelled());
|
||||
let new_body = second.clone();
|
||||
let new_caller = tokio::spawn(async move { call_ack_frame(&client, address, new_body, "second").await });
|
||||
queue.wait_for_packets(2).await.expect("second retained packet");
|
||||
assert_eq!(backend.received.load(Ordering::SeqCst), 0);
|
||||
queue.settle(1, ScopedAckFault::Forward).expect("release newer packet first");
|
||||
let response = new_caller
|
||||
.await
|
||||
.expect("new caller task")
|
||||
.expect("new response")
|
||||
.into_body()
|
||||
.collect()
|
||||
.await
|
||||
.expect("complete body");
|
||||
assert_eq!(response.trailers().expect("response trailers")["x-proof"], "unchanged-proof");
|
||||
assert_eq!(response.to_bytes(), second);
|
||||
queue.wait_for_completion(1).await.expect("new packet complete");
|
||||
queue
|
||||
.settle(0, ScopedAckFault::Forward)
|
||||
.expect("release old packet after its caller disappeared");
|
||||
let old = queue.wait_for_completion(0).await.expect("detached old packet complete");
|
||||
assert_eq!(old["backend_complete"], true);
|
||||
assert_eq!(old["request_sha256"], ack_hash(&first));
|
||||
assert_eq!(old["response"]["sha256"], ack_hash(&first));
|
||||
assert_eq!(backend.received.load(Ordering::SeqCst), 2);
|
||||
assert_eq!(
|
||||
*backend.auth_headers.lock().expect("backend header observations"),
|
||||
vec![
|
||||
("second".to_string(), "opaque-signature-second".to_string()),
|
||||
("first".to_string(), "opaque-signature-first".to_string())
|
||||
]
|
||||
);
|
||||
assert!(
|
||||
queue.settle(0, ScopedAckFault::Forward).is_err(),
|
||||
"one original packet may only be forwarded once"
|
||||
);
|
||||
})
|
||||
.await
|
||||
.expect("bounded detached retention check");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scoped_ack_queue_bounds_capacity_and_excludes_other_scopes() {
|
||||
tokio::time::timeout(Duration::from_secs(10), async {
|
||||
let backend = backend().await.expect("backend");
|
||||
let proxy = RpcFaultProxy::start_on("127.0.0.1:0".parse().expect("loopback"), backend.address)
|
||||
.await
|
||||
.expect("proxy");
|
||||
assert!(proxy.scoped_ack_queue(ack_scope(), 0).is_err());
|
||||
assert!(proxy.scoped_ack_queue(ack_scope(), 5).is_err());
|
||||
let mut invalid = ack_scope();
|
||||
invalid.nonce.clear();
|
||||
assert!(proxy.scoped_ack_queue(invalid, 1).is_err());
|
||||
let mut scope = ack_scope();
|
||||
scope.buckets.insert("cold-bucket-z".to_string());
|
||||
let queue = proxy.scoped_ack_queue(scope, 1).expect("bounded ACK queue");
|
||||
let client: ProxyClient = Client::builder(TokioExecutor::new()).http2_only(true).build_http();
|
||||
let mut probe = ack_request("cold-bucket", 7);
|
||||
probe.probe_only = true;
|
||||
let mut other_peer = ack_request("cold-bucket", 7);
|
||||
other_peer.owner_id = uuid::Uuid::new_v4().to_string();
|
||||
let mut other_instance = ack_request("cold-bucket", 7);
|
||||
other_instance.instance_id = "b".repeat(32);
|
||||
let mut invalid_challenge = ack_request("cold-bucket", 7);
|
||||
invalid_challenge.challenge = vec![7; 15].into();
|
||||
let mut noncanonical_entries = ack_request("cold-bucket-z", 7);
|
||||
noncanonical_entries.entries.extend(ack_request("cold-bucket", 7).entries);
|
||||
let unselected = [
|
||||
probe,
|
||||
other_peer,
|
||||
other_instance,
|
||||
ack_request("cold-bucket-other", 7),
|
||||
ack_request("cold-bucket", u64::MAX),
|
||||
invalid_challenge,
|
||||
noncanonical_entries,
|
||||
];
|
||||
let unselected_count = unselected.len();
|
||||
for (index, request) in unselected.into_iter().enumerate() {
|
||||
let body = ack_frame(&request);
|
||||
let response = call_ack_frame(&client, proxy.local_addr(), body.clone(), "unselected")
|
||||
.await
|
||||
.expect("unselected ACK progresses");
|
||||
assert_eq!(response.into_body().collect().await.expect("unselected body").to_bytes(), body);
|
||||
assert!(queue.receipts().is_empty(), "unselected or protocol-invalid ACK must not consume a slot");
|
||||
assert_eq!(backend.received.load(Ordering::SeqCst), index + 1);
|
||||
}
|
||||
assert!(queue.receipts().is_empty());
|
||||
let address = proxy.local_addr();
|
||||
let held_client = client.clone();
|
||||
let valid_frame = ack_frame(&ack_request("cold-bucket", 7));
|
||||
let held_body = valid_frame.clone();
|
||||
let held = tokio::spawn(async move { call_ack_frame(&held_client, address, held_body, "held").await });
|
||||
queue.wait_for_packets(1).await.expect("one matching packet");
|
||||
assert!(!held.is_finished(), "the subsequent valid ACK must be retained");
|
||||
assert_eq!(queue.receipts()[0]["request_sha256"], ack_hash(&valid_frame));
|
||||
assert_eq!(backend.received.load(Ordering::SeqCst), unselected_count);
|
||||
let extra = call_ack_frame(&client, address, ack_frame(&ack_request("cold-bucket", 8)), "beyond-capacity")
|
||||
.await
|
||||
.expect("capacity overflow passes without retention");
|
||||
extra.into_body().collect().await.expect("overflow response");
|
||||
assert_eq!(queue.receipts().len(), 1);
|
||||
assert_eq!(backend.received.load(Ordering::SeqCst), unselected_count + 1);
|
||||
drop(queue);
|
||||
assert!(
|
||||
held.await.expect("held caller").is_err(),
|
||||
"dropping the session must cancel its retained packet"
|
||||
);
|
||||
assert_eq!(
|
||||
backend.received.load(Ordering::SeqCst),
|
||||
unselected_count + 1,
|
||||
"session drop must never forward the held packet"
|
||||
);
|
||||
})
|
||||
.await
|
||||
.expect("bounded scope and lifecycle checks");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scoped_ack_queue_expiry_never_forwards_retained_packet() {
|
||||
let (backend, _proxy, queue, caller) = tokio::time::timeout(Duration::from_secs(10), async {
|
||||
let backend = backend().await.expect("backend");
|
||||
let proxy = RpcFaultProxy::start_on("127.0.0.1:0".parse().expect("loopback"), backend.address)
|
||||
.await
|
||||
.expect("proxy");
|
||||
let queue = proxy.scoped_ack_queue(ack_scope(), 1).expect("ACK queue");
|
||||
let client: ProxyClient = Client::builder(TokioExecutor::new()).http2_only(true).build_http();
|
||||
let address = proxy.local_addr();
|
||||
let caller = tokio::spawn(async move {
|
||||
call_ack_frame(&client, address, ack_frame(&ack_request("cold-bucket", 7)), "expires").await
|
||||
});
|
||||
queue.wait_for_packets(1).await.expect("retained packet");
|
||||
(backend, proxy, queue, caller)
|
||||
})
|
||||
.await
|
||||
.expect("bounded queue setup");
|
||||
tokio::time::pause();
|
||||
tokio::time::advance(Duration::from_secs(120)).await;
|
||||
tokio::time::resume();
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_secs(10), caller)
|
||||
.await
|
||||
.expect("expiry completes caller")
|
||||
.expect("caller task")
|
||||
.is_err()
|
||||
);
|
||||
let receipt = queue.receipts().into_iter().next().expect("expired receipt");
|
||||
assert_eq!(receipt["completed"], true);
|
||||
assert_eq!(receipt["forwarded"], false);
|
||||
assert_eq!(receipt["backend_complete"], false);
|
||||
assert_eq!(backend.received.load(Ordering::SeqCst), 0);
|
||||
assert_eq!(
|
||||
queue
|
||||
.settle(0, ScopedAckFault::Forward)
|
||||
.expect_err("expired packet cannot be released")
|
||||
.kind(),
|
||||
io::ErrorKind::TimedOut
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scanner_ack_fault_matches_exact_bucket_and_excludes_probes() {
|
||||
use rustfs_protos::proto_gen::node_service::ScannerScopedDirtyUsageEntry;
|
||||
|
||||
tokio::time::timeout(Duration::from_secs(10), async {
|
||||
let backend = backend().await.expect("backend listener");
|
||||
let proxy = RpcFaultProxy::start_on("127.0.0.1:0".parse().expect("loopback"), backend.address)
|
||||
.await
|
||||
.expect("proxy listener");
|
||||
let client: ProxyClient = Client::builder(TokioExecutor::new()).http2_only(true).build_http();
|
||||
let pause = proxy
|
||||
.pause(&["ScannerScopedDirtyUsageAck"], 1, Some("cold-bucket"))
|
||||
.expect("scoped ACK fault");
|
||||
let frame = |bucket: &str, probe_only: bool| {
|
||||
let payload = ScannerScopedDirtyUsageAckRequest {
|
||||
challenge: vec![3; 16].into(),
|
||||
protocol_version: 1,
|
||||
owner_id: "11111111-1111-1111-1111-111111111111".to_string(),
|
||||
instance_id: "a".repeat(32),
|
||||
scope: 1,
|
||||
probe_only,
|
||||
entries: vec![ScannerScopedDirtyUsageEntry {
|
||||
bucket: bucket.to_string(),
|
||||
bucket_incarnation: vec![4; 16].into(),
|
||||
generation: 17,
|
||||
}],
|
||||
}
|
||||
.encode_to_vec();
|
||||
let mut frame = vec![0];
|
||||
frame.extend_from_slice(&u32::try_from(payload.len()).expect("small ACK frame").to_be_bytes());
|
||||
frame.extend_from_slice(&payload);
|
||||
Bytes::from(frame)
|
||||
};
|
||||
for bytes in [frame("cold-bucket", true), frame("cold-bucket-other", false)] {
|
||||
let response = call_body(&client, proxy.local_addr(), "ScannerScopedDirtyUsageAck", bytes.clone())
|
||||
.await
|
||||
.expect("probe and other bucket must progress");
|
||||
assert_eq!(response.into_body().collect().await.expect("unselected response").to_bytes(), bytes);
|
||||
}
|
||||
assert_eq!(pause.hits(), 0);
|
||||
let signed_bytes = frame("cold-bucket", false);
|
||||
let task_bytes = signed_bytes.clone();
|
||||
let address = proxy.local_addr();
|
||||
let blocked =
|
||||
tokio::spawn(async move { call_body(&client, address, "ScannerScopedDirtyUsageAck", task_bytes).await });
|
||||
pause
|
||||
.wait_for_hits(1)
|
||||
.await
|
||||
.expect("real ScannerControlService method must reach the fault");
|
||||
assert!(!blocked.is_finished());
|
||||
assert_eq!(backend.received.load(Ordering::SeqCst), 2, "held ACK must not apply at the backend");
|
||||
pause.release();
|
||||
let collected = blocked
|
||||
.await
|
||||
.expect("ACK task")
|
||||
.expect("released ACK")
|
||||
.into_body()
|
||||
.collect()
|
||||
.await
|
||||
.expect("ACK body and trailers");
|
||||
assert_eq!(collected.trailers().expect("ACK trailers")["x-proof"], "unchanged-proof");
|
||||
assert_eq!(collected.to_bytes(), signed_bytes, "proxy must preserve the complete signed ACK frame");
|
||||
})
|
||||
.await
|
||||
.expect("scanner ACK fault deadline");
|
||||
}
|
||||
|
||||
fn native_abort_request(path: &str) -> RecoverMultipartSourceRequest {
|
||||
use crate::storage_api::multipart_source_rpc::{
|
||||
MultipartAbortCertificate, MultipartAbortProof, MultipartAbortProofKind, MultipartAbortRound, MultipartSourcePhase,
|
||||
@@ -2146,4 +1719,3 @@ mod tests {
|
||||
);
|
||||
}
|
||||
}
|
||||
// event-drill: isolated harness-only change
|
||||
|
||||
@@ -1,404 +0,0 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
//! Bounded retention of original scoped ACK frames, independent of caller cancellation.
|
||||
|
||||
use super::{ProxyBody, ProxyClient, forward_to_backend, unary_payload};
|
||||
use bytes::Bytes;
|
||||
use http_body_util::{BodyExt as _, Full, Limited};
|
||||
use hyper::{Request, Response};
|
||||
use prost::Message as _;
|
||||
use rustfs_protos::proto_gen::node_service::{ScannerScopedDirtyUsageAckRequest, ScannerScopedDirtyUsageAckResponse};
|
||||
use rustfs_protos::scoped_dirty_usage::validate_scoped_dirty_usage_request;
|
||||
use serde_json::{Value, json};
|
||||
use sha2::{Digest as _, Sha256};
|
||||
use std::{
|
||||
collections::BTreeSet,
|
||||
io,
|
||||
net::SocketAddr,
|
||||
sync::{
|
||||
Arc, Mutex,
|
||||
atomic::{AtomicBool, Ordering},
|
||||
},
|
||||
time::Duration,
|
||||
};
|
||||
use tokio::{
|
||||
sync::{Notify, oneshot, watch},
|
||||
task::AbortHandle,
|
||||
time::Instant,
|
||||
};
|
||||
|
||||
const MAX_RETAINED: usize = 4;
|
||||
const MAX_BYTES: usize = 1024 * 1024;
|
||||
const RETENTION: Duration = Duration::from_secs(120);
|
||||
|
||||
pub(crate) struct ScopedAckScope {
|
||||
pub nonce: String,
|
||||
pub owner_id: String,
|
||||
pub instance_id: String,
|
||||
pub buckets: BTreeSet<String>,
|
||||
}
|
||||
|
||||
impl ScopedAckScope {
|
||||
fn validate(&self) -> io::Result<()> {
|
||||
let valid_uuid = |value: &str| uuid::Uuid::parse_str(value).is_ok_and(|value| !value.is_nil());
|
||||
if !valid_uuid(&self.nonce)
|
||||
|| !valid_uuid(&self.owner_id)
|
||||
|| self.instance_id.is_empty()
|
||||
|| self.instance_id.len() > 128
|
||||
|| self.buckets.is_empty()
|
||||
|| self.buckets.len() > 16
|
||||
|| self.buckets.iter().any(|bucket| bucket.is_empty() || bucket.len() > 63)
|
||||
{
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::InvalidInput,
|
||||
"scoped ACK queue needs a bounded session, receiver and bucket scope",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn matches(&self, request: &ScannerScopedDirtyUsageAckRequest) -> bool {
|
||||
!request.probe_only
|
||||
&& validate_scoped_dirty_usage_request(request).is_ok()
|
||||
&& request.owner_id == self.owner_id
|
||||
&& request.instance_id == self.instance_id
|
||||
&& request.entries.iter().all(|entry| self.buckets.contains(&entry.bucket))
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(crate) enum ScopedAckFault {
|
||||
Forward,
|
||||
RequestLoss,
|
||||
ResponseLoss,
|
||||
}
|
||||
|
||||
struct Packet {
|
||||
action: watch::Sender<Option<ScopedAckFault>>,
|
||||
receipt: Mutex<Value>,
|
||||
task: Mutex<Option<AbortHandle>>,
|
||||
}
|
||||
|
||||
pub(super) struct QueueState {
|
||||
scope: ScopedAckScope,
|
||||
capacity: usize,
|
||||
started: Instant,
|
||||
deadline: Instant,
|
||||
closed: AtomicBool,
|
||||
// The packet inventory lock is released before any packet lock or await.
|
||||
packets: Mutex<Vec<Arc<Packet>>>,
|
||||
changed: Notify,
|
||||
}
|
||||
|
||||
pub(super) type QueueSlot = Arc<Mutex<Option<Arc<QueueState>>>>;
|
||||
|
||||
pub(super) fn empty_slot() -> QueueSlot {
|
||||
Arc::new(Mutex::new(None))
|
||||
}
|
||||
|
||||
pub(crate) struct ScopedAckQueue {
|
||||
state: Arc<QueueState>,
|
||||
slot: QueueSlot,
|
||||
}
|
||||
|
||||
impl ScopedAckQueue {
|
||||
pub(super) fn arm(slot: QueueSlot, scope: ScopedAckScope, capacity: usize) -> io::Result<Self> {
|
||||
scope.validate()?;
|
||||
if !(1..=MAX_RETAINED).contains(&capacity) {
|
||||
return Err(io::Error::new(io::ErrorKind::InvalidInput, "scoped ACK capacity must be 1..=4"));
|
||||
}
|
||||
let mut active = slot.lock().expect("scoped ACK queue slot");
|
||||
if active.is_some() {
|
||||
return Err(io::Error::new(io::ErrorKind::AlreadyExists, "scoped ACK queue is already armed"));
|
||||
}
|
||||
let started = Instant::now();
|
||||
let state = Arc::new(QueueState {
|
||||
scope,
|
||||
capacity,
|
||||
started,
|
||||
deadline: started + RETENTION,
|
||||
closed: AtomicBool::new(false),
|
||||
packets: Mutex::new(Vec::new()),
|
||||
changed: Notify::new(),
|
||||
});
|
||||
*active = Some(state.clone());
|
||||
drop(active);
|
||||
Ok(Self { state, slot })
|
||||
}
|
||||
|
||||
pub(crate) fn receipts(&self) -> Vec<Value> {
|
||||
let packets = self.state.packets.lock().expect("scoped ACK packets").clone();
|
||||
packets
|
||||
.iter()
|
||||
.map(|packet| packet.receipt.lock().expect("scoped ACK receipt").clone())
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub(crate) async fn wait_for_packets(&self, count: usize) -> io::Result<()> {
|
||||
self.wait_until(|| self.receipts().len() >= count).await
|
||||
}
|
||||
|
||||
pub(crate) async fn wait_for_completion(&self, id: usize) -> io::Result<Value> {
|
||||
self.wait_until(|| {
|
||||
self.receipts()
|
||||
.get(id)
|
||||
.is_some_and(|receipt| receipt["completed"].as_bool() == Some(true))
|
||||
})
|
||||
.await?;
|
||||
self.receipts()
|
||||
.get(id)
|
||||
.cloned()
|
||||
.ok_or_else(|| io::Error::other("scoped ACK receipt disappeared"))
|
||||
}
|
||||
|
||||
async fn wait_until(&self, ready: impl Fn() -> bool) -> io::Result<()> {
|
||||
tokio::time::timeout_at(self.state.deadline, async {
|
||||
loop {
|
||||
let changed = self.state.changed.notified();
|
||||
if ready() {
|
||||
return;
|
||||
}
|
||||
changed.await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.map_err(|_| io::Error::new(io::ErrorKind::TimedOut, "scoped ACK queue lifetime elapsed"))
|
||||
}
|
||||
|
||||
pub(crate) fn settle(&self, id: usize, fault: ScopedAckFault) -> io::Result<()> {
|
||||
if Instant::now() >= self.state.deadline {
|
||||
return Err(io::Error::new(io::ErrorKind::TimedOut, "scoped ACK retention expired"));
|
||||
}
|
||||
let packet = self
|
||||
.state
|
||||
.packets
|
||||
.lock()
|
||||
.expect("scoped ACK packets")
|
||||
.get(id)
|
||||
.cloned()
|
||||
.ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, "scoped ACK packet is not retained"))?;
|
||||
if !packet.action.send_if_modified(|current| {
|
||||
if current.is_none() {
|
||||
*current = Some(fault);
|
||||
true
|
||||
} else {
|
||||
false
|
||||
}
|
||||
}) {
|
||||
return Err(io::Error::new(io::ErrorKind::AlreadyExists, "scoped ACK packet was already settled"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for ScopedAckQueue {
|
||||
fn drop(&mut self) {
|
||||
self.state.closed.store(true, Ordering::SeqCst);
|
||||
let mut slot = self.slot.lock().expect("scoped ACK queue slot");
|
||||
if slot.as_ref().is_some_and(|current| Arc::ptr_eq(current, &self.state)) {
|
||||
*slot = None;
|
||||
}
|
||||
drop(slot);
|
||||
let packets = self.state.packets.lock().expect("scoped ACK packets").clone();
|
||||
for packet in packets {
|
||||
if let Some(task) = packet.task.lock().expect("scoped ACK task").take() {
|
||||
task.abort();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) enum Interception {
|
||||
Pass(Request<ProxyBody>),
|
||||
Handled(Response<ProxyBody>),
|
||||
}
|
||||
|
||||
fn restored_body(bytes: Bytes, trailers: Option<http::HeaderMap>) -> ProxyBody {
|
||||
Full::new(bytes)
|
||||
.map_err(|never| match never {})
|
||||
.with_trailers(async move { trailers.map(Ok::<_, io::Error>) })
|
||||
.boxed()
|
||||
}
|
||||
|
||||
fn hash(bytes: &[u8]) -> String {
|
||||
hex(Sha256::digest(bytes).as_ref())
|
||||
}
|
||||
|
||||
fn hex(bytes: &[u8]) -> String {
|
||||
hex_simd::encode_to_string(bytes, hex_simd::AsciiCase::Lower)
|
||||
}
|
||||
|
||||
fn trailer_receipt(trailers: Option<&http::HeaderMap>) -> Value {
|
||||
trailers.map_or(Value::Null, |headers| {
|
||||
json!(
|
||||
headers
|
||||
.iter()
|
||||
.map(|(name, value)| { json!({"name": name.as_str(), "value_hex": hex(value.as_bytes())}) })
|
||||
.collect::<Vec<_>>()
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
fn header_hash(headers: &http::HeaderMap) -> String {
|
||||
let mut hasher = Sha256::new();
|
||||
for (name, value) in headers {
|
||||
hasher.update(name.as_str().as_bytes());
|
||||
hasher.update([0]);
|
||||
hasher.update(value.as_bytes());
|
||||
hasher.update([0]);
|
||||
}
|
||||
hex(hasher.finalize().as_ref())
|
||||
}
|
||||
|
||||
fn millis(state: &QueueState) -> u64 {
|
||||
u64::try_from(state.started.elapsed().as_millis()).unwrap_or(u64::MAX)
|
||||
}
|
||||
|
||||
pub(super) async fn intercept(
|
||||
request: Request<ProxyBody>,
|
||||
client: ProxyClient,
|
||||
target: SocketAddr,
|
||||
state: Arc<QueueState>,
|
||||
) -> io::Result<Interception> {
|
||||
if Instant::now() >= state.deadline || state.closed.load(Ordering::SeqCst) {
|
||||
return Ok(Interception::Pass(request));
|
||||
}
|
||||
let (parts, body) = request.into_parts();
|
||||
let collected = tokio::time::timeout_at(state.deadline, Limited::new(body, MAX_BYTES).collect())
|
||||
.await
|
||||
.map_err(|_| io::Error::new(io::ErrorKind::TimedOut, "scoped ACK frame collection expired"))?
|
||||
.map_err(io::Error::other)?;
|
||||
let trailers = collected.trailers().cloned();
|
||||
let bytes = collected.to_bytes();
|
||||
let decoded = ScannerScopedDirtyUsageAckRequest::decode(unary_payload(&bytes)?).map_err(io::Error::other)?;
|
||||
let request = Request::from_parts(parts, restored_body(bytes.clone(), trailers.clone()));
|
||||
if Instant::now() >= state.deadline || !state.scope.matches(&decoded) {
|
||||
return Ok(Interception::Pass(request));
|
||||
}
|
||||
let header = |name| request.headers().get(name).and_then(|value| value.to_str().ok());
|
||||
let (action, mut selected) = watch::channel(None);
|
||||
let packet = {
|
||||
let mut packets = state.packets.lock().expect("scoped ACK packets");
|
||||
if packets.len() >= state.capacity || state.closed.load(Ordering::SeqCst) {
|
||||
return Ok(Interception::Pass(request));
|
||||
}
|
||||
let id = packets.len();
|
||||
let packet = Arc::new(Packet {
|
||||
action,
|
||||
task: Mutex::new(None),
|
||||
receipt: Mutex::new(json!({
|
||||
"id": id, "session_nonce": state.scope.nonce, "receiver": target.to_string(),
|
||||
"authority": request.uri().authority().map(|value| value.as_str()),
|
||||
"captured_at_ms": millis(&state), "completed": false, "forwarded": false,
|
||||
"backend_complete": false, "outcome": "held", "request_bytes": bytes.len(),
|
||||
"request_sha256": hash(&bytes), "request_frame_hex": hex(&bytes), "request_headers_sha256": header_hash(request.headers()),
|
||||
"request_trailers_sha256": trailers.as_ref().map(header_hash),
|
||||
"request_trailers": trailer_receipt(trailers.as_ref()),
|
||||
"auth_nonce": header("x-rustfs-rpc-nonce"), "replay_nonce": header("x-rustfs-rpc-replay-nonce"),
|
||||
"signature_sha256": header("x-rustfs-rpc-signature-v2").map(|value| hash(value.as_bytes())),
|
||||
"replay_signature_sha256": header("x-rustfs-rpc-signature-v3").map(|value| hash(value.as_bytes())),
|
||||
"request": {"owner_id": decoded.owner_id, "instance_id": decoded.instance_id,
|
||||
"probe_only": decoded.probe_only, "entries": decoded.entries.iter().map(|entry| json!({
|
||||
"bucket": entry.bucket, "bucket_incarnation": uuid::Uuid::from_slice(&entry.bucket_incarnation).ok().map(|value| value.to_string()),
|
||||
"generation": entry.generation })).collect::<Vec<_>>() }
|
||||
})),
|
||||
});
|
||||
packets.push(packet.clone());
|
||||
packet
|
||||
};
|
||||
let (reply, response) = oneshot::channel();
|
||||
let worker_packet = packet.clone();
|
||||
let worker_state = state.clone();
|
||||
// This owner survives an HTTP/2 stream reset from the original 5 s caller.
|
||||
// Every request is settled once and is discarded at the session deadline.
|
||||
let task = tokio::spawn(async move {
|
||||
let result = tokio::time::timeout_at(worker_state.deadline, async {
|
||||
let fault = loop {
|
||||
if let Some(fault) = *selected.borrow_and_update() {
|
||||
break fault;
|
||||
}
|
||||
selected
|
||||
.changed()
|
||||
.await
|
||||
.map_err(|_| io::Error::other("scoped ACK queue closed"))?;
|
||||
};
|
||||
if fault == ScopedAckFault::RequestLoss {
|
||||
worker_packet.receipt.lock().expect("scoped ACK receipt")["outcome"] = json!("request_lost");
|
||||
return Err(io::Error::new(io::ErrorKind::ConnectionAborted, "injected scoped ACK request loss"));
|
||||
}
|
||||
if worker_state.closed.load(Ordering::SeqCst) || Instant::now() >= worker_state.deadline {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::ConnectionAborted,
|
||||
"scoped ACK session ended or expired before forwarding",
|
||||
));
|
||||
}
|
||||
{
|
||||
let mut receipt = worker_packet.receipt.lock().expect("scoped ACK receipt");
|
||||
receipt["forwarded"] = json!(true);
|
||||
receipt["forwarded_at_ms"] = json!(millis(&worker_state));
|
||||
}
|
||||
let backend = forward_to_backend(request, client, target).await?;
|
||||
let (parts, body) = backend.into_parts();
|
||||
let collected = Limited::new(body, MAX_BYTES).collect().await.map_err(io::Error::other)?;
|
||||
let trailers = collected.trailers().cloned();
|
||||
let bytes = collected.to_bytes();
|
||||
let grpc_status = trailers
|
||||
.as_ref()
|
||||
.and_then(|values| values.get("grpc-status"))
|
||||
.or_else(|| parts.headers.get("grpc-status"))
|
||||
.and_then(|value| value.to_str().ok());
|
||||
let response = unary_payload(&bytes)
|
||||
.ok()
|
||||
.and_then(|payload| ScannerScopedDirtyUsageAckResponse::decode(payload).ok())
|
||||
.map(|response| {
|
||||
json!({"owner_id": response.owner_id, "instance_id": response.instance_id,
|
||||
"supported": response.supported, "cleared": response.cleared,
|
||||
"response_proof_bytes": response.response_proof.len()})
|
||||
});
|
||||
{
|
||||
let mut receipt = worker_packet.receipt.lock().expect("scoped ACK receipt");
|
||||
receipt["backend_complete"] = json!(true);
|
||||
receipt["backend_completed_at_ms"] = json!(millis(&worker_state));
|
||||
receipt["response"] = json!({"status": parts.status.as_u16(), "grpc_status": grpc_status,
|
||||
"bytes": bytes.len(), "sha256": hash(&bytes), "frame_hex": hex(&bytes), "headers_sha256": header_hash(&parts.headers),
|
||||
"trailers": trailer_receipt(trailers.as_ref()),
|
||||
"trailers_sha256": trailers.as_ref().map(header_hash), "ack": response});
|
||||
receipt["outcome"] = json!(if fault == ScopedAckFault::ResponseLoss {
|
||||
"response_lost"
|
||||
} else {
|
||||
"forwarded"
|
||||
});
|
||||
}
|
||||
if fault == ScopedAckFault::ResponseLoss {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::ConnectionAborted,
|
||||
"injected scoped ACK response loss after backend trailers",
|
||||
));
|
||||
}
|
||||
Ok(Response::from_parts(parts, restored_body(bytes, trailers)))
|
||||
})
|
||||
.await
|
||||
.unwrap_or_else(|_| Err(io::Error::new(io::ErrorKind::TimedOut, "scoped ACK retention expired")));
|
||||
{
|
||||
let mut receipt = worker_packet.receipt.lock().expect("scoped ACK receipt");
|
||||
if result.is_err() && receipt["outcome"] == "held" {
|
||||
receipt["outcome"] = json!("transport_failed_or_expired");
|
||||
}
|
||||
receipt["completed"] = json!(true);
|
||||
receipt["completed_at_ms"] = json!(millis(&worker_state));
|
||||
}
|
||||
let _ = reply.send(result);
|
||||
worker_state.changed.notify_one();
|
||||
});
|
||||
*packet.task.lock().expect("scoped ACK task") = Some(task.abort_handle());
|
||||
if state.closed.load(Ordering::SeqCst) {
|
||||
task.abort();
|
||||
}
|
||||
drop(task);
|
||||
state.changed.notify_one();
|
||||
response
|
||||
.await
|
||||
.map_err(|_| io::Error::new(io::ErrorKind::ConnectionAborted, "scoped ACK queue ended"))?
|
||||
.map(Interception::Handled)
|
||||
}
|
||||
@@ -70,9 +70,6 @@ pub(crate) mod multipart_rpc_delay {
|
||||
#[cfg(test)]
|
||||
pub(crate) mod multipart_fence_acceptance {
|
||||
pub(crate) use super::RUSTFS_META_MULTIPART_BUCKET;
|
||||
|
||||
/// Targeted lock diagnostics for the metadata-delay quorum case.
|
||||
pub(crate) const LOCK_DIAGNOSTICS_FILTER: &str = "warn,rustfs_lock::distributed_lock=debug,rustfs_ecstore::object_lock_diag=trace,rustfs_ecstore::cluster::rpc::remote_locker=info";
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -83,14 +80,3 @@ pub(crate) mod multipart_source_rpc {
|
||||
PublicationIdentity, SourceGeneration, SourceGeometry, SourceLineage,
|
||||
};
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) mod scanner_peer_bucket {
|
||||
use rustfs_ecstore::api::disk as ecstore_disk;
|
||||
use rustfs_ecstore::api::erasure as ecstore_erasure;
|
||||
use rustfs_ecstore::api::rpc as ecstore_rpc;
|
||||
|
||||
pub(crate) use ecstore_disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET};
|
||||
pub(crate) use ecstore_erasure::{ReedSolomonEncoder, calc_shard_size};
|
||||
pub(crate) use ecstore_rpc::PeerRestClient;
|
||||
}
|
||||
|
||||
@@ -1112,9 +1112,8 @@ async fn wait_for_upgrade_writes(
|
||||
match response {
|
||||
Ok(Ok(_)) => break,
|
||||
Ok(Err(error)) => {
|
||||
let code = error.as_service_error().and_then(ProvideErrorMetadata::code);
|
||||
if error.raw_response().map(|response| response.status().as_u16()) != Some(503)
|
||||
|| !matches!(code, Some("ServiceUnavailable" | "SlowDown" | "SlowDownRead"))
|
||||
|| error.as_service_error().and_then(ProvideErrorMetadata::code) != Some("ServiceUnavailable")
|
||||
{
|
||||
return Err(format!("{phase}: node {node} {probe} failed: {error:?}").into());
|
||||
}
|
||||
|
||||
@@ -31,7 +31,6 @@ workspace = true
|
||||
|
||||
[features]
|
||||
default = []
|
||||
fips = ["rustfs-s3-client/fips", "rustfs-utils/fips"]
|
||||
diagnostic-read-version-caller = ["rustfs-io-metrics/diagnostic-read-version-caller"]
|
||||
# Sampled wall/CPU diagnostics only; does not change the read backend.
|
||||
diagnostic-local-batch = ["rustix/time"]
|
||||
@@ -140,6 +139,7 @@ rustfs-lifecycle.workspace = true
|
||||
rustfs-s3-client = { workspace = true }
|
||||
rustfs-s3-types = { workspace = true }
|
||||
rustfs-data-usage.workspace = true
|
||||
rustfs-gateway-types.workspace = true
|
||||
rustfs-object-capacity.workspace = true
|
||||
rustfs-object-data-cache = { workspace = true, features = ["runtime-memory"] }
|
||||
arc-swap.workspace = true
|
||||
@@ -158,7 +158,6 @@ serde = { workspace = true, features = ["derive"] }
|
||||
time = { workspace = true, features = ["parsing", "formatting", "macros", "serde"] }
|
||||
bytesize.workspace = true
|
||||
serde_json = { workspace = true, features = ["raw_value"] }
|
||||
s3s = { workspace = true, features = ["minio"] }
|
||||
http.workspace = true
|
||||
opentelemetry.workspace = true
|
||||
http-body = { workspace = true }
|
||||
@@ -226,11 +225,9 @@ metrics = { workspace = true }
|
||||
# The guard scripts/check_no_tokio_io_uring.sh allows an explicit
|
||||
# io-uring integration; only the tokio "io-uring" runtime feature is banned.
|
||||
[target.'cfg(target_os = "linux")'.dependencies]
|
||||
# Native completion owners keep physical I/O leases alive after cancellation.
|
||||
rustfs-uring = { git = "https://github.com/rustfs/uring.git", rev = "eb8fc160e71d804427de1e905f5c226d580f71bf" }
|
||||
|
||||
[target.'cfg(target_os = "linux")'.dev-dependencies]
|
||||
rustfs-uring = { git = "https://github.com/rustfs/uring.git", rev = "eb8fc160e71d804427de1e905f5c226d580f71bf", features = ["fault-injection"] }
|
||||
# SharedReadBudget and the bounded read APIs are available in the 0.3.0
|
||||
# registry release; keep the application on the published contract.
|
||||
rustfs-uring = "0.3.0"
|
||||
|
||||
[target.'cfg(windows)'.dependencies]
|
||||
winapi-util.workspace = true
|
||||
@@ -249,6 +246,7 @@ windows-sys = { workspace = true, features = [
|
||||
windows-sys = { workspace = true, features = ["Win32_System_Ioctl"] }
|
||||
|
||||
[dev-dependencies]
|
||||
s3s = { workspace = true, features = ["minio"] }
|
||||
aws-smithy-async.workspace = true
|
||||
rustfs-s3-ops.workspace = true
|
||||
tokio = { workspace = true, features = ["rt-multi-thread", "macros", "test-util", "fs"] }
|
||||
|
||||
@@ -66,6 +66,8 @@ pub mod bucket {
|
||||
}
|
||||
|
||||
pub mod bucket_lifecycle_ops {
|
||||
pub use rustfs_lifecycle::{RestoreOutputLocation, RestoreRequest, RestoreS3Location, RestoreTag, RestoreTagging};
|
||||
|
||||
pub use crate::bucket::lifecycle::bucket_lifecycle_ops::{
|
||||
ExpiryState, LifecycleOps, ManualTransitionCancelCheck, ManualTransitionProgressSink,
|
||||
ManualTransitionQueueSnapshot, ManualTransitionRunExecution, ManualTransitionRunOptions,
|
||||
@@ -137,7 +139,7 @@ pub mod bucket {
|
||||
pub use crate::bucket::lifecycle::lifecycle::{
|
||||
Event, ExpirationOptions, IlmAction, LIFECYCLE_MALFORMED_XML_ERROR_KIND, Lifecycle, LifecycleCalculate,
|
||||
ObjectOpts, RuleValidate, TRANSITION_COMPLETE, TRANSITION_PENDING, TransitionOptions, expected_expiry_time,
|
||||
object_opts_from_object_info,
|
||||
format_lifecycle_date, object_opts_from_object_info, parse_lifecycle_date,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -235,7 +237,7 @@ pub mod bucket {
|
||||
}
|
||||
|
||||
pub mod object_lock {
|
||||
pub use crate::bucket::object_lock::{ObjectLockApi, ObjectLockStatusExt};
|
||||
pub use crate::bucket::object_lock::ObjectLockApi;
|
||||
|
||||
pub mod types {
|
||||
pub use crate::bucket::object_lock::types::{
|
||||
@@ -330,9 +332,9 @@ pub mod bucket {
|
||||
|
||||
pub mod utils {
|
||||
pub use crate::bucket::utils::{
|
||||
check_bucket_and_object_names, check_list_objs_args, check_object_name_for_length_and_slash,
|
||||
check_valid_bucket_name_strict, deserialize, has_bad_path_component, is_meta_bucketname, is_valid_object_prefix,
|
||||
serialize,
|
||||
BucketConfigXml, BucketConfigXmlError, check_bucket_and_object_names, check_list_objs_args,
|
||||
check_object_name_for_length_and_slash, check_valid_bucket_name_strict, deserialize, has_bad_path_component,
|
||||
is_meta_bucketname, is_valid_object_prefix, serialize,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -359,7 +361,7 @@ pub mod capacity {
|
||||
}
|
||||
|
||||
pub mod object_api_utils {
|
||||
pub use crate::object_api::object_api_utils::{ObjReaderFn, PutObjReader, get_raw_etag, new_getobjectreader, to_s3s_etag};
|
||||
pub use crate::object_api::object_api_utils::{ObjReaderFn, PutObjReader, get_raw_etag, new_getobjectreader};
|
||||
}
|
||||
|
||||
pub mod cluster {
|
||||
@@ -447,9 +449,6 @@ pub mod data_usage {
|
||||
}
|
||||
|
||||
pub mod disk {
|
||||
pub use crate::cluster::rpc::background_pipe::{BackgroundPipeReader, BackgroundPipeWriter, background_pipe};
|
||||
pub use crate::cluster::rpc::background_reader_stream::BackgroundReaderStream;
|
||||
pub use crate::disk::background_memory::BackgroundEncodeBuffer;
|
||||
pub use crate::disk::disk_store::get_object_disk_read_timeout;
|
||||
pub use crate::disk::local::{ReplacementExecutionLease, ScanGuard};
|
||||
#[cfg(feature = "test-util")]
|
||||
@@ -478,10 +477,6 @@ pub mod disk {
|
||||
UpdateMetadataOpts, VolumeInfo, WalkDirOptions, is_quota_mutation_fence_path, new_disk,
|
||||
validate_batch_read_version_item_count,
|
||||
};
|
||||
pub use crate::disk::{
|
||||
BackgroundMemoryBudget, PhysicalDiskBinding, PhysicalDiskPlan, PhysicalDiskSample, PhysicalDiskSampler,
|
||||
PhysicalResourcePlan, RemoteBackgroundIo, with_background_resource_unit, with_remote_background_io,
|
||||
};
|
||||
pub use bytes::Bytes;
|
||||
pub use endpoint::Endpoint;
|
||||
pub use error::DiskError;
|
||||
@@ -622,7 +617,6 @@ pub mod rio {
|
||||
}
|
||||
|
||||
pub mod rpc {
|
||||
pub use crate::cluster::rpc::http_auth::with_rpc_background_request;
|
||||
pub use crate::cluster::rpc::{
|
||||
AuthenticatedChannel, KMS_SIGNAL_SUBSYSTEM, LocalPeerS3Client, MAX_NETWORK_PROBE_BYTES, MAX_NETWORK_PROBE_DURATION,
|
||||
NetworkPeerProbeClient, NetworkPeerProbeError, NetworkPeerProbeMeasurement, NetworkPeerTarget, PEER_RESTDRY_RUN,
|
||||
@@ -632,8 +626,8 @@ pub mod rpc {
|
||||
TONIC_RPC_PREFIX, TonicInterceptor, build_put_file_auth_trailer, check_and_record_signed_rpc_nonce,
|
||||
decode_heal_bucket_rpc_options, encode_heal_bucket_rpc_options, gen_signature_headers, gen_tonic_replay_scope_headers,
|
||||
gen_tonic_signature_headers, gen_tonic_signature_interceptor, node_service_time_out_client,
|
||||
node_service_time_out_client_no_auth, normalize_tonic_rpc_audience, rpc_background_admission,
|
||||
set_tonic_canonical_body_digest, sign_ns_scanner_capability, sign_ns_scanner_capability_with_checkpoint_coverage,
|
||||
node_service_time_out_client_no_auth, normalize_tonic_rpc_audience, set_tonic_canonical_body_digest,
|
||||
sign_ns_scanner_capability, sign_ns_scanner_capability_with_checkpoint_coverage,
|
||||
sign_ns_scanner_capability_with_tier_registry_generation, sign_put_file_capability, sign_tonic_rpc_response_proof,
|
||||
tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason,
|
||||
verify_ns_scanner_capability, verify_ns_scanner_capability_with_checkpoint_coverage,
|
||||
@@ -683,9 +677,8 @@ pub mod storage {
|
||||
pub use crate::store::HealWalkVersion;
|
||||
pub use crate::store::{
|
||||
BootstrapLocalTarget, ECStore, SCANNER_PUBLICATION_LEASE_TTL_MS, ScannerDataMovementPauseStatus, all_local_disk,
|
||||
all_local_disk_path, find_local_disk_by_ref, find_local_disk_by_ref_for_rpc, init_local_disks,
|
||||
init_local_disks_with_instance_ctx, init_lock_clients, prewarm_local_disk_id_map,
|
||||
prewarm_local_disk_id_map_with_instance_ctx,
|
||||
all_local_disk_path, find_local_disk_by_ref, init_local_disks, init_local_disks_with_instance_ctx, init_lock_clients,
|
||||
prewarm_local_disk_id_map, prewarm_local_disk_id_map_with_instance_ctx,
|
||||
};
|
||||
pub use crate::store::{HealObjectAbsenceProof, HealObjectStorageResult};
|
||||
}
|
||||
|
||||
@@ -88,12 +88,14 @@ use rustfs_filemeta::{
|
||||
FileInfo, FileInfoOpts, NULL_VERSION_ID, RestoreStatus, RestoreStatusOps, TRANSITION_COMPLETE, get_file_info,
|
||||
is_restored_object_on_disk,
|
||||
};
|
||||
use rustfs_gateway_types::persistence::PersistedObjectLockConfiguration as ObjectLockConfiguration;
|
||||
use rustfs_gateway_types::persistence::{PersistedLifecycleConfiguration, PersistedLifecycleFilter};
|
||||
use rustfs_lifecycle::{RestoreRequest, RestoreS3Location};
|
||||
use rustfs_lock::{NamespaceLockGuard, error::LockError};
|
||||
use rustfs_scanner_metrics::metrics::{
|
||||
IlmAction, Metrics, ScannerLifecycleExpiryStateUpdate, ScannerLifecycleTransitionStateUpdate, global_metrics,
|
||||
};
|
||||
use rustfs_utils::{get_env_i64, get_env_usize, path::encode_dir_object, string::parse_bool};
|
||||
use s3s::dto::{BucketLifecycleConfiguration, ExpirationStatus, ObjectLockConfiguration, RestoreRequest, RestoreRequestType};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::any::Any;
|
||||
use std::collections::{BTreeMap, HashMap, HashSet};
|
||||
@@ -300,7 +302,7 @@ impl LifecycleSys {
|
||||
Arc::new(Self)
|
||||
}
|
||||
|
||||
pub async fn get(&self, bucket: &str) -> Option<BucketLifecycleConfiguration> {
|
||||
pub async fn get(&self, bucket: &str) -> Option<PersistedLifecycleConfiguration> {
|
||||
match metadata_boundary::get_lifecycle_config(bucket).await {
|
||||
Ok((lc, _)) => Some(lc),
|
||||
Err(Error::ConfigNotFound) => None,
|
||||
@@ -3557,7 +3559,7 @@ async fn stale_upload_lifecycle_due(
|
||||
Err(_) => return None,
|
||||
};
|
||||
|
||||
let has_size_filter = |filter: &s3s::dto::LifecycleRuleFilter| {
|
||||
let has_size_filter = |filter: &PersistedLifecycleFilter| {
|
||||
filter.object_size_greater_than.is_some()
|
||||
|| filter.object_size_less_than.is_some()
|
||||
|| filter
|
||||
@@ -3566,7 +3568,7 @@ async fn stale_upload_lifecycle_due(
|
||||
.is_some_and(|and| and.object_size_greater_than.is_some() || and.object_size_less_than.is_some())
|
||||
};
|
||||
let needs_size = lifecycle.rules.iter().any(|rule| {
|
||||
rule.status.as_str() != ExpirationStatus::DISABLED
|
||||
rule.status.as_str() != "Disabled"
|
||||
&& rule.abort_incomplete_multipart_upload.is_some()
|
||||
&& rule.filter.as_ref().is_some_and(has_size_filter)
|
||||
});
|
||||
@@ -4000,7 +4002,7 @@ pub fn init_background_stale_multipart_upload_cleanup(api: Arc<ECStore>) {
|
||||
});
|
||||
}
|
||||
|
||||
pub async fn validate_transition_tier(lc: &BucketLifecycleConfiguration) -> Result<(), std::io::Error> {
|
||||
pub async fn validate_transition_tier(lc: &PersistedLifecycleConfiguration) -> Result<(), std::io::Error> {
|
||||
for rule in &lc.rules {
|
||||
if let Some(transitions) = &rule.transitions {
|
||||
for transition in transitions {
|
||||
@@ -4117,8 +4119,9 @@ pub(crate) async fn enqueue_immediate_expiry(api: Arc<ECStore>, oi: &ObjectInfo,
|
||||
.map(lifecycle::object_opts_from_object_info)
|
||||
.collect::<Vec<ObjectOpts>>();
|
||||
let lock_config = configs.object_lock;
|
||||
let evaluator_lock_config = lock_config.clone();
|
||||
let events = match Evaluator::new(lifecycle)
|
||||
.with_lock_retention(lock_config.clone())
|
||||
.with_lock_retention(evaluator_lock_config)
|
||||
.eval(&object_opts)
|
||||
.await
|
||||
{
|
||||
@@ -4598,19 +4601,26 @@ pub async fn enqueue_transition_for_existing_objects_scoped(
|
||||
}
|
||||
}
|
||||
|
||||
fn lifecycle_rule_has_date_expiration(lc: &BucketLifecycleConfiguration, rule_id: &str) -> bool {
|
||||
fn lifecycle_rule_has_date_expiration(lc: &PersistedLifecycleConfiguration, rule_id: &str) -> bool {
|
||||
lc.rules.iter().any(|rule| {
|
||||
rule.status == ExpirationStatus::from_static(ExpirationStatus::ENABLED)
|
||||
rule.status == "Enabled"
|
||||
&& rule.id.as_deref() == Some(rule_id)
|
||||
&& rule.expiration.as_ref().is_some_and(|expiration| expiration.date.is_some())
|
||||
})
|
||||
}
|
||||
|
||||
fn should_defer_date_expiry_for_recent_config_update(lc: &BucketLifecycleConfiguration, now: OffsetDateTime) -> bool {
|
||||
lc.expiry_updated_at.as_ref().is_some_and(|updated_at| {
|
||||
let updated_at = OffsetDateTime::from(updated_at.clone());
|
||||
now.unix_timestamp().saturating_sub(updated_at.unix_timestamp()) < DATE_EXPIRY_EXISTING_OBJECTS_GRACE_SECS
|
||||
})
|
||||
fn should_defer_date_expiry_for_recent_config_update(lc: &PersistedLifecycleConfiguration, now: OffsetDateTime) -> bool {
|
||||
lc.expiry_updated_at
|
||||
.as_ref()
|
||||
.is_some_and(|updated_at| match rustfs_lifecycle::parse_lifecycle_date(updated_at) {
|
||||
Ok(updated_at) => {
|
||||
now.unix_timestamp().saturating_sub(updated_at.unix_timestamp()) < DATE_EXPIRY_EXISTING_OBJECTS_GRACE_SECS
|
||||
}
|
||||
Err(error) => {
|
||||
warn!(error = %error, "Deferring date expiry with an unreadable update timestamp");
|
||||
true
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
async fn apply_existing_object_expiry(
|
||||
@@ -4630,7 +4640,7 @@ async fn apply_existing_object_expiry(
|
||||
struct ExistingObjectExpiryContext<'a> {
|
||||
api: Arc<ECStore>,
|
||||
bucket: &'a str,
|
||||
lc: Arc<BucketLifecycleConfiguration>,
|
||||
lc: Arc<PersistedLifecycleConfiguration>,
|
||||
lock_config: Option<Arc<ObjectLockConfiguration>>,
|
||||
bucket_incarnation_id: Uuid,
|
||||
src: &'a LcEventSrc,
|
||||
@@ -4650,8 +4660,9 @@ async fn enqueue_expiry_for_existing_object_group(
|
||||
.iter()
|
||||
.map(lifecycle::object_opts_from_object_info)
|
||||
.collect::<Vec<ObjectOpts>>();
|
||||
let evaluator_lock_config = context.lock_config.clone();
|
||||
let events = match Evaluator::new(context.lc.clone())
|
||||
.with_lock_retention(context.lock_config.clone())
|
||||
.with_lock_retention(evaluator_lock_config)
|
||||
.eval(&object_opts)
|
||||
.await
|
||||
{
|
||||
@@ -4845,7 +4856,7 @@ fn manual_transition_version_marker(oi: &ObjectInfo) -> String {
|
||||
async fn enqueue_transition_with_lifecycle_report(
|
||||
api: Option<Arc<ECStore>>,
|
||||
oi: &ObjectInfo,
|
||||
lc: &BucketLifecycleConfiguration,
|
||||
lc: &PersistedLifecycleConfiguration,
|
||||
src: &LcEventSrc,
|
||||
options: &ManualTransitionRunOptions,
|
||||
report: &mut ManualTransitionRunReport,
|
||||
@@ -4911,7 +4922,7 @@ async fn enqueue_transition_with_lifecycle_report(
|
||||
false
|
||||
}
|
||||
|
||||
async fn enqueue_transition_with_lifecycle(oi: &ObjectInfo, lc: &BucketLifecycleConfiguration, src: &LcEventSrc) -> bool {
|
||||
async fn enqueue_transition_with_lifecycle(oi: &ObjectInfo, lc: &PersistedLifecycleConfiguration, src: &LcEventSrc) -> bool {
|
||||
let event = lc.eval(&oi.to_lifecycle_opts()).await;
|
||||
match event.action {
|
||||
IlmAction::TransitionAction | IlmAction::TransitionVersionAction => {
|
||||
@@ -5425,12 +5436,8 @@ pub async fn post_restore_opts(version_id: &str, bucket: &str, object: &str) ->
|
||||
})
|
||||
}
|
||||
|
||||
fn select_restore_s3_location(rreq: &RestoreRequest) -> Result<Option<&s3s::dto::S3Location>, std::io::Error> {
|
||||
if rreq
|
||||
.type_
|
||||
.as_ref()
|
||||
.is_none_or(|type_| type_.as_str() != RestoreRequestType::SELECT)
|
||||
{
|
||||
fn select_restore_s3_location(rreq: &RestoreRequest) -> Result<Option<&RestoreS3Location>, std::io::Error> {
|
||||
if rreq.type_.as_ref().is_none_or(|type_| type_.as_str() != "SELECT") {
|
||||
return Ok(None);
|
||||
}
|
||||
let output_location = rreq
|
||||
@@ -5472,27 +5479,25 @@ pub async fn put_restore_opts(
|
||||
// the OutputLocation metadata instead of a write to `OutputLocation.S3`
|
||||
// (backlog#1341). RestoreObject rejects SELECT at the API boundary; this
|
||||
// is the fail-closed backstop for any other caller.
|
||||
if rreq
|
||||
.type_
|
||||
.as_ref()
|
||||
.is_some_and(|type_| type_.as_str() == RestoreRequestType::SELECT)
|
||||
{
|
||||
if rreq.type_.as_ref().is_some_and(|type_| type_.as_str() == "SELECT") {
|
||||
return Err(std::io::Error::other("SELECT restore requests are not supported"));
|
||||
}
|
||||
for (k, v) in oi.user_defined.iter() {
|
||||
meta.insert(k.to_string(), v.clone());
|
||||
}
|
||||
// The fleet capability gate guarantees physical copy-back support. Keep
|
||||
// the completed transition identity for orphan recovery and remote cleanup.
|
||||
rustfs_utils::http::metadata_compat::remove_str(&mut meta, rustfs_utils::http::metadata_compat::SUFFIX_RESTORE_OPERATION_ID);
|
||||
rustfs_utils::http::metadata_compat::remove_str(&mut meta, rustfs_utils::http::metadata_compat::SUFFIX_RESTORE_WORKER_LOCK);
|
||||
if !oi.user_tags.is_empty() {
|
||||
meta.insert(AMZ_OBJECT_TAGGING.to_string(), (*oi.user_tags).clone());
|
||||
}
|
||||
let restore_expiry = lifecycle::expected_expiry_time(OffsetDateTime::now_utc(), rreq.days.unwrap_or(1));
|
||||
// Finalization publishes completion and its expiry under the object lock.
|
||||
// The local copy-back commit must not make unfinished work eligible for expiry.
|
||||
meta.insert(
|
||||
metadata_keys::RESTORE.to_string(),
|
||||
RestoreStatus {
|
||||
is_restore_in_progress: Some(false),
|
||||
restore_expiry_date: Some(restore_expiry),
|
||||
is_restore_in_progress: Some(true),
|
||||
restore_expiry_date: None,
|
||||
}
|
||||
.to_string(),
|
||||
);
|
||||
@@ -5506,6 +5511,7 @@ pub async fn put_restore_opts(
|
||||
// computed MD5 is not the object's public plaintext ETag.
|
||||
preserve_etag: oi.etag.clone(),
|
||||
shard_integrity_write_mode: Some(oi.shard_integrity_write_mode()),
|
||||
restore_copyback: true,
|
||||
//expires: oi.expires,
|
||||
..Default::default()
|
||||
})
|
||||
@@ -5525,6 +5531,18 @@ impl LifecycleOps for ObjectInfo {
|
||||
if self.transitioned_object.status != lifecycle::TRANSITION_COMPLETE {
|
||||
return false;
|
||||
}
|
||||
if rustfs_utils::http::metadata_compat::get_consistent_str(
|
||||
&self.user_defined,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_RESTORE_COPYBACK,
|
||||
) == Some(rustfs_utils::http::metadata_compat::RESTORE_COPYBACK_PROTOCOL_V1)
|
||||
&& self
|
||||
.user_defined
|
||||
.get(metadata_keys::RESTORE)
|
||||
.and_then(|value| rustfs_filemeta::parse_restore_obj_status(value).ok())
|
||||
.is_some_and(|status| status.is_restore_in_progress == Some(true))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
!is_restored_object_on_disk(&self.user_defined)
|
||||
}
|
||||
}
|
||||
@@ -5536,25 +5554,25 @@ pub trait RestoreRequestOps {
|
||||
impl RestoreRequestOps for RestoreRequest {
|
||||
fn validate(&self, _api: Arc<ECStore>) -> Result<(), std::io::Error> {
|
||||
// SELECT type requires select_parameters, and vice versa
|
||||
if self.type_.as_ref().is_none_or(|t| t.as_str() != RestoreRequestType::SELECT) && self.select_parameters.is_some() {
|
||||
if self.type_.as_ref().is_none_or(|t| t.as_str() != "SELECT") && self.select_parameters.is_some() {
|
||||
return Err(std::io::Error::other("Select parameters can only be specified with SELECT request type"));
|
||||
}
|
||||
if let Some(type_) = &self.type_
|
||||
&& type_.as_str() == RestoreRequestType::SELECT
|
||||
&& type_.as_str() == "SELECT"
|
||||
&& self.select_parameters.is_none()
|
||||
{
|
||||
return Err(std::io::Error::other("SELECT restore request requires select parameters to be specified"));
|
||||
}
|
||||
|
||||
// OutputLocation is only valid for SELECT requests
|
||||
if self.type_.as_ref().is_none_or(|t| t.as_str() != RestoreRequestType::SELECT) && self.output_location.is_some() {
|
||||
if self.type_.as_ref().is_none_or(|t| t.as_str() != "SELECT") && self.output_location.is_some() {
|
||||
return Err(std::io::Error::other("OutputLocation can only be specified with SELECT request type"));
|
||||
}
|
||||
select_restore_s3_location(self)?;
|
||||
|
||||
// Days must not be specified with SELECT requests
|
||||
if let Some(type_) = &self.type_
|
||||
&& type_.as_str() == RestoreRequestType::SELECT
|
||||
&& type_.as_str() == "SELECT"
|
||||
&& self.days.is_some_and(|d| d > 0)
|
||||
{
|
||||
return Err(std::io::Error::other("Days cannot be specified with SELECT restore request"));
|
||||
@@ -5596,7 +5614,7 @@ pub fn lifecycle_version_delete_target(oi: &ObjectInfo) -> Option<ObjectToDelete
|
||||
}
|
||||
|
||||
pub async fn eval_action_from_lifecycle(
|
||||
lc: &BucketLifecycleConfiguration,
|
||||
lc: &PersistedLifecycleConfiguration,
|
||||
lock_config: Option<&ObjectLockConfiguration>,
|
||||
oi: &ObjectInfo,
|
||||
) -> lifecycle::Event {
|
||||
@@ -6301,14 +6319,19 @@ mod tests {
|
||||
use rustfs_data_usage::TierStats;
|
||||
use rustfs_filemeta::metadata_keys;
|
||||
use rustfs_filemeta::{FileInfo, FileMeta};
|
||||
use rustfs_gateway_types::dto::MetadataEntry;
|
||||
use rustfs_gateway_types::persistence::{
|
||||
PersistedDefaultRetention as DefaultRetention, PersistedObjectLockConfiguration as ObjectLockConfiguration,
|
||||
PersistedObjectLockRule as ObjectLockRule,
|
||||
};
|
||||
use rustfs_gateway_types::persistence::{
|
||||
PersistedLifecycleConfiguration, PersistedLifecycleExpiration, PersistedLifecycleRule,
|
||||
PersistedNoncurrentVersionExpiration, PersistedTransition,
|
||||
};
|
||||
use rustfs_lifecycle::{RestoreOutputLocation as OutputLocation, RestoreRequest, RestoreS3Location as S3Location};
|
||||
#[cfg(feature = "test-util")]
|
||||
use rustfs_s3_client::transition_api::ReaderImpl;
|
||||
use rustfs_scanner_metrics::metrics::{IlmAction, global_metrics};
|
||||
use s3s::dto::{
|
||||
BucketLifecycleConfiguration, DefaultRetention, ExpirationStatus, LifecycleExpiration, LifecycleRule, MetadataEntry,
|
||||
NoncurrentVersionExpiration, ObjectLockConfiguration, ObjectLockEnabled, ObjectLockRetentionMode, ObjectLockRule,
|
||||
OutputLocation, RestoreRequest, RestoreRequestType, S3Location, Timestamp, Transition, TransitionStorageClass,
|
||||
};
|
||||
use serial_test::serial;
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::collections::HashMap;
|
||||
@@ -9315,7 +9338,7 @@ mod tests {
|
||||
output_location,
|
||||
select_parameters: None,
|
||||
tier: None,
|
||||
type_: Some(RestoreRequestType::from_static(RestoreRequestType::SELECT)),
|
||||
type_: Some("SELECT".to_owned()),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -9632,12 +9655,12 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn lifecycle_rule_has_date_expiration_detects_enabled_date_rule() {
|
||||
let lc = BucketLifecycleConfiguration {
|
||||
let lc = PersistedLifecycleConfiguration {
|
||||
expiry_updated_at: None,
|
||||
rules: vec![LifecycleRule {
|
||||
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
|
||||
expiration: Some(LifecycleExpiration {
|
||||
date: Some(Timestamp::from(OffsetDateTime::now_utc())),
|
||||
rules: vec![PersistedLifecycleRule {
|
||||
status: "Enabled".to_owned(),
|
||||
expiration: Some(PersistedLifecycleExpiration {
|
||||
date: Some(lifecycle::format_lifecycle_date(OffsetDateTime::now_utc()).unwrap()),
|
||||
..Default::default()
|
||||
}),
|
||||
id: Some("rule-date".to_string()),
|
||||
@@ -10154,14 +10177,15 @@ mod tests {
|
||||
#[test]
|
||||
fn should_defer_date_expiry_for_recent_config_update_respects_grace_window() {
|
||||
let now = OffsetDateTime::now_utc();
|
||||
let recent = BucketLifecycleConfiguration {
|
||||
expiry_updated_at: Some(Timestamp::from(now - time::Duration::seconds(1))),
|
||||
let recent = PersistedLifecycleConfiguration {
|
||||
expiry_updated_at: Some(lifecycle::format_lifecycle_date(now - time::Duration::seconds(1)).unwrap()),
|
||||
rules: Vec::new(),
|
||||
};
|
||||
let stale = BucketLifecycleConfiguration {
|
||||
expiry_updated_at: Some(Timestamp::from(
|
||||
now - time::Duration::seconds(DATE_EXPIRY_EXISTING_OBJECTS_GRACE_SECS + 1),
|
||||
)),
|
||||
let stale = PersistedLifecycleConfiguration {
|
||||
expiry_updated_at: Some(
|
||||
lifecycle::format_lifecycle_date(now - time::Duration::seconds(DATE_EXPIRY_EXISTING_OBJECTS_GRACE_SECS + 1))
|
||||
.unwrap(),
|
||||
),
|
||||
rules: Vec::new(),
|
||||
};
|
||||
|
||||
@@ -10289,12 +10313,12 @@ mod tests {
|
||||
assert_eq!(deleted.version_id, None);
|
||||
}
|
||||
|
||||
fn expired_delete_marker_lifecycle() -> BucketLifecycleConfiguration {
|
||||
BucketLifecycleConfiguration {
|
||||
fn expired_delete_marker_lifecycle() -> PersistedLifecycleConfiguration {
|
||||
PersistedLifecycleConfiguration {
|
||||
expiry_updated_at: None,
|
||||
rules: vec![LifecycleRule {
|
||||
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
|
||||
expiration: Some(LifecycleExpiration {
|
||||
rules: vec![PersistedLifecycleRule {
|
||||
status: "Enabled".to_owned(),
|
||||
expiration: Some(PersistedLifecycleExpiration {
|
||||
expired_object_delete_marker: Some(true),
|
||||
..Default::default()
|
||||
}),
|
||||
@@ -10310,12 +10334,12 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
fn latest_expiration_lifecycle() -> BucketLifecycleConfiguration {
|
||||
BucketLifecycleConfiguration {
|
||||
fn latest_expiration_lifecycle() -> PersistedLifecycleConfiguration {
|
||||
PersistedLifecycleConfiguration {
|
||||
expiry_updated_at: None,
|
||||
rules: vec![LifecycleRule {
|
||||
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
|
||||
expiration: Some(LifecycleExpiration {
|
||||
rules: vec![PersistedLifecycleRule {
|
||||
status: "Enabled".to_owned(),
|
||||
expiration: Some(PersistedLifecycleExpiration {
|
||||
days: Some(1),
|
||||
..Default::default()
|
||||
}),
|
||||
@@ -10331,12 +10355,12 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
fn current_and_noncurrent_expiration_lifecycle() -> BucketLifecycleConfiguration {
|
||||
BucketLifecycleConfiguration {
|
||||
fn current_and_noncurrent_expiration_lifecycle() -> PersistedLifecycleConfiguration {
|
||||
PersistedLifecycleConfiguration {
|
||||
expiry_updated_at: None,
|
||||
rules: vec![LifecycleRule {
|
||||
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
|
||||
expiration: Some(LifecycleExpiration {
|
||||
rules: vec![PersistedLifecycleRule {
|
||||
status: "Enabled".to_owned(),
|
||||
expiration: Some(PersistedLifecycleExpiration {
|
||||
days: Some(30),
|
||||
..Default::default()
|
||||
}),
|
||||
@@ -10344,7 +10368,7 @@ mod tests {
|
||||
del_marker_expiration: None,
|
||||
filter: None,
|
||||
id: Some("expire-current-and-noncurrent".to_string()),
|
||||
noncurrent_version_expiration: Some(NoncurrentVersionExpiration {
|
||||
noncurrent_version_expiration: Some(PersistedNoncurrentVersionExpiration {
|
||||
noncurrent_days: Some(1),
|
||||
newer_noncurrent_versions: None,
|
||||
}),
|
||||
@@ -10355,12 +10379,12 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
fn all_versions_expiration_lifecycle() -> BucketLifecycleConfiguration {
|
||||
BucketLifecycleConfiguration {
|
||||
fn all_versions_expiration_lifecycle() -> PersistedLifecycleConfiguration {
|
||||
PersistedLifecycleConfiguration {
|
||||
expiry_updated_at: None,
|
||||
rules: vec![LifecycleRule {
|
||||
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
|
||||
expiration: Some(LifecycleExpiration {
|
||||
rules: vec![PersistedLifecycleRule {
|
||||
status: "Enabled".to_owned(),
|
||||
expiration: Some(PersistedLifecycleExpiration {
|
||||
days: Some(1),
|
||||
expired_object_all_versions: Some(true),
|
||||
..Default::default()
|
||||
@@ -10379,29 +10403,29 @@ mod tests {
|
||||
|
||||
fn lock_enabled_without_default_retention() -> ObjectLockConfiguration {
|
||||
ObjectLockConfiguration {
|
||||
object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
|
||||
object_lock_enabled: Some("Enabled".to_owned()),
|
||||
rule: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn lock_enabled_with_default_retention() -> ObjectLockConfiguration {
|
||||
ObjectLockConfiguration {
|
||||
object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
|
||||
object_lock_enabled: Some("Enabled".to_owned()),
|
||||
rule: Some(ObjectLockRule {
|
||||
default_retention: Some(DefaultRetention {
|
||||
days: Some(30),
|
||||
mode: Some(ObjectLockRetentionMode::from_static(ObjectLockRetentionMode::COMPLIANCE)),
|
||||
mode: Some("COMPLIANCE".to_owned()),
|
||||
years: None,
|
||||
}),
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
fn latest_transition_lifecycle() -> BucketLifecycleConfiguration {
|
||||
BucketLifecycleConfiguration {
|
||||
fn latest_transition_lifecycle() -> PersistedLifecycleConfiguration {
|
||||
PersistedLifecycleConfiguration {
|
||||
expiry_updated_at: None,
|
||||
rules: vec![LifecycleRule {
|
||||
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
|
||||
rules: vec![PersistedLifecycleRule {
|
||||
status: "Enabled".to_owned(),
|
||||
expiration: None,
|
||||
abort_incomplete_multipart_upload: None,
|
||||
del_marker_expiration: None,
|
||||
@@ -10410,10 +10434,10 @@ mod tests {
|
||||
noncurrent_version_expiration: None,
|
||||
noncurrent_version_transitions: None,
|
||||
prefix: None,
|
||||
transitions: Some(vec![Transition {
|
||||
transitions: Some(vec![PersistedTransition {
|
||||
days: Some(1),
|
||||
date: None,
|
||||
storage_class: Some(TransitionStorageClass::from_static("WARM")),
|
||||
storage_class: Some("WARM".to_owned()),
|
||||
}]),
|
||||
}],
|
||||
}
|
||||
@@ -12950,8 +12974,9 @@ mod tests {
|
||||
|
||||
for (name, object, lock_config, expected) in cases {
|
||||
let object_opts = lifecycle::object_opts_from_object_info(&object);
|
||||
let evaluator_lock_config = lock_config.clone().map(Arc::new);
|
||||
let batch_event = Evaluator::new(Arc::new(lifecycle.clone()))
|
||||
.with_lock_retention(lock_config.clone().map(Arc::new))
|
||||
.with_lock_retention(evaluator_lock_config)
|
||||
.eval(&[object_opts])
|
||||
.await
|
||||
.expect("batch lifecycle evaluation should succeed")
|
||||
@@ -14208,10 +14233,7 @@ mod tests {
|
||||
let object = current_object_with_metadata(
|
||||
ReplicationStatusType::Completed,
|
||||
HashMap::from([
|
||||
(
|
||||
metadata_keys::OBJECT_LOCK_MODE.to_string(),
|
||||
s3s::dto::ObjectLockRetentionMode::COMPLIANCE.to_string(),
|
||||
),
|
||||
(metadata_keys::OBJECT_LOCK_MODE.to_string(), "COMPLIANCE".to_string()),
|
||||
(metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string(), retain_until),
|
||||
]),
|
||||
);
|
||||
@@ -14223,7 +14245,7 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn restored_copy_expiry_is_not_blocked_by_retention() {
|
||||
let lifecycle = BucketLifecycleConfiguration {
|
||||
let lifecycle = PersistedLifecycleConfiguration {
|
||||
expiry_updated_at: None,
|
||||
rules: Vec::new(),
|
||||
};
|
||||
@@ -14233,10 +14255,7 @@ mod tests {
|
||||
let mut object = current_object_with_metadata(
|
||||
ReplicationStatusType::Completed,
|
||||
HashMap::from([
|
||||
(
|
||||
metadata_keys::OBJECT_LOCK_MODE.to_string(),
|
||||
ObjectLockRetentionMode::COMPLIANCE.to_string(),
|
||||
),
|
||||
(metadata_keys::OBJECT_LOCK_MODE.to_string(), "COMPLIANCE".to_string()),
|
||||
(metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string(), retain_until),
|
||||
]),
|
||||
);
|
||||
@@ -14453,7 +14472,7 @@ mod tests {
|
||||
"removed" => {
|
||||
metadata_sys::delete(&bucket, BUCKET_LIFECYCLE_CONFIG).await.unwrap();
|
||||
}
|
||||
"disabled" => changed.rules[0].status = ExpirationStatus::from_static(ExpirationStatus::DISABLED),
|
||||
"disabled" => changed.rules[0].status = "Disabled".to_owned(),
|
||||
"longer-days" => {
|
||||
changed.rules[0]
|
||||
.noncurrent_version_expiration
|
||||
@@ -14569,7 +14588,7 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
async fn install_test_expiry_policy(ecstore: &Arc<ECStore>, bucket: &str) -> Arc<BucketLifecycleConfiguration> {
|
||||
async fn install_test_expiry_policy(ecstore: &Arc<ECStore>, bucket: &str) -> Arc<PersistedLifecycleConfiguration> {
|
||||
let xml = br#"<LifecycleConfiguration><Rule><ID>expire</ID><Status>Enabled</Status><Filter><Prefix></Prefix></Filter><Expiration><Days>0</Days></Expiration><NoncurrentVersionExpiration><NoncurrentDays>0</NoncurrentDays></NoncurrentVersionExpiration></Rule></LifecycleConfiguration>"#;
|
||||
metadata_sys::update_in(&ecstore.ctx, bucket, BUCKET_LIFECYCLE_CONFIG, xml.to_vec())
|
||||
.await
|
||||
@@ -15262,7 +15281,11 @@ mod tests {
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn tier_free_version_recovery_production_entrypoint_enqueues_seeded_item() {
|
||||
let (disk_paths, ecstore) = setup_test_env().await;
|
||||
let (disk_paths, ecstore) =
|
||||
temp_env::async_with_vars([(super::ENV_TIER_FREE_VERSION_RECOVERY_ENABLED, Some("false"))], async {
|
||||
setup_test_env().await
|
||||
})
|
||||
.await;
|
||||
let tier = format!("RECOVERY{}", Uuid::new_v4().simple()).to_uppercase();
|
||||
crate::services::tier::test_util::register_mock_tier(&ecstore.tier_config_mgr(), &tier).await;
|
||||
let runtime_state = install_unconsumed_runtime_expiry_worker(&ecstore, 1).await;
|
||||
|
||||
@@ -17,7 +17,7 @@ use crate::object_api::ObjectInfo;
|
||||
pub use rustfs_lifecycle::{
|
||||
Event, ExpirationOptions, IlmAction, LIFECYCLE_MALFORMED_XML_ERROR_KIND, Lifecycle, LifecycleCalculate, ObjectOpts,
|
||||
RuleValidate, TRANSITION_COMPLETE, TRANSITION_PENDING, TransitionOptions, abort_incomplete_multipart_upload_due,
|
||||
expected_expiry_time, expiration_action_has_valid_target,
|
||||
expected_expiry_time, expiration_action_has_valid_target, format_lifecycle_date, parse_lifecycle_date,
|
||||
};
|
||||
|
||||
pub fn object_opts_from_object_info(oi: &ObjectInfo) -> ObjectOpts {
|
||||
|
||||
@@ -14,7 +14,8 @@
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use s3s::dto::{BucketLifecycleConfiguration, ObjectLockConfiguration};
|
||||
use rustfs_gateway_types::persistence::PersistedLifecycleConfiguration;
|
||||
use rustfs_gateway_types::persistence::PersistedObjectLockConfiguration as ObjectLockConfiguration;
|
||||
use time::OffsetDateTime;
|
||||
use uuid::Uuid;
|
||||
|
||||
@@ -24,7 +25,7 @@ use crate::error::{Error, Result};
|
||||
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct LifecycleExpiryConfigs {
|
||||
pub(crate) lifecycle: Option<Arc<BucketLifecycleConfiguration>>,
|
||||
pub(crate) lifecycle: Option<Arc<PersistedLifecycleConfiguration>>,
|
||||
pub(crate) object_lock: Option<Arc<ObjectLockConfiguration>>,
|
||||
pub(crate) bucket_incarnation_id: Uuid,
|
||||
pub(crate) table_bucket_enabled: bool,
|
||||
@@ -100,7 +101,7 @@ pub(crate) async fn get_expiry_configs_for_options(
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) async fn get_lifecycle_config(bucket: &str) -> Result<(BucketLifecycleConfiguration, OffsetDateTime)> {
|
||||
pub(crate) async fn get_lifecycle_config(bucket: &str) -> Result<(PersistedLifecycleConfiguration, OffsetDateTime)> {
|
||||
metadata_sys::get_lifecycle_config(bucket).await
|
||||
}
|
||||
|
||||
@@ -110,15 +111,15 @@ mod tests {
|
||||
use crate::bucket::metadata::BucketMetadata;
|
||||
use crate::bucket::metadata_sys::{self, test_support::isolated_store_over_temp_disks};
|
||||
use crate::storage_api_contracts::bucket::MakeBucketOptions;
|
||||
use s3s::dto::{ExpirationStatus, LifecycleExpiration, LifecycleRule};
|
||||
use rustfs_gateway_types::persistence::{PersistedLifecycleExpiration, PersistedLifecycleRule};
|
||||
use serial_test::serial;
|
||||
|
||||
fn lifecycle_config() -> BucketLifecycleConfiguration {
|
||||
BucketLifecycleConfiguration {
|
||||
fn lifecycle_config() -> PersistedLifecycleConfiguration {
|
||||
PersistedLifecycleConfiguration {
|
||||
expiry_updated_at: None,
|
||||
rules: vec![LifecycleRule {
|
||||
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
|
||||
expiration: Some(LifecycleExpiration {
|
||||
rules: vec![PersistedLifecycleRule {
|
||||
status: "Enabled".to_owned(),
|
||||
expiration: Some(PersistedLifecycleExpiration {
|
||||
days: Some(1),
|
||||
..Default::default()
|
||||
}),
|
||||
|
||||
@@ -22,7 +22,7 @@ pub(crate) fn is_object_locked_by_metadata(user_defined: &HashMap<String, String
|
||||
}
|
||||
|
||||
pub(crate) fn check_object_lock_for_deletion_with_config(
|
||||
config: Option<&s3s::dto::ObjectLockConfiguration>,
|
||||
config: Option<&rustfs_gateway_types::persistence::PersistedObjectLockConfiguration>,
|
||||
obj_info: &ObjectInfo,
|
||||
bypass_governance: bool,
|
||||
) -> crate::error::Result<Option<ObjectLockBlockReason>> {
|
||||
|
||||
@@ -82,12 +82,13 @@ mod tests {
|
||||
use crate::bucket::replication::{DeleteReplicationConfigSnapshot, ReplicationObjectBridge};
|
||||
use crate::object_api::{ObjectInfo, ObjectOptions};
|
||||
use crate::storage_api_contracts::object::ObjectToDelete;
|
||||
use rustfs_scanner_metrics::metrics::IlmAction;
|
||||
use s3s::dto::{
|
||||
BucketVersioningStatus, DeleteMarkerReplication, DeleteMarkerReplicationStatus, DeleteReplication,
|
||||
DeleteReplicationStatus, Destination, ReplicationConfiguration, ReplicationRule, ReplicationRuleStatus,
|
||||
VersioningConfiguration,
|
||||
use rustfs_gateway_types::dto::Status;
|
||||
use rustfs_gateway_types::persistence::PersistedVersioningConfiguration as VersioningConfiguration;
|
||||
use rustfs_gateway_types::persistence::{
|
||||
PersistedOptionalReplicationStatus, PersistedReplicationConfiguration, PersistedReplicationDestination,
|
||||
PersistedReplicationRule, PersistedReplicationStatus,
|
||||
};
|
||||
use rustfs_scanner_metrics::metrics::IlmAction;
|
||||
use uuid::Uuid;
|
||||
|
||||
use super::*;
|
||||
@@ -160,27 +161,27 @@ mod tests {
|
||||
for purge_enabled in [false, true] {
|
||||
let snapshot = DeleteReplicationConfigSnapshot::from_configs_for_test(
|
||||
VersioningConfiguration {
|
||||
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
|
||||
status: Some("Enabled".to_owned()),
|
||||
..Default::default()
|
||||
},
|
||||
Some(ReplicationConfiguration {
|
||||
Some(PersistedReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![ReplicationRule {
|
||||
delete_marker_replication: Some(DeleteMarkerReplication {
|
||||
rules: vec![PersistedReplicationRule {
|
||||
delete_marker_replication: Some(PersistedOptionalReplicationStatus {
|
||||
status: Some(if marker_enabled {
|
||||
DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::ENABLED)
|
||||
Status::ENABLED.to_string()
|
||||
} else {
|
||||
DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::DISABLED)
|
||||
Status::DISABLED.to_string()
|
||||
}),
|
||||
}),
|
||||
delete_replication: Some(DeleteReplication {
|
||||
delete_replication: Some(PersistedReplicationStatus {
|
||||
status: if purge_enabled {
|
||||
DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED)
|
||||
Status::ENABLED.to_string()
|
||||
} else {
|
||||
DeleteReplicationStatus::from_static(DeleteReplicationStatus::DISABLED)
|
||||
Status::DISABLED.to_string()
|
||||
},
|
||||
}),
|
||||
destination: Destination {
|
||||
destination: PersistedReplicationDestination {
|
||||
bucket: "arn:rustfs:replication:target".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
@@ -190,7 +191,7 @@ mod tests {
|
||||
prefix: Some(String::new()),
|
||||
priority: Some(1),
|
||||
source_selection_criteria: None,
|
||||
status: ReplicationRuleStatus::from_static(ReplicationRuleStatus::ENABLED),
|
||||
status: Status::ENABLED.to_string(),
|
||||
}],
|
||||
}),
|
||||
);
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use s3s::dto::BucketLifecycleConfiguration;
|
||||
use rustfs_gateway_types::persistence::PersistedLifecycleConfiguration;
|
||||
use tokio::sync::RwLock;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
@@ -52,7 +52,7 @@ pub(crate) fn deployment_id() -> Option<String> {
|
||||
sources::deployment_id()
|
||||
}
|
||||
|
||||
pub(crate) async fn bucket_lifecycle_config(bucket: &str) -> Option<BucketLifecycleConfiguration> {
|
||||
pub(crate) async fn bucket_lifecycle_config(bucket: &str) -> Option<PersistedLifecycleConfiguration> {
|
||||
sources::bucket_lifecycle_config(bucket).await
|
||||
}
|
||||
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
// limitations under the License.
|
||||
|
||||
use super::msgp_decode::{read_msgp_ext8_time, skip_msgp_value, write_msgp_time};
|
||||
use super::object_lock::{ObjectLockApi, ObjectLockStatusExt};
|
||||
use super::object_lock::ObjectLockApi;
|
||||
use super::versioning::VersioningApi;
|
||||
use super::{quota::BucketQuota, target::BucketTargets};
|
||||
use crate::bucket::replication::invalid_replication_config_status_field;
|
||||
@@ -25,12 +25,17 @@ use crate::object_api::WriteCompletion;
|
||||
use crate::runtime::sources as runtime_sources;
|
||||
use crate::store::ECStore;
|
||||
use byteorder::{BigEndian, ByteOrder, LittleEndian};
|
||||
use rustfs_policy::policy::BucketPolicy;
|
||||
use s3s::dto::{
|
||||
AccelerateConfiguration, BucketLifecycleConfiguration, BucketLoggingStatus, BucketVersioningStatus, CORSConfiguration,
|
||||
NotificationConfiguration, ObjectLockConfiguration, PublicAccessBlockConfiguration, ReplicationConfiguration,
|
||||
RequestPaymentConfiguration, ServerSideEncryptionConfiguration, Tagging, VersioningConfiguration, WebsiteConfiguration,
|
||||
use rustfs_gateway_types::cors_tagging::{PersistedCorsConfiguration as CORSConfiguration, PersistedTagging as Tagging};
|
||||
use rustfs_gateway_types::persistence::{
|
||||
PersistedAccelerateConfiguration as AccelerateConfiguration,
|
||||
PersistedBucketEncryptionConfiguration as ServerSideEncryptionConfiguration,
|
||||
PersistedBucketLoggingStatus as BucketLoggingStatus, PersistedLifecycleConfiguration,
|
||||
PersistedNotificationConfiguration as NotificationConfiguration, PersistedObjectLockConfiguration as ObjectLockConfiguration,
|
||||
PersistedPublicAccessBlockConfiguration as PublicAccessBlockConfiguration, PersistedReplicationConfiguration,
|
||||
PersistedRequestPaymentConfiguration as RequestPaymentConfiguration,
|
||||
PersistedVersioningConfiguration as VersioningConfiguration, PersistedWebsiteConfiguration as WebsiteConfiguration,
|
||||
};
|
||||
use rustfs_policy::policy::BucketPolicy;
|
||||
use serde::Serializer;
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::collections::HashMap;
|
||||
@@ -40,23 +45,11 @@ use time::{Date, OffsetDateTime, PrimitiveDateTime, Time as CivilTime, UtcOffset
|
||||
use tracing::error;
|
||||
use uuid::Uuid;
|
||||
|
||||
// The serving-layer DTO impls for the storage-level Object Lock traits live
|
||||
// here because this module owns the persisted `ObjectLockConfiguration`
|
||||
// during the s3s ratchet migration (rustfs/backlog#1842).
|
||||
// The persisted Object Lock configuration answers the storage-level trait
|
||||
// here because this module owns the persisted `ObjectLockConfiguration`.
|
||||
impl ObjectLockApi for ObjectLockConfiguration {
|
||||
fn enabled(&self) -> bool {
|
||||
self.object_lock_enabled
|
||||
.as_ref()
|
||||
.is_some_and(|v| v.as_str() == s3s::dto::ObjectLockEnabled::ENABLED)
|
||||
}
|
||||
}
|
||||
|
||||
impl ObjectLockStatusExt for s3s::dto::ObjectLockLegalHoldStatus {
|
||||
fn valid(&self) -> bool {
|
||||
matches!(
|
||||
self.as_str(),
|
||||
s3s::dto::ObjectLockLegalHoldStatus::ON | s3s::dto::ObjectLockLegalHoldStatus::OFF
|
||||
)
|
||||
self.object_lock_enabled.as_deref() == Some("Enabled")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -439,6 +432,8 @@ pub struct BucketMetadata {
|
||||
pub bucket_incarnation_id: Uuid,
|
||||
pub(crate) bucket_incarnation_sidecar: bool,
|
||||
pub(crate) bucket_creation_committed: bool,
|
||||
// A legacy default is not proof of an interrupted creation.
|
||||
pub(crate) bucket_creation_commit_record_present: bool,
|
||||
pub policy_config_json: Vec<u8>,
|
||||
pub notification_config_xml: Vec<u8>,
|
||||
pub lifecycle_config_xml: Vec<u8>,
|
||||
@@ -487,13 +482,13 @@ pub struct BucketMetadata {
|
||||
|
||||
pub policy_config: Option<BucketPolicy>,
|
||||
pub notification_config: Option<NotificationConfiguration>,
|
||||
pub lifecycle_config: Option<BucketLifecycleConfiguration>,
|
||||
pub lifecycle_config: Option<PersistedLifecycleConfiguration>,
|
||||
pub object_lock_config: Option<ObjectLockConfiguration>,
|
||||
pub versioning_config: Option<VersioningConfiguration>,
|
||||
pub sse_config: Option<ServerSideEncryptionConfiguration>,
|
||||
pub tagging_config: Option<Tagging>,
|
||||
pub quota_config: Option<BucketQuota>,
|
||||
pub replication_config: Option<ReplicationConfiguration>,
|
||||
pub replication_config: Option<PersistedReplicationConfiguration>,
|
||||
pub bucket_target_config: Option<BucketTargets>,
|
||||
pub bucket_target_config_meta: Option<HashMap<String, String>>,
|
||||
pub cors_config: Option<CORSConfiguration>,
|
||||
@@ -514,6 +509,7 @@ impl Default for BucketMetadata {
|
||||
bucket_incarnation_id: Uuid::nil(),
|
||||
bucket_incarnation_sidecar: false,
|
||||
bucket_creation_committed: false,
|
||||
bucket_creation_commit_record_present: false,
|
||||
policy_config_json: Default::default(),
|
||||
notification_config_xml: Default::default(),
|
||||
lifecycle_config_xml: Default::default(),
|
||||
@@ -752,7 +748,10 @@ impl BucketMetadata {
|
||||
"TableBucketConfigUpdatedAt" => self.table_bucket_config_updated_at = read_msgp_time_value(rd)?,
|
||||
"DurabilityConfigUpdatedAt" => self.durability_config_updated_at = read_msgp_time_value(rd)?,
|
||||
"OnDemandMigrationConfigUpdatedAt" => self.on_demand_migration_config_updated_at = read_msgp_time_value(rd)?,
|
||||
"BucketCreationCommitted" => self.bucket_creation_committed = read_msgp_bool(rd)?,
|
||||
"BucketCreationCommitted" => {
|
||||
self.bucket_creation_committed = read_msgp_bool(rd)?;
|
||||
self.bucket_creation_commit_record_present = true;
|
||||
}
|
||||
other => {
|
||||
tracing::debug!(field = %other, "BucketMetadata decode_from: skipping unknown field");
|
||||
skip_msgp_value(rd)?;
|
||||
@@ -765,8 +764,10 @@ impl BucketMetadata {
|
||||
|
||||
/// Encode to msgp bytes. Field order follows MinIO BucketMetadata for compatibility.
|
||||
pub fn encode_to<W: Write>(&self, wr: &mut W) -> Result<()> {
|
||||
// Map size: MinIO fields (25) + RustFS extensions (22)
|
||||
let map_len: u32 = 47;
|
||||
// Preserve the absence of a legacy commit record across config rewrites.
|
||||
let commit_record_present = self.bucket_creation_commit_record_present || self.bucket_creation_committed;
|
||||
// Map size: MinIO fields (25) + RustFS extensions (21) + optional commit record.
|
||||
let map_len: u32 = 46 + u32::from(commit_record_present);
|
||||
rmp::encode::write_map_len(wr, map_len)?;
|
||||
|
||||
// MinIO field order (same as Go struct)
|
||||
@@ -847,8 +848,10 @@ impl BucketMetadata {
|
||||
write_msgp_time(wr, self.durability_config_updated_at)?;
|
||||
rmp::encode::write_str(wr, "OnDemandMigrationConfigUpdatedAt")?;
|
||||
write_msgp_time(wr, self.on_demand_migration_config_updated_at)?;
|
||||
rmp::encode::write_str(wr, "BucketCreationCommitted")?;
|
||||
rmp::encode::write_bool(wr, self.bucket_creation_committed)?;
|
||||
if commit_record_present {
|
||||
rmp::encode::write_str(wr, "BucketCreationCommitted")?;
|
||||
rmp::encode::write_bool(wr, self.bucket_creation_committed)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -1018,9 +1021,11 @@ impl BucketMetadata {
|
||||
None
|
||||
} else {
|
||||
let config = deserialize::<VersioningConfiguration>(&data)?;
|
||||
if config.status.as_ref().is_some_and(|status| {
|
||||
!matches!(status.as_str(), BucketVersioningStatus::ENABLED | BucketVersioningStatus::SUSPENDED)
|
||||
}) {
|
||||
if config
|
||||
.status
|
||||
.as_ref()
|
||||
.is_some_and(|status| !matches!(status.as_str(), "Enabled" | "Suspended"))
|
||||
{
|
||||
return Err(Error::other("bucket versioning configuration has an invalid status"));
|
||||
}
|
||||
Some(config)
|
||||
@@ -1033,7 +1038,7 @@ impl BucketMetadata {
|
||||
let config = if data.is_empty() {
|
||||
None
|
||||
} else {
|
||||
let config = deserialize::<ReplicationConfiguration>(&data)?;
|
||||
let config = deserialize::<PersistedReplicationConfiguration>(&data)?;
|
||||
if let Some(field) = invalid_replication_config_status_field(&config) {
|
||||
return Err(Error::other(format!("replication field {field} has an invalid status")));
|
||||
}
|
||||
@@ -1227,8 +1232,8 @@ impl BucketMetadata {
|
||||
}
|
||||
self.lifecycle_config = None;
|
||||
if !self.lifecycle_config_xml.is_empty()
|
||||
&& let Err(e) =
|
||||
deserialize::<BucketLifecycleConfiguration>(&self.lifecycle_config_xml).map(|c| self.lifecycle_config = Some(c))
|
||||
&& let Err(e) = deserialize::<PersistedLifecycleConfiguration>(&self.lifecycle_config_xml)
|
||||
.map(|c| self.lifecycle_config = Some(c))
|
||||
{
|
||||
tracing::warn!(
|
||||
event = "bucket_metadata_parse_failed",
|
||||
@@ -1314,8 +1319,8 @@ impl BucketMetadata {
|
||||
}
|
||||
self.replication_config = None;
|
||||
if !self.replication_config_xml.is_empty()
|
||||
&& let Err(e) =
|
||||
deserialize::<ReplicationConfiguration>(&self.replication_config_xml).map(|c| self.replication_config = Some(c))
|
||||
&& let Err(e) = deserialize::<PersistedReplicationConfiguration>(&self.replication_config_xml)
|
||||
.map(|c| self.replication_config = Some(c))
|
||||
{
|
||||
tracing::warn!(
|
||||
event = "bucket_metadata_parse_failed",
|
||||
@@ -1503,11 +1508,7 @@ pub(crate) async fn load_bucket_metadata_parse_with_presence(
|
||||
bucket: &str,
|
||||
parse: bool,
|
||||
) -> Result<(BucketMetadata, bool)> {
|
||||
let (metadata_res, incarnation_res) = tokio::join!(
|
||||
Box::pin(read_bucket_metadata(api.clone(), bucket)),
|
||||
Box::pin(load_bucket_incarnation(api.clone(), bucket)),
|
||||
);
|
||||
let (mut bm, persisted) = match metadata_res {
|
||||
let (mut bm, persisted) = match read_bucket_metadata(api.clone(), bucket).await {
|
||||
Ok(res) => (res, true),
|
||||
Err(err) => {
|
||||
if err != Error::ConfigNotFound {
|
||||
@@ -1518,7 +1519,7 @@ pub(crate) async fn load_bucket_metadata_parse_with_presence(
|
||||
}
|
||||
};
|
||||
|
||||
let incarnation = incarnation_res?;
|
||||
let incarnation = load_bucket_incarnation(api.clone(), bucket).await?;
|
||||
if persisted {
|
||||
if bm.name.is_empty() {
|
||||
bm.name = bucket.to_string();
|
||||
@@ -1695,6 +1696,13 @@ mod test {
|
||||
let mut bm = BucketMetadata::unmarshal(&body[4..]).expect("unmarshal recovered blob");
|
||||
assert_eq!(bm.name, "interop");
|
||||
assert!(!bm.bucket_creation_committed, "legacy metadata must default to uncommitted");
|
||||
assert!(!bm.bucket_creation_commit_record_present, "legacy metadata has no creation-commit record");
|
||||
let rewritten = BucketMetadata::unmarshal(&bm.marshal_msg().expect("rewrite legacy metadata"))
|
||||
.expect("read rewritten legacy metadata");
|
||||
assert!(
|
||||
!rewritten.bucket_creation_commit_record_present,
|
||||
"rewriting legacy metadata must not fabricate a creation intent"
|
||||
);
|
||||
bm.parse_all_configs().expect("parse recovered configs");
|
||||
assert!(bm.lifecycle_config.is_some());
|
||||
}
|
||||
@@ -1712,6 +1720,7 @@ mod test {
|
||||
|
||||
assert_eq!(bm.name, new.name);
|
||||
assert!(new.bucket_creation_committed);
|
||||
assert!(new.bucket_creation_commit_record_present);
|
||||
assert!(!bm.bucket_incarnation_id.is_nil());
|
||||
assert_eq!(bm.bucket_incarnation_id, new.bucket_incarnation_id);
|
||||
}
|
||||
@@ -1816,17 +1825,15 @@ mod test {
|
||||
|
||||
#[test]
|
||||
fn object_locking_requires_lock_metadata_not_plain_versioning() {
|
||||
use s3s::dto::ObjectLockEnabled;
|
||||
|
||||
let mut bm = BucketMetadata::new("test-bucket");
|
||||
bm.versioning_config = Some(VersioningConfiguration {
|
||||
status: Some(s3s::dto::BucketVersioningStatus::from_static("Enabled")),
|
||||
status: Some("Enabled".to_owned()),
|
||||
..Default::default()
|
||||
});
|
||||
assert!(!bm.object_locking());
|
||||
|
||||
bm.object_lock_config = Some(ObjectLockConfiguration {
|
||||
object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
|
||||
object_lock_enabled: Some("Enabled".to_owned()),
|
||||
..Default::default()
|
||||
});
|
||||
assert!(bm.object_locking());
|
||||
|
||||
@@ -30,14 +30,18 @@ use crate::storage_api_contracts::heal::HealOperations as _;
|
||||
use crate::storage_api_contracts::namespace::NamespaceLocking as _;
|
||||
use crate::store::{ECStore, await_bucket_namespace_operation};
|
||||
use futures::future::join_all;
|
||||
use rustfs_gateway_types::cors_tagging::{PersistedCorsConfiguration as CORSConfiguration, PersistedTagging as Tagging};
|
||||
use rustfs_gateway_types::persistence::{
|
||||
PersistedAccelerateConfiguration as AccelerateConfiguration,
|
||||
PersistedBucketEncryptionConfiguration as ServerSideEncryptionConfiguration,
|
||||
PersistedBucketLoggingStatus as BucketLoggingStatus, PersistedLifecycleConfiguration,
|
||||
PersistedNotificationConfiguration as NotificationConfiguration, PersistedObjectLockConfiguration as ObjectLockConfiguration,
|
||||
PersistedPublicAccessBlockConfiguration as PublicAccessBlockConfiguration, PersistedReplicationConfiguration,
|
||||
PersistedRequestPaymentConfiguration as RequestPaymentConfiguration,
|
||||
PersistedVersioningConfiguration as VersioningConfiguration, PersistedWebsiteConfiguration as WebsiteConfiguration,
|
||||
};
|
||||
use rustfs_heal_contracts::heal_channel::HealOpts;
|
||||
use rustfs_policy::policy::BucketPolicy;
|
||||
use s3s::dto::ReplicationConfiguration;
|
||||
use s3s::dto::{
|
||||
AccelerateConfiguration, BucketLifecycleConfiguration, BucketLoggingStatus, CORSConfiguration, NotificationConfiguration,
|
||||
ObjectLockConfiguration, ObjectLockEnabled, ObjectLockRetentionMode, PublicAccessBlockConfiguration,
|
||||
RequestPaymentConfiguration, ServerSideEncryptionConfiguration, Tagging, VersioningConfiguration, WebsiteConfiguration,
|
||||
};
|
||||
use std::collections::HashSet;
|
||||
use std::time::Duration;
|
||||
use std::{
|
||||
@@ -168,7 +172,7 @@ pub(crate) fn object_lock_config_state_from_authoritative_metadata(bm: &BucketMe
|
||||
if bm.lock_enabled {
|
||||
return Ok(ObjectLockConfigState::Configured {
|
||||
config: ObjectLockConfiguration {
|
||||
object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
|
||||
object_lock_enabled: Some("Enabled".to_owned()),
|
||||
rule: None,
|
||||
},
|
||||
updated_at: bm.object_lock_config_updated_at,
|
||||
@@ -212,17 +216,10 @@ pub(crate) fn configured_object_lock_state_for_tests(
|
||||
) -> ObjectLockConfigState {
|
||||
ObjectLockConfigState::Configured {
|
||||
config: ObjectLockConfiguration {
|
||||
object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
|
||||
rule: Some(s3s::dto::ObjectLockRule {
|
||||
default_retention: Some(s3s::dto::DefaultRetention {
|
||||
mode: Some(s3s::dto::ObjectLockRetentionMode::from_static(match mode {
|
||||
crate::bucket::object_lock::types::RetentionMode::Governance => {
|
||||
s3s::dto::ObjectLockRetentionMode::GOVERNANCE
|
||||
}
|
||||
crate::bucket::object_lock::types::RetentionMode::Compliance => {
|
||||
s3s::dto::ObjectLockRetentionMode::COMPLIANCE
|
||||
}
|
||||
})),
|
||||
object_lock_enabled: Some("Enabled".to_owned()),
|
||||
rule: Some(rustfs_gateway_types::persistence::PersistedObjectLockRule {
|
||||
default_retention: Some(rustfs_gateway_types::persistence::PersistedDefaultRetention {
|
||||
mode: Some(mode.as_str().to_owned()),
|
||||
days: Some(days),
|
||||
years: None,
|
||||
}),
|
||||
@@ -233,7 +230,7 @@ pub(crate) fn configured_object_lock_state_for_tests(
|
||||
}
|
||||
|
||||
fn validate_authoritative_object_lock_config(config: &ObjectLockConfiguration) -> Result<()> {
|
||||
if config.object_lock_enabled.as_ref().map(ObjectLockEnabled::as_str) != Some(ObjectLockEnabled::ENABLED) {
|
||||
if config.object_lock_enabled.as_deref() != Some("Enabled") {
|
||||
return Err(Error::other("persisted bucket Object Lock enabled state is invalid"));
|
||||
}
|
||||
|
||||
@@ -246,7 +243,7 @@ fn validate_authoritative_object_lock_config(config: &ObjectLockConfiguration) -
|
||||
if !retention
|
||||
.mode
|
||||
.as_ref()
|
||||
.is_some_and(|mode| matches!(mode.as_str(), ObjectLockRetentionMode::COMPLIANCE | ObjectLockRetentionMode::GOVERNANCE))
|
||||
.is_some_and(|mode| matches!(mode.as_str(), "COMPLIANCE" | "GOVERNANCE"))
|
||||
{
|
||||
return Err(Error::other("persisted bucket Object Lock retention mode is invalid"));
|
||||
}
|
||||
@@ -683,7 +680,6 @@ pub(crate) async fn update_validated_in<F>(
|
||||
bucket: &str,
|
||||
config_file: &str,
|
||||
data: Vec<u8>,
|
||||
expected_incarnation_id: Option<Uuid>,
|
||||
validate: F,
|
||||
) -> Result<OffsetDateTime>
|
||||
where
|
||||
@@ -691,7 +687,7 @@ where
|
||||
{
|
||||
Box::pin(async {
|
||||
let sys = require_bucket_metadata_sys_in(ctx)?;
|
||||
let guard = acquire_config_write_guard_for_incarnation(sys.clone(), bucket, expected_incarnation_id).await?;
|
||||
let guard = acquire_config_write_guard(sys.clone(), bucket).await?;
|
||||
validate()?;
|
||||
update_under_config_write_guard(sys, &guard, config_file, data, None).await
|
||||
})
|
||||
@@ -1396,7 +1392,7 @@ pub async fn get_public_access_block_config(bucket: &str) -> Result<(PublicAcces
|
||||
bucket_meta_sys.get_public_access_block_config(bucket).await
|
||||
}
|
||||
|
||||
pub async fn get_lifecycle_config(bucket: &str) -> Result<(BucketLifecycleConfiguration, OffsetDateTime)> {
|
||||
pub async fn get_lifecycle_config(bucket: &str) -> Result<(PersistedLifecycleConfiguration, OffsetDateTime)> {
|
||||
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
|
||||
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
|
||||
|
||||
@@ -1436,7 +1432,6 @@ pub(crate) async fn get_object_lock_config_state_in(
|
||||
|
||||
/// Re-read the Object Lock state and bucket incarnation from the same
|
||||
/// authoritative metadata blob while the caller holds the transaction lock.
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
pub(crate) async fn get_object_lock_config_and_incarnation_from_disk_in(
|
||||
ctx: &crate::runtime::instance::InstanceContext,
|
||||
bucket: &str,
|
||||
@@ -1455,27 +1450,6 @@ pub(crate) async fn get_object_lock_config_and_incarnation_from_disk_with_guard_
|
||||
get_delete_config_from_disk_in(ctx, bucket, guard, None).await
|
||||
}
|
||||
|
||||
#[allow(clippy::type_complexity)]
|
||||
pub(crate) async fn get_object_lock_and_quota_config_from_disk_with_guard_in(
|
||||
ctx: &crate::runtime::instance::InstanceContext,
|
||||
bucket: &str,
|
||||
guard: Option<&rustfs_lock::NamespaceLockGuard>,
|
||||
) -> Result<(
|
||||
ObjectLockConfigState,
|
||||
Uuid,
|
||||
OffsetDateTime,
|
||||
std::result::Result<(Option<BucketQuota>, Uuid, OffsetDateTime), String>,
|
||||
)> {
|
||||
let (state, incarnation_id, updated_at, quota_res) =
|
||||
get_delete_and_quota_config_from_disk_in(ctx, bucket, guard, None, true).await?;
|
||||
Ok((
|
||||
state,
|
||||
incarnation_id,
|
||||
updated_at,
|
||||
quota_res.unwrap_or(Ok((None, incarnation_id, updated_at))),
|
||||
))
|
||||
}
|
||||
|
||||
/// Validate queued expiration against the same fresh metadata used for Object
|
||||
/// Lock admission. The caller holds the transaction read lock through commit,
|
||||
/// so a successful lifecycle update cannot be followed by an old-policy delete.
|
||||
@@ -1485,24 +1459,6 @@ pub(crate) async fn get_delete_config_from_disk_in(
|
||||
guard: Option<&rustfs_lock::NamespaceLockGuard>,
|
||||
expiry_policy: Option<&rustfs_lifecycle::ExpiryPolicySnapshot>,
|
||||
) -> Result<(ObjectLockConfigState, Uuid, OffsetDateTime)> {
|
||||
let (state, incarnation_id, updated_at, _) =
|
||||
get_delete_and_quota_config_from_disk_in(ctx, bucket, guard, expiry_policy, false).await?;
|
||||
Ok((state, incarnation_id, updated_at))
|
||||
}
|
||||
|
||||
#[allow(clippy::type_complexity)]
|
||||
async fn get_delete_and_quota_config_from_disk_in(
|
||||
ctx: &crate::runtime::instance::InstanceContext,
|
||||
bucket: &str,
|
||||
guard: Option<&rustfs_lock::NamespaceLockGuard>,
|
||||
expiry_policy: Option<&rustfs_lifecycle::ExpiryPolicySnapshot>,
|
||||
include_quota: bool,
|
||||
) -> Result<(
|
||||
ObjectLockConfigState,
|
||||
Uuid,
|
||||
OffsetDateTime,
|
||||
Option<std::result::Result<(Option<BucketQuota>, Uuid, OffsetDateTime), String>>,
|
||||
)> {
|
||||
let bucket_meta_sys_lock = bucket_metadata_sys_of(ctx)?;
|
||||
let bucket_meta_sys = bucket_meta_sys_lock.read().await.clone();
|
||||
|
||||
@@ -1518,7 +1474,7 @@ async fn get_delete_and_quota_config_from_disk_in(
|
||||
.await?
|
||||
}
|
||||
};
|
||||
match &authority {
|
||||
match authority {
|
||||
BucketMetadataAuthority::Authoritative(metadata)
|
||||
if metadata.bucket_incarnation_sidecar && !metadata.bucket_incarnation_id.is_nil() =>
|
||||
{
|
||||
@@ -1530,11 +1486,11 @@ async fn get_delete_and_quota_config_from_disk_in(
|
||||
}
|
||||
}
|
||||
let object_lock_config_updated_at = metadata.object_lock_config_updated_at;
|
||||
let state = object_lock_config_state_from_authoritative_metadata(metadata)?;
|
||||
let incarnation_id = metadata.bucket_incarnation_id;
|
||||
let quota_res = include_quota
|
||||
.then(|| quota_config_and_incarnation_from_authority(bucket, authority).map_err(|err| err.to_string()));
|
||||
Ok((state, incarnation_id, object_lock_config_updated_at, quota_res))
|
||||
Ok((
|
||||
object_lock_config_state_from_authoritative_metadata(&metadata)?,
|
||||
metadata.bucket_incarnation_id,
|
||||
object_lock_config_updated_at,
|
||||
))
|
||||
}
|
||||
BucketMetadataAuthority::Authoritative(_) => {
|
||||
Err(Error::other(format!("bucket incarnation metadata is not authoritative: {bucket}")))
|
||||
@@ -1605,23 +1561,6 @@ pub(crate) async fn get_quota_config_and_incarnation_from_disk_for_options_in(
|
||||
return Err(Error::other("quota snapshot metadata transaction fence was released"));
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
let has_injected_error = sys.object_lock_disk_read_errors.read().await.contains(bucket);
|
||||
#[cfg(not(test))]
|
||||
let has_injected_error = false;
|
||||
if !ctx.lock_manager().is_disabled()
|
||||
&& !has_injected_error
|
||||
&& let Some(fresh_quota) = snapshot.take_fresh_put_quota()
|
||||
{
|
||||
if snapshot_guard.is_lock_lost()
|
||||
|| snapshot_guard.is_released()
|
||||
|| !snapshot.is_valid_for_destructive_put(store.id, bucket, opts.expected_bucket_incarnation_id.unwrap_or_default())
|
||||
{
|
||||
return Err(Error::other("quota snapshot fence or bucket incarnation changed during the disk read"));
|
||||
}
|
||||
return fresh_quota.map_err(Error::other);
|
||||
}
|
||||
|
||||
// The snapshot carries the caller's lifecycle fence, excluding deletion.
|
||||
// Its transaction guard excludes config writes, so a same-name object
|
||||
// owner need not recursively acquire the bucket namespace here.
|
||||
@@ -1672,7 +1611,7 @@ pub(crate) async fn get_cached_quota_config_and_incarnation_in(
|
||||
bucket_meta_sys.cached_quota_config_and_incarnation(bucket).await
|
||||
}
|
||||
|
||||
pub async fn get_replication_config(bucket: &str) -> Result<(ReplicationConfiguration, OffsetDateTime)> {
|
||||
pub async fn get_replication_config(bucket: &str) -> Result<(PersistedReplicationConfiguration, OffsetDateTime)> {
|
||||
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
|
||||
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
|
||||
|
||||
@@ -1682,7 +1621,7 @@ pub async fn get_replication_config(bucket: &str) -> Result<(ReplicationConfigur
|
||||
pub(crate) async fn get_replication_config_in(
|
||||
ctx: &crate::runtime::instance::InstanceContext,
|
||||
bucket: &str,
|
||||
) -> Result<(ReplicationConfiguration, OffsetDateTime)> {
|
||||
) -> Result<(PersistedReplicationConfiguration, OffsetDateTime)> {
|
||||
let bucket_meta_sys_lock = bucket_metadata_sys_of(ctx)?;
|
||||
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
|
||||
|
||||
@@ -1808,22 +1747,27 @@ struct MetadataPublishGuard {
|
||||
_guard: tokio::sync::OwnedMutexGuard<MetadataPublishLockState>,
|
||||
}
|
||||
|
||||
static LOCAL_META_MUTATION_GEN: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(1);
|
||||
|
||||
pub(crate) fn bump_local_meta_mutation_gen() {
|
||||
LOCAL_META_MUTATION_GEN.fetch_add(1, std::sync::atomic::Ordering::Release);
|
||||
#[derive(Debug, Clone)]
|
||||
pub(crate) struct PendingBucketCreationProof {
|
||||
pub(crate) generation: Uuid,
|
||||
pub(crate) intent: Vec<u8>,
|
||||
pub(crate) versioning_requested: bool,
|
||||
pub(crate) witnesses: Vec<crate::disk::os::NativeBucketCreationWitness>,
|
||||
}
|
||||
|
||||
pub(crate) fn current_local_meta_mutation_gen() -> u64 {
|
||||
LOCAL_META_MUTATION_GEN.load(std::sync::atomic::Ordering::Acquire)
|
||||
impl PendingBucketCreationProof {
|
||||
pub(crate) fn matches_metadata(&self, metadata: &BucketMetadata) -> Result<bool> {
|
||||
Ok(!metadata.bucket_creation_committed
|
||||
&& metadata.bucket_incarnation_sidecar
|
||||
&& metadata.bucket_incarnation_id == self.generation
|
||||
&& metadata.marshal_msg()? == self.intent)
|
||||
}
|
||||
}
|
||||
|
||||
type LocalAuthorityCache = Arc<StdMutex<HashMap<String, (u64, Arc<BucketMetadata>)>>>;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct BucketMetadataSys {
|
||||
metadata_map: Arc<RwLock<HashMap<String, Arc<BucketMetadata>>>>,
|
||||
local_authority_cache: LocalAuthorityCache,
|
||||
pending_bucket_creations: Arc<RwLock<HashMap<String, PendingBucketCreationProof>>>,
|
||||
/// Serializes metadata-map commits and their derived cache updates for one
|
||||
/// bucket. Namespace locks, when present, are acquired before this lock.
|
||||
metadata_publish_locks: Arc<MetadataPublishLockRegistry>,
|
||||
@@ -1867,7 +1811,7 @@ impl BucketMetadataSys {
|
||||
pub fn new(api: Arc<ECStore>) -> Self {
|
||||
Self {
|
||||
metadata_map: Arc::new(RwLock::new(HashMap::new())),
|
||||
local_authority_cache: Arc::new(StdMutex::new(HashMap::new())),
|
||||
pending_bucket_creations: Arc::new(RwLock::new(HashMap::new())),
|
||||
metadata_publish_locks: Arc::new(MetadataPublishLockRegistry {
|
||||
locks: StdMutex::new(HashMap::new()),
|
||||
}),
|
||||
@@ -1898,17 +1842,83 @@ impl BucketMetadataSys {
|
||||
.expect("bucket metadata object store should still be live")
|
||||
}
|
||||
|
||||
fn object_store_if_live(&self) -> Option<Arc<ECStore>> {
|
||||
self.api.upgrade()
|
||||
pub(crate) async fn pending_bucket_creation(&self, bucket: &str) -> Option<PendingBucketCreationProof> {
|
||||
self.pending_bucket_creations.read().await.get(bucket).cloned()
|
||||
}
|
||||
|
||||
/// Whether the backing `ECStore` has been dropped.
|
||||
///
|
||||
/// Used by `InstanceContext::clear_bucket_metadata_sys_if_stale` during
|
||||
/// `ECStore::drop` so a shared bootstrap context does not retain a
|
||||
/// `BucketMetadataSys` whose weak handle is dangling.
|
||||
pub(crate) fn is_store_stale(&self) -> bool {
|
||||
self.api.upgrade().is_none()
|
||||
pub(crate) async fn grant_pending_bucket_creation(&self, bucket: &str, proof: PendingBucketCreationProof) {
|
||||
self.pending_bucket_creations.write().await.insert(bucket.to_owned(), proof);
|
||||
}
|
||||
|
||||
pub(crate) async fn revoke_pending_bucket_creation(&self, bucket: &str) {
|
||||
self.pending_bucket_creations.write().await.remove(bucket);
|
||||
}
|
||||
|
||||
pub(crate) async fn consume_pending_bucket_creation(
|
||||
&self,
|
||||
bucket: &str,
|
||||
metadata: &BucketMetadata,
|
||||
witnesses: &[crate::disk::os::NativeBucketCreationWitness],
|
||||
) -> Result<bool> {
|
||||
let mut pending = self.pending_bucket_creations.write().await;
|
||||
let Some(proof) = pending.get(bucket) else {
|
||||
return Ok(false);
|
||||
};
|
||||
if !proof.matches_metadata(metadata)?
|
||||
|| proof.witnesses.len() != witnesses.len()
|
||||
|| !proof
|
||||
.witnesses
|
||||
.iter()
|
||||
.zip(witnesses)
|
||||
.all(|(old, current)| old.matches(current))
|
||||
|| !crate::disk::os::consume_native_creation_witnesses(witnesses)
|
||||
{
|
||||
return Ok(false);
|
||||
}
|
||||
pending.remove(bucket);
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn refresh_bucket_for_test(&self, bucket: &str) -> Result<()> {
|
||||
let expected = self.metadata_map.read().await.get(bucket).cloned();
|
||||
let lock = self.object_store().new_ns_lock(bucket, bucket).await?;
|
||||
let guard = lock.get_read_lock(crate::set_disk::get_lock_acquire_timeout()).await?;
|
||||
self.load_bucket_under_namespace(bucket, MetadataLoadMode::Refresh, expected.as_ref(), &guard)
|
||||
.await
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn publish_persisted_metadata_for_test(&self, bucket: &str) -> Result<()> {
|
||||
let expected = self.metadata_map.read().await.get(bucket).cloned();
|
||||
let lock = self.object_store().new_ns_lock(bucket, bucket).await?;
|
||||
let guard = lock.get_read_lock(crate::set_disk::get_lock_acquire_timeout()).await?;
|
||||
let (metadata, persisted) = await_bucket_namespace_operation(
|
||||
Some(&guard),
|
||||
bucket,
|
||||
"test cached metadata load",
|
||||
load_bucket_metadata_parse_with_presence(self.object_store(), bucket, true),
|
||||
)
|
||||
.await?;
|
||||
self.publish_if_unchanged(bucket, expected.as_ref(), metadata, persisted, &guard)
|
||||
.await
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn migrate_bucket_creation_commit_for_test(&self, bucket: &str) -> Result<()> {
|
||||
self.migrate_bucket_creation_commit(bucket).await.map(|_| ())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn legacy_migration_reports_missing_bucket_for_test(&self, bucket: &str) -> Result<bool> {
|
||||
Ok(matches!(
|
||||
self.migrate_legacy_metadata(bucket).await?,
|
||||
BucketMetadataAuthority::MissingBucket
|
||||
))
|
||||
}
|
||||
|
||||
fn object_store_if_live(&self) -> Option<Arc<ECStore>> {
|
||||
self.api.upgrade()
|
||||
}
|
||||
|
||||
fn metadata_publish_lock(&self, bucket: &str) -> Arc<Mutex<MetadataPublishLockState>> {
|
||||
@@ -2097,6 +2107,7 @@ impl BucketMetadataSys {
|
||||
expected: Option<&Arc<BucketMetadata>>,
|
||||
namespace_guard: &rustfs_lock::NamespaceLockGuard,
|
||||
) -> Result<()> {
|
||||
self.revoke_pending_bucket_creation(bucket).await;
|
||||
if !await_bucket_namespace_operation(
|
||||
Some(namespace_guard),
|
||||
bucket,
|
||||
@@ -2180,6 +2191,7 @@ impl BucketMetadataSys {
|
||||
persisted: bool,
|
||||
namespace_guard: &rustfs_lock::NamespaceLockGuard,
|
||||
) -> Result<()> {
|
||||
self.revoke_pending_bucket_creation(bucket).await;
|
||||
if !persisted {
|
||||
let _publish_guard = self
|
||||
.lock_metadata_publish(bucket, namespace_guard, "refreshed bucket metadata absence publish")
|
||||
@@ -2241,7 +2253,7 @@ impl BucketMetadataSys {
|
||||
}
|
||||
|
||||
pub async fn set(&self, bucket: String, bm: Arc<BucketMetadata>) {
|
||||
bump_local_meta_mutation_gen();
|
||||
self.revoke_pending_bucket_creation(&bucket).await;
|
||||
if !is_meta_bucketname(&bucket) {
|
||||
let publish_lock = self.metadata_publish_lock(&bucket);
|
||||
let _publish_guard = publish_lock.lock().await;
|
||||
@@ -2260,10 +2272,10 @@ impl BucketMetadataSys {
|
||||
///
|
||||
/// Returns `true` if an entry was present. Reserved meta buckets are ignored.
|
||||
pub async fn remove(&self, bucket: &str) -> bool {
|
||||
bump_local_meta_mutation_gen();
|
||||
if is_meta_bucketname(bucket) {
|
||||
return false;
|
||||
}
|
||||
self.revoke_pending_bucket_creation(bucket).await;
|
||||
let publish_lock = self.metadata_publish_lock(bucket);
|
||||
let _publish_guard = publish_lock.lock().await;
|
||||
let mut map = self.metadata_map.write().await;
|
||||
@@ -2281,7 +2293,7 @@ impl BucketMetadataSys {
|
||||
}
|
||||
|
||||
async fn _reset(&mut self) {
|
||||
bump_local_meta_mutation_gen();
|
||||
self.pending_bucket_creations.write().await.clear();
|
||||
let mut map = self.metadata_map.write().await;
|
||||
map.clear();
|
||||
drop(map);
|
||||
@@ -2420,6 +2432,7 @@ impl BucketMetadataSys {
|
||||
/// server's metadata never leaks into the ambient (first) instance.
|
||||
pub(crate) async fn persist_and_set(&self, bm: BucketMetadata) -> Result<()> {
|
||||
let mut bm = bm;
|
||||
self.revoke_pending_bucket_creation(&bm.name).await;
|
||||
bm.save_with_store(self.object_store()).await?;
|
||||
|
||||
self.set(bm.name.clone(), Arc::new(bm)).await;
|
||||
@@ -2428,6 +2441,7 @@ impl BucketMetadataSys {
|
||||
}
|
||||
|
||||
async fn persist_new_and_set(&self, mut bm: BucketMetadata) -> Result<()> {
|
||||
self.revoke_pending_bucket_creation(&bm.name).await;
|
||||
bm.bucket_creation_committed = true;
|
||||
bm.save_with_store_committed(self.object_store()).await?;
|
||||
save_bucket_incarnation(self.object_store(), &bm.name, bm.bucket_incarnation_id).await?;
|
||||
@@ -2437,7 +2451,9 @@ impl BucketMetadataSys {
|
||||
}
|
||||
|
||||
async fn persist_new_bucket_metadata_intent(&self, mut bm: BucketMetadata) -> Result<()> {
|
||||
self.revoke_pending_bucket_creation(&bm.name).await;
|
||||
bm.bucket_creation_committed = false;
|
||||
bm.bucket_creation_commit_record_present = true;
|
||||
bm.save_with_store(self.object_store()).await?;
|
||||
save_bucket_incarnation(self.object_store(), &bm.name, bm.bucket_incarnation_id).await?;
|
||||
bm.bucket_incarnation_sidecar = true;
|
||||
@@ -2447,6 +2463,7 @@ impl BucketMetadataSys {
|
||||
}
|
||||
|
||||
async fn commit_bucket_metadata(&self, mut bm: BucketMetadata) -> Result<()> {
|
||||
self.revoke_pending_bucket_creation(&bm.name).await;
|
||||
bm.parse_all_configs()?;
|
||||
bm.bucket_incarnation_sidecar = true;
|
||||
// The caller holds the full bucket creation fence (lifecycle, metadata
|
||||
@@ -2495,6 +2512,7 @@ impl BucketMetadataSys {
|
||||
bucket: &str,
|
||||
namespace_guard: &rustfs_lock::NamespaceLockGuard,
|
||||
) -> Result<()> {
|
||||
self.revoke_pending_bucket_creation(bucket).await;
|
||||
let expected = self.metadata_map.read().await.get(bucket).cloned();
|
||||
if !self
|
||||
.bucket_exists(bucket, namespace_guard, "peer bucket metadata existence check")
|
||||
@@ -2551,6 +2569,7 @@ impl BucketMetadataSys {
|
||||
#[cfg(test)]
|
||||
self.lazy_disk_loads.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
|
||||
|
||||
self.revoke_pending_bucket_creation(bucket).await;
|
||||
let lock = self.object_store().new_ns_lock(bucket, bucket).await?;
|
||||
let guard = lock.get_read_lock(crate::set_disk::get_lock_acquire_timeout()).await?;
|
||||
#[cfg(test)]
|
||||
@@ -2876,6 +2895,7 @@ impl BucketMetadataSys {
|
||||
/// revalidated at write quorum, so a stale snapshot can neither revert an
|
||||
/// acknowledged configuration update nor outlive a delete/recreate.
|
||||
async fn migrate_bucket_creation_commit(&self, bucket: &str) -> Result<BucketMetadataAuthority> {
|
||||
self.revoke_pending_bucket_creation(bucket).await;
|
||||
let transaction_lock = self
|
||||
.object_store()
|
||||
.new_ns_lock(RUSTFS_META_BUCKET, &bucket_metadata_transaction_lock_key(bucket))
|
||||
@@ -2953,6 +2973,7 @@ impl BucketMetadataSys {
|
||||
&self,
|
||||
bucket: &str,
|
||||
) -> Result<BucketMetadataAuthority> {
|
||||
self.revoke_pending_bucket_creation(bucket).await;
|
||||
#[cfg(test)]
|
||||
if self.object_lock_disk_read_errors.write().await.remove(bucket) {
|
||||
return Err(Error::other(format!("injected Object Lock metadata disk read failure: {bucket}")));
|
||||
@@ -3085,77 +3106,30 @@ impl BucketMetadataSys {
|
||||
return Err(Error::other(format!("injected Object Lock metadata disk read failure: {bucket}")));
|
||||
}
|
||||
|
||||
let store = self.object_store();
|
||||
let can_use_local_authority_cache = !store.ctx.lock_manager().is_disabled() && store.all_disks_local_and_online();
|
||||
let gen_before = current_local_meta_mutation_gen();
|
||||
let cached_authority = can_use_local_authority_cache
|
||||
.then(|| {
|
||||
self.local_authority_cache
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.get(bucket)
|
||||
.filter(|(cached_gen, _)| *cached_gen == gen_before)
|
||||
.map(|(_, bm)| Arc::clone(bm))
|
||||
})
|
||||
.flatten();
|
||||
if let Some(cached) = cached_authority {
|
||||
await_bucket_namespace_operation(
|
||||
Some(guard),
|
||||
bucket,
|
||||
"bucket metadata authoritative snapshot",
|
||||
std::future::ready(Ok::<_, Error>(())),
|
||||
)
|
||||
.await?;
|
||||
if current_local_meta_mutation_gen() == gen_before && store.all_disks_local_and_online() {
|
||||
return Ok(BucketMetadataAuthority::Authoritative(cached));
|
||||
}
|
||||
}
|
||||
|
||||
let (bucket_info_res, metadata_res) = tokio::join!(
|
||||
Box::pin(await_bucket_namespace_operation(
|
||||
Some(guard),
|
||||
bucket,
|
||||
"bucket metadata snapshot existence check",
|
||||
async {
|
||||
store
|
||||
.get_bucket_info_from_sets_at_read_quorum(
|
||||
bucket,
|
||||
&crate::storage_api_contracts::bucket::BucketOptions::default(),
|
||||
)
|
||||
.await
|
||||
},
|
||||
)),
|
||||
Box::pin(await_bucket_namespace_operation(
|
||||
Some(guard),
|
||||
bucket,
|
||||
"bucket metadata authoritative snapshot",
|
||||
load_bucket_metadata_parse_with_presence(Arc::clone(&store), bucket, true),
|
||||
)),
|
||||
);
|
||||
match bucket_info_res {
|
||||
match await_bucket_namespace_operation(Some(guard), bucket, "bucket metadata snapshot existence check", async {
|
||||
self.object_store()
|
||||
.get_bucket_info_from_sets_at_read_quorum(bucket, &crate::storage_api_contracts::bucket::BucketOptions::default())
|
||||
.await
|
||||
})
|
||||
.await
|
||||
{
|
||||
Ok(_) => {}
|
||||
Err(Error::VolumeNotFound) => return Ok(BucketMetadataAuthority::MissingBucket),
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
|
||||
let (metadata, persisted) = metadata_res?;
|
||||
let (metadata, persisted) = await_bucket_namespace_operation(
|
||||
Some(guard),
|
||||
bucket,
|
||||
"bucket metadata authoritative snapshot",
|
||||
load_bucket_metadata_parse_with_presence(self.object_store(), bucket, true),
|
||||
)
|
||||
.await?;
|
||||
if persisted && metadata.lock_enabled && !metadata.bucket_creation_committed {
|
||||
return Err(Error::ErasureWriteQuorum);
|
||||
}
|
||||
if persisted {
|
||||
let metadata = Arc::new(metadata);
|
||||
if can_use_local_authority_cache
|
||||
&& metadata.bucket_incarnation_sidecar
|
||||
&& !metadata.bucket_incarnation_id.is_nil()
|
||||
&& current_local_meta_mutation_gen() == gen_before
|
||||
&& store.all_disks_local_and_online()
|
||||
{
|
||||
self.local_authority_cache
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.insert(bucket.to_string(), (gen_before, Arc::clone(&metadata)));
|
||||
}
|
||||
Ok(BucketMetadataAuthority::Authoritative(metadata))
|
||||
Ok(BucketMetadataAuthority::Authoritative(Arc::new(metadata)))
|
||||
} else {
|
||||
Ok(BucketMetadataAuthority::Fabricated)
|
||||
}
|
||||
@@ -3175,7 +3149,7 @@ impl BucketMetadataSys {
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn get_lifecycle_config(&self, bucket: &str) -> Result<(BucketLifecycleConfiguration, OffsetDateTime)> {
|
||||
pub async fn get_lifecycle_config(&self, bucket: &str) -> Result<(PersistedLifecycleConfiguration, OffsetDateTime)> {
|
||||
let (bm, _) = self.get_config(bucket).await?;
|
||||
|
||||
match ConfigState::of(&bm.lifecycle_config_xml, &bm.lifecycle_config).require(bucket, BUCKET_LIFECYCLE_CONFIG)? {
|
||||
@@ -3298,7 +3272,7 @@ impl BucketMetadataSys {
|
||||
}))
|
||||
}
|
||||
|
||||
pub async fn get_replication_config(&self, bucket: &str) -> Result<(ReplicationConfiguration, OffsetDateTime)> {
|
||||
pub async fn get_replication_config(&self, bucket: &str) -> Result<(PersistedReplicationConfiguration, OffsetDateTime)> {
|
||||
let (bm, _) = self.get_config(bucket).await?;
|
||||
|
||||
if !bm.replication_config_xml.is_empty() && bm.replication_config.is_none() {
|
||||
@@ -3400,6 +3374,7 @@ mod tests {
|
||||
};
|
||||
use crate::bucket::target::{BucketTarget, BucketTargetType, Credentials};
|
||||
use crate::config::com::read_config;
|
||||
use crate::disk::{DiskAPI as _, RUSTFS_META_TMP_BUCKET};
|
||||
use crate::storage_api_contracts::bucket::{BucketOperations as _, DeleteBucketOptions, MakeBucketOptions};
|
||||
use byteorder::{ByteOrder as _, LittleEndian};
|
||||
use serial_test::serial;
|
||||
@@ -3454,6 +3429,180 @@ mod tests {
|
||||
),
|
||||
];
|
||||
|
||||
#[tokio::test]
|
||||
#[serial(storage_class_env)]
|
||||
async fn persistence_entry_failures_revoke_pending_creation_before_cache_publish() {
|
||||
let (dirs, store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let sys = bucket_metadata_sys_of(&store.ctx).expect("instance metadata system");
|
||||
let sys = sys.read().await.clone();
|
||||
let options = MakeBucketOptions {
|
||||
lock_enabled: true,
|
||||
..Default::default()
|
||||
};
|
||||
for entry in ["save", "persist", "persist-new", "commit"] {
|
||||
let bucket = format!("revoke-{entry}-{}", Uuid::new_v4());
|
||||
store.fail_next_bucket_creation_after_intent_for_test(&bucket);
|
||||
store
|
||||
.make_bucket(&bucket, &options)
|
||||
.await
|
||||
.expect_err("stop the real producer before physical publication");
|
||||
sys.pending_bucket_creation(&bucket)
|
||||
.await
|
||||
.expect("the producer must leave a real live creation permission");
|
||||
let (intent, persisted) = get_config_from_disk_with_presence_in(&store.ctx, &bucket)
|
||||
.await
|
||||
.expect("read the original producer intent");
|
||||
assert!(persisted, "{entry}: the complete original intent must be persisted");
|
||||
let original_intent = intent.marshal_msg().expect("encode the original intent");
|
||||
|
||||
// Keep the creation fence order used by the real producer.
|
||||
let lifecycle_guard = store
|
||||
.acquire_bucket_lifecycle_write_lock(&bucket)
|
||||
.await
|
||||
.expect("hold the bucket lifecycle fence");
|
||||
let transaction_guard = acquire_bucket_metadata_transaction_lock_in(&store.ctx, &bucket)
|
||||
.await
|
||||
.expect("hold the metadata transaction fence");
|
||||
let namespace_lock = store
|
||||
.new_ns_lock(&bucket, &bucket)
|
||||
.await
|
||||
.expect("create the exact bucket namespace lock");
|
||||
let namespace_guard = namespace_lock
|
||||
.get_write_lock(crate::set_disk::get_lock_acquire_timeout())
|
||||
.await
|
||||
.expect("hold the exact bucket namespace fence");
|
||||
|
||||
// A file at each native staging root causes actual IO to fail before
|
||||
// metadata can reach the cache-publishing set call. Preserve the
|
||||
// original staging directories so no fixture data is discarded.
|
||||
for dir in &dirs {
|
||||
let staging = dir.path().join(RUSTFS_META_TMP_BUCKET);
|
||||
let saved = dir.path().join(".rustfs.sys/revocation-saved-tmp");
|
||||
std::fs::rename(&staging, &saved).expect("preserve the native staging directory");
|
||||
std::fs::write(&staging, b"block metadata staging with a non-directory")
|
||||
.expect("install an actual non-directory IO obstruction");
|
||||
}
|
||||
let result = match entry {
|
||||
"save" => sys.save(intent).await,
|
||||
"persist" => sys.persist_and_set(intent).await,
|
||||
"persist-new" => sys.persist_new_and_set(intent).await,
|
||||
"commit" => sys.commit_bucket_metadata(intent).await,
|
||||
_ => unreachable!(),
|
||||
};
|
||||
for dir in &dirs {
|
||||
let staging = dir.path().join(RUSTFS_META_TMP_BUCKET);
|
||||
let saved = dir.path().join(".rustfs.sys/revocation-saved-tmp");
|
||||
std::fs::remove_file(&staging).expect("remove only the fixture's staging obstruction");
|
||||
std::fs::rename(&saved, &staging).expect("restore the complete original staging directory");
|
||||
}
|
||||
drop((namespace_guard, transaction_guard, lifecycle_guard));
|
||||
result.expect_err("native persistence must fail before its downstream cache set");
|
||||
assert!(
|
||||
sys.pending_bucket_creation(&bucket).await.is_none(),
|
||||
"{entry}: the entry revoker must retire live permission even when persistence fails"
|
||||
);
|
||||
let (after_failure, present) = get_config_from_disk_with_presence_in(&store.ctx, &bucket)
|
||||
.await
|
||||
.expect("read the intent after the failed persistence entry");
|
||||
assert!(present, "{entry}: a failed persistence entry must retain the original metadata");
|
||||
assert_eq!(
|
||||
after_failure
|
||||
.marshal_msg()
|
||||
.expect("encode the original intent after persistence failure"),
|
||||
original_intent,
|
||||
"{entry}: native IO failure must not change the original generation or complete intent"
|
||||
);
|
||||
let error = store
|
||||
.make_bucket(&bucket, &options)
|
||||
.await
|
||||
.expect_err("a failed ordinary persistence entry cannot preserve creation permission");
|
||||
assert!(error.to_string().contains("administrator recovery"), "{entry}: {error}");
|
||||
let (after_retry, _) = get_config_from_disk_with_presence_in(&store.ctx, &bucket)
|
||||
.await
|
||||
.expect("read the original intent after the rejected retry");
|
||||
assert_eq!(
|
||||
after_retry.marshal_msg().expect("encode the intent after the rejected retry"),
|
||||
original_intent,
|
||||
"{entry}: the rejected retry must preserve the complete original intent"
|
||||
);
|
||||
for disk in store.pools[0].disk_set[0].disks.read().await.iter() {
|
||||
assert!(
|
||||
matches!(
|
||||
disk.as_ref().expect("native disk").stat_volume(&bucket).await,
|
||||
Err(crate::disk::error::DiskError::VolumeNotFound)
|
||||
),
|
||||
"{entry}: failed persistence and the rejected retry must not publish a physical volume"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial(storage_class_env)]
|
||||
async fn metadata_reset_revokes_real_pending_creation_without_changing_persisted_intent() {
|
||||
let (_dirs, store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let bucket = format!("revoke-reset-{}", Uuid::new_v4());
|
||||
let options = MakeBucketOptions {
|
||||
lock_enabled: true,
|
||||
..Default::default()
|
||||
};
|
||||
store.fail_next_bucket_creation_after_intent_for_test(&bucket);
|
||||
store
|
||||
.make_bucket(&bucket, &options)
|
||||
.await
|
||||
.expect_err("stop the real producer before physical publication");
|
||||
let sys = bucket_metadata_sys_of(&store.ctx).expect("instance metadata system");
|
||||
let mut sys = sys.read().await.clone();
|
||||
sys.pending_bucket_creation(&bucket)
|
||||
.await
|
||||
.expect("the real producer must leave live creation permission before reset");
|
||||
let (intent, persisted) = get_config_from_disk_with_presence_in(&store.ctx, &bucket)
|
||||
.await
|
||||
.expect("read the original intent before cache reset");
|
||||
assert!(persisted, "cache reset starts from a complete real persisted intent");
|
||||
let original_intent = intent.marshal_msg().expect("encode the original reset intent");
|
||||
sys._reset().await;
|
||||
assert!(
|
||||
sys.pending_bucket_creation(&bucket).await.is_none(),
|
||||
"reset must revoke the producer-issued permission shared with the instance metadata system"
|
||||
);
|
||||
let (after_reset, present) = get_config_from_disk_with_presence_in(&store.ctx, &bucket)
|
||||
.await
|
||||
.expect("read the original intent after cache reset");
|
||||
assert!(present, "cache reset must retain the persisted intent");
|
||||
assert_eq!(
|
||||
after_reset.marshal_msg().expect("encode the intent after cache reset"),
|
||||
original_intent,
|
||||
"cache reset must leave the original generation and complete persisted intent unchanged"
|
||||
);
|
||||
let error = store
|
||||
.make_bucket(&bucket, &options)
|
||||
.await
|
||||
.expect_err("reset cannot regrant the producer's old creation permission");
|
||||
assert!(error.to_string().contains("administrator recovery"), "{error}");
|
||||
let (after_retry, _) = get_config_from_disk_with_presence_in(&store.ctx, &bucket)
|
||||
.await
|
||||
.expect("read the intent after the reset-induced retry rejection");
|
||||
assert_eq!(
|
||||
after_retry
|
||||
.marshal_msg()
|
||||
.expect("encode the intent after the rejected reset retry"),
|
||||
original_intent,
|
||||
"the reset-induced rejection must preserve the original complete intent"
|
||||
);
|
||||
for disk in store.pools[0].disk_set[0].disks.read().await.iter() {
|
||||
assert!(
|
||||
matches!(
|
||||
disk.as_ref().expect("native disk").stat_volume(&bucket).await,
|
||||
Err(crate::disk::error::DiskError::VolumeNotFound)
|
||||
),
|
||||
"cache reset and the rejected retry must not publish a physical volume"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn g_d3_003_new_writer_replication_loads_without_fail_closed_state() {
|
||||
let (dirs, store) = isolated_store_over_temp_disks().await;
|
||||
@@ -3642,7 +3791,7 @@ mod tests {
|
||||
let mut versioned = BucketMetadata::new("authoritative-versioned");
|
||||
versioned.versioning_config_xml = b"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>".to_vec();
|
||||
versioned.versioning_config = Some(VersioningConfiguration {
|
||||
status: Some(s3s::dto::BucketVersioningStatus::from_static(s3s::dto::BucketVersioningStatus::ENABLED)),
|
||||
status: Some("Enabled".to_owned()),
|
||||
..Default::default()
|
||||
});
|
||||
sys.set("authoritative-versioned".to_string(), Arc::new(versioned)).await;
|
||||
@@ -3765,25 +3914,24 @@ mod tests {
|
||||
else {
|
||||
panic!("legacy lock-enabled metadata must remain configured");
|
||||
};
|
||||
assert_eq!(
|
||||
config.object_lock_enabled.as_ref().map(|value| value.as_str()),
|
||||
Some(ObjectLockEnabled::ENABLED)
|
||||
);
|
||||
assert_eq!(config.object_lock_enabled.as_deref(), Some("Enabled"));
|
||||
assert!(config.rule.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn authoritative_object_lock_state_rejects_invalid_retention_periods() {
|
||||
use s3s::dto::{DefaultRetention, ObjectLockRule};
|
||||
use rustfs_gateway_types::persistence::{
|
||||
PersistedDefaultRetention as DefaultRetention, PersistedObjectLockRule as ObjectLockRule,
|
||||
};
|
||||
|
||||
for (days, years) in [(Some(0), None), (Some(-1), None), (Some(1), Some(1)), (None, None)] {
|
||||
let mut metadata = BucketMetadata::new("invalid-object-lock");
|
||||
metadata.object_lock_config = Some(ObjectLockConfiguration {
|
||||
object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
|
||||
object_lock_enabled: Some("Enabled".to_owned()),
|
||||
rule: Some(ObjectLockRule {
|
||||
default_retention: Some(DefaultRetention {
|
||||
days,
|
||||
mode: Some(ObjectLockRetentionMode::from_static(ObjectLockRetentionMode::COMPLIANCE)),
|
||||
mode: Some("COMPLIANCE".to_owned()),
|
||||
years,
|
||||
}),
|
||||
}),
|
||||
@@ -3798,9 +3946,11 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn authoritative_object_lock_state_rejects_invalid_enabled_mode_and_rule() {
|
||||
use s3s::dto::{DefaultRetention, ObjectLockRule};
|
||||
use rustfs_gateway_types::persistence::{
|
||||
PersistedDefaultRetention as DefaultRetention, PersistedObjectLockRule as ObjectLockRule,
|
||||
};
|
||||
|
||||
for enabled in [None, Some(ObjectLockEnabled::from("Disabled".to_string()))] {
|
||||
for enabled in [None, Some("Disabled".to_string())] {
|
||||
let mut metadata = BucketMetadata::new("invalid-object-lock-enabled");
|
||||
metadata.lock_enabled = true;
|
||||
metadata.object_lock_config = Some(ObjectLockConfiguration {
|
||||
@@ -3812,11 +3962,11 @@ mod tests {
|
||||
|
||||
let mut invalid_mode = BucketMetadata::new("invalid-object-lock-mode");
|
||||
invalid_mode.object_lock_config = Some(ObjectLockConfiguration {
|
||||
object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
|
||||
object_lock_enabled: Some("Enabled".to_owned()),
|
||||
rule: Some(ObjectLockRule {
|
||||
default_retention: Some(DefaultRetention {
|
||||
days: Some(1),
|
||||
mode: Some(ObjectLockRetentionMode::from("invalid".to_string())),
|
||||
mode: Some("invalid".to_string()),
|
||||
years: None,
|
||||
}),
|
||||
}),
|
||||
@@ -3825,7 +3975,7 @@ mod tests {
|
||||
|
||||
let mut missing_retention = BucketMetadata::new("missing-object-lock-retention");
|
||||
missing_retention.object_lock_config = Some(ObjectLockConfiguration {
|
||||
object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
|
||||
object_lock_enabled: Some("Enabled".to_owned()),
|
||||
rule: Some(ObjectLockRule { default_retention: None }),
|
||||
});
|
||||
assert!(object_lock_config_state_from_authoritative_metadata(&missing_retention).is_err());
|
||||
@@ -3835,7 +3985,7 @@ mod tests {
|
||||
fn object_lock_ever_enabled_marker_survives_config_removal() {
|
||||
let mut metadata = BucketMetadata::new("lock-config-removed");
|
||||
let config = ObjectLockConfiguration {
|
||||
object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
|
||||
object_lock_enabled: Some("Enabled".to_owned()),
|
||||
rule: None,
|
||||
};
|
||||
metadata
|
||||
@@ -3853,10 +4003,7 @@ mod tests {
|
||||
else {
|
||||
panic!("an ever-enabled bucket must remain Object Lock configured");
|
||||
};
|
||||
assert_eq!(
|
||||
config.object_lock_enabled.as_ref().map(ObjectLockEnabled::as_str),
|
||||
Some(ObjectLockEnabled::ENABLED)
|
||||
);
|
||||
assert_eq!(config.object_lock_enabled.as_deref(), Some("Enabled"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -3872,7 +4019,7 @@ mod tests {
|
||||
|
||||
let node = Arc::new(RwLock::new(BucketMetadataSys::new(ecstore.clone())));
|
||||
let config = ObjectLockConfiguration {
|
||||
object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
|
||||
object_lock_enabled: Some("Enabled".to_owned()),
|
||||
rule: None,
|
||||
};
|
||||
update_with_sys(
|
||||
@@ -4853,7 +5000,7 @@ mod tests {
|
||||
#[tokio::test]
|
||||
async fn update_config_with_persists_tagging_rewrite_across_disk_reload() {
|
||||
use crate::bucket::metadata::BUCKET_TAGGING_CONFIG;
|
||||
use s3s::dto::Tag;
|
||||
use rustfs_gateway_types::cors_tagging::PersistedTag as Tag;
|
||||
|
||||
let (dirs, ecstore) = isolated_store_over_temp_disks().await;
|
||||
|
||||
@@ -5017,7 +5164,7 @@ mod tests {
|
||||
#[tokio::test]
|
||||
async fn concurrent_update_config_with_calls_do_not_lose_writes() {
|
||||
use crate::bucket::metadata::BUCKET_TAGGING_CONFIG;
|
||||
use s3s::dto::Tag;
|
||||
use rustfs_gateway_types::cors_tagging::PersistedTag as Tag;
|
||||
|
||||
let (dirs, ecstore) = isolated_store_over_temp_disks().await;
|
||||
let sys = Arc::new(RwLock::new(BucketMetadataSys::new(ecstore)));
|
||||
@@ -5117,7 +5264,7 @@ mod tests {
|
||||
let mut persisted = BucketMetadata::new("reload-bucket");
|
||||
persisted.policy_config_json = b"persisted-marker".to_vec();
|
||||
let lock_config = ObjectLockConfiguration {
|
||||
object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
|
||||
object_lock_enabled: Some("Enabled".to_owned()),
|
||||
rule: None,
|
||||
};
|
||||
persisted
|
||||
@@ -5210,61 +5357,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn validated_config_update_rejects_a_different_bucket_incarnation() {
|
||||
let (_dirs, store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let bucket = "catalog-cleanup-incarnation";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
let incarnation = store
|
||||
.bucket_incarnation_id_from_disk(bucket)
|
||||
.await
|
||||
.expect("read bucket incarnation");
|
||||
let called = std::sync::atomic::AtomicBool::new(false);
|
||||
let result = store
|
||||
.update_bucket_metadata_config_for_incarnation_validated(
|
||||
bucket,
|
||||
"quota.json",
|
||||
br#"{"quota":1}"#.to_vec(),
|
||||
Uuid::new_v4(),
|
||||
|| {
|
||||
called.store(true, std::sync::atomic::Ordering::SeqCst);
|
||||
Ok(())
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(matches!(result, Err(Error::BucketNotFound(name)) if name == bucket));
|
||||
assert!(
|
||||
!called.load(std::sync::atomic::Ordering::SeqCst),
|
||||
"incarnation must be checked before the external fence"
|
||||
);
|
||||
let (metadata, _) = get_config_from_disk_with_presence_in(&store.ctx, bucket)
|
||||
.await
|
||||
.expect("read unchanged bucket metadata");
|
||||
assert!(
|
||||
metadata.quota_config_json.is_empty(),
|
||||
"a stale cleanup must not modify the replacement bucket"
|
||||
);
|
||||
store
|
||||
.update_bucket_metadata_config_for_incarnation_validated(
|
||||
bucket,
|
||||
"quota.json",
|
||||
br#"{"quota":1}"#.to_vec(),
|
||||
incarnation,
|
||||
|| Ok(()),
|
||||
)
|
||||
.await
|
||||
.expect("the current incarnation permits the validated update");
|
||||
let (metadata, _) = get_config_from_disk_with_presence_in(&store.ctx, bucket)
|
||||
.await
|
||||
.expect("read updated bucket metadata");
|
||||
assert_eq!(metadata.quota_config_json, br#"{"quota":1}"#);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn quota_admission_rejects_stale_cached_no_quota() {
|
||||
@@ -5451,7 +5543,7 @@ mod tests {
|
||||
#[serial]
|
||||
async fn bucket_delete_waits_for_config_mutation_fence() {
|
||||
use crate::bucket::metadata::BUCKET_TAGGING_CONFIG;
|
||||
use s3s::dto::Tag;
|
||||
use rustfs_gateway_types::cors_tagging::PersistedTag as Tag;
|
||||
|
||||
let (_dirs, store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
|
||||
@@ -17,15 +17,8 @@ pub mod objectlock_sys;
|
||||
pub mod types;
|
||||
|
||||
/// Whether a bucket Object Lock configuration has locking enabled. The
|
||||
/// serving-layer `ObjectLockConfiguration` DTO implements this in
|
||||
/// the bucket-metadata module, which owns the persisted configuration type
|
||||
/// during the s3s ratchet migration (rustfs/backlog#1842).
|
||||
/// persisted `ObjectLockConfiguration` implements this in the bucket-metadata
|
||||
/// module, which owns the persisted configuration type.
|
||||
pub trait ObjectLockApi {
|
||||
fn enabled(&self) -> bool;
|
||||
}
|
||||
|
||||
/// Whether a legal-hold status value is one of the two valid wire values.
|
||||
/// Implemented for the serving-layer DTO in the bucket-metadata module.
|
||||
pub trait ObjectLockStatusExt {
|
||||
fn valid(&self) -> bool;
|
||||
}
|
||||
|
||||
@@ -12,6 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
pub use rustfs_replication::config_cache_serde;
|
||||
pub use rustfs_replication::{
|
||||
ObjectOpts, OperatorRuleContract, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_READ_ONLY_HISTORICAL_FIELDS,
|
||||
REMOTE_TARGET_UNSUPPORTED_FIELDS, REMOTE_TARGET_WRITABLE_FIELDS, REPLICATION_CAPABILITY_CONTRACT_VERSION,
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use s3s::dto::ReplicationConfiguration;
|
||||
use rustfs_gateway_types::persistence::PersistedReplicationConfiguration;
|
||||
|
||||
use super::replication_config_boundary::ReplicationConfigurationExt as _;
|
||||
use super::replication_filemeta_boundary::{
|
||||
@@ -42,7 +42,7 @@ impl ReplicationLifecycleBridge {
|
||||
dead_code,
|
||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) fn new_config(config: ReplicationConfiguration) -> ReplicationLifecycleConfig {
|
||||
pub(crate) fn new_config(config: PersistedReplicationConfiguration) -> ReplicationLifecycleConfig {
|
||||
ReplicationConfig::new(Some(config), None)
|
||||
}
|
||||
|
||||
@@ -118,16 +118,17 @@ impl ReplicationLifecycleBridge {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use s3s::dto::{Destination, ReplicationRule, ReplicationRuleStatus};
|
||||
use rustfs_gateway_types::dto::Status;
|
||||
use rustfs_gateway_types::persistence::{PersistedReplicationDestination, PersistedReplicationRule};
|
||||
|
||||
use super::super::replication_filemeta_boundary::{ReplicateTargetDecision, VersionPurgeStatusType};
|
||||
use super::*;
|
||||
|
||||
fn replication_rule() -> ReplicationRule {
|
||||
ReplicationRule {
|
||||
fn replication_rule() -> PersistedReplicationRule {
|
||||
PersistedReplicationRule {
|
||||
delete_marker_replication: None,
|
||||
delete_replication: None,
|
||||
destination: Destination {
|
||||
destination: PersistedReplicationDestination {
|
||||
bucket: "arn:aws:s3:::target-bucket".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
@@ -137,13 +138,13 @@ mod tests {
|
||||
prefix: Some(String::new()),
|
||||
priority: Some(1),
|
||||
source_selection_criteria: None,
|
||||
status: ReplicationRuleStatus::from_static(ReplicationRuleStatus::ENABLED),
|
||||
status: Status::ENABLED.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn has_pending_version_purge_preserves_replication_active_rule_behavior() {
|
||||
let config = ReplicationLifecycleBridge::new_config(ReplicationConfiguration {
|
||||
let config = ReplicationLifecycleBridge::new_config(PersistedReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![replication_rule()],
|
||||
});
|
||||
|
||||
@@ -17,8 +17,8 @@ use std::sync::Arc;
|
||||
use crate::bucket::{metadata::BucketMetadata, metadata_sys};
|
||||
use crate::disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET};
|
||||
use crate::runtime::instance::InstanceContext;
|
||||
use rustfs_gateway_types::persistence::PersistedReplicationConfiguration;
|
||||
use rustfs_utils::path::path_join_buf;
|
||||
use s3s::dto::ReplicationConfiguration;
|
||||
use time::OffsetDateTime;
|
||||
|
||||
use super::replication_error_boundary::{Error, Result};
|
||||
@@ -33,15 +33,14 @@ pub(crate) struct ReplicationMetadataStore;
|
||||
impl ReplicationMetadataStore {
|
||||
pub(crate) const MRF_REPLICATION_FILE: &'static str = "config/replication/mrf.bin";
|
||||
pub(crate) const MRF_REPLICATION_RECOVERY_LOCK: &'static str = "config/replication/mrf.bin.recovery";
|
||||
pub(crate) const MRF_REPLICATION_REPLAY_STAMP: &'static str = "config/replication/mrf.bin.replay";
|
||||
pub(crate) const FORCE_DELETE_REPLICATION_FILE: &'static str = "config/replication/force-delete.bin";
|
||||
pub(crate) const FORCE_DELETE_REPLICATION_TRANSACTION_LOCK: &'static str = "config/replication/force-delete.bin.transaction";
|
||||
|
||||
pub(crate) async fn replication_config(bucket: &str) -> Result<(ReplicationConfiguration, OffsetDateTime)> {
|
||||
pub(crate) async fn replication_config(bucket: &str) -> Result<(PersistedReplicationConfiguration, OffsetDateTime)> {
|
||||
metadata_sys::get_replication_config(bucket).await
|
||||
}
|
||||
|
||||
pub(crate) async fn optional_replication_config(bucket: &str) -> Result<Option<ReplicationConfiguration>> {
|
||||
pub(crate) async fn optional_replication_config(bucket: &str) -> Result<Option<PersistedReplicationConfiguration>> {
|
||||
let config = match Self::replication_config(bucket).await {
|
||||
Ok((config, _)) => Some(config),
|
||||
Err(err) => {
|
||||
@@ -117,10 +116,6 @@ mod tests {
|
||||
ReplicationMetadataStore::MRF_REPLICATION_RECOVERY_LOCK,
|
||||
"config/replication/mrf.bin.recovery"
|
||||
);
|
||||
assert_eq!(
|
||||
ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP,
|
||||
"config/replication/mrf.bin.replay"
|
||||
);
|
||||
assert_eq!(
|
||||
ReplicationMetadataStore::FORCE_DELETE_REPLICATION_FILE,
|
||||
"config/replication/force-delete.bin"
|
||||
|
||||
@@ -16,7 +16,9 @@ use std::{collections::HashMap, fmt, sync::Arc};
|
||||
|
||||
use super::replication_filemeta_boundary::metadata_keys;
|
||||
use crate::bucket::metadata::BucketMetadata;
|
||||
use s3s::dto::{BucketVersioningStatus, ReplicationConfiguration, ReplicationRuleStatus, VersioningConfiguration};
|
||||
use rustfs_gateway_types::dto::Status;
|
||||
use rustfs_gateway_types::persistence::PersistedReplicationConfiguration;
|
||||
use rustfs_gateway_types::persistence::PersistedVersioningConfiguration as VersioningConfiguration;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use time::OffsetDateTime;
|
||||
use tracing::error;
|
||||
@@ -39,7 +41,7 @@ use super::replication_target_boundary::{BucketTargets, ReplicationTargetStore};
|
||||
use super::replication_versioning_boundary::ReplicationVersioningStore;
|
||||
use super::runtime_boundary as runtime_sources;
|
||||
|
||||
pub(crate) async fn get_replication_config(bucket: &str) -> Result<Option<ReplicationConfiguration>> {
|
||||
pub(crate) async fn get_replication_config(bucket: &str) -> Result<Option<PersistedReplicationConfiguration>> {
|
||||
let config = ReplicationMetadataStore::optional_replication_config(bucket).await?;
|
||||
validate_delete_replication_config(&VersioningConfiguration::default(), config.as_ref())?;
|
||||
Ok(config)
|
||||
@@ -64,7 +66,7 @@ impl DeleteReplicationConfigSnapshot {
|
||||
#[cfg(test)]
|
||||
pub(crate) fn from_configs_for_test(
|
||||
versioning: VersioningConfiguration,
|
||||
replication: Option<ReplicationConfiguration>,
|
||||
replication: Option<PersistedReplicationConfiguration>,
|
||||
) -> Self {
|
||||
let metadata = replication.map(|config| {
|
||||
let mut metadata = BucketMetadata::new("test-bucket");
|
||||
@@ -78,7 +80,7 @@ impl DeleteReplicationConfigSnapshot {
|
||||
&self.versioning
|
||||
}
|
||||
|
||||
pub fn replication_config(&self) -> Option<&ReplicationConfiguration> {
|
||||
pub fn replication_config(&self) -> Option<&PersistedReplicationConfiguration> {
|
||||
self.metadata
|
||||
.as_ref()
|
||||
.and_then(|metadata| metadata.replication_config.as_ref())
|
||||
@@ -101,7 +103,7 @@ impl DeleteReplicationConfigSnapshot {
|
||||
pub(crate) fn active_delete_marker_rules_require_tags(&self, object: &str) -> bool {
|
||||
self.replication_config().is_some_and(|config| {
|
||||
config.rules.iter().any(|rule| {
|
||||
if rule.status == ReplicationRuleStatus::from_static(ReplicationRuleStatus::DISABLED) {
|
||||
if rule.status == Status::DISABLED.as_str() {
|
||||
return false;
|
||||
}
|
||||
if !object.starts_with(rule.prefix()) {
|
||||
@@ -122,12 +124,12 @@ impl DeleteReplicationConfigSnapshot {
|
||||
|
||||
fn validate_delete_replication_config(
|
||||
versioning: &VersioningConfiguration,
|
||||
config: Option<&ReplicationConfiguration>,
|
||||
config: Option<&PersistedReplicationConfiguration>,
|
||||
) -> Result<()> {
|
||||
if versioning
|
||||
.status
|
||||
.as_ref()
|
||||
.is_some_and(|status| !matches!(status.as_str(), BucketVersioningStatus::ENABLED | BucketVersioningStatus::SUSPENDED))
|
||||
.is_some_and(|status| !matches!(status.as_str(), "Enabled" | "Suspended"))
|
||||
{
|
||||
return Err(super::replication_error_boundary::Error::other(
|
||||
"bucket versioning configuration has an invalid status",
|
||||
@@ -144,7 +146,7 @@ fn validate_delete_replication_config(
|
||||
let role = config.role.trim();
|
||||
let mut role_destination = None;
|
||||
for rule in &config.rules {
|
||||
if rule.status.as_str() == ReplicationRuleStatus::ENABLED {
|
||||
if rule.status.as_str() == Status::ENABLED.as_str() {
|
||||
let destination = rule.destination.bucket.trim();
|
||||
if role.is_empty() && destination.is_empty() {
|
||||
return Err(super::replication_error_boundary::Error::other(
|
||||
@@ -169,7 +171,7 @@ fn validate_delete_replication_config(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn replication_config_from_metadata(metadata: &BucketMetadata) -> Result<Option<&ReplicationConfiguration>> {
|
||||
fn replication_config_from_metadata(metadata: &BucketMetadata) -> Result<Option<&PersistedReplicationConfiguration>> {
|
||||
if !metadata.replication_config_xml.is_empty() && metadata.replication_config.is_none() {
|
||||
return Err(super::replication_error_boundary::Error::other(
|
||||
"persisted bucket replication configuration is invalid",
|
||||
@@ -250,12 +252,15 @@ pub(crate) async fn load_delete_replication_config_in(
|
||||
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
|
||||
pub struct ReplicationConfig {
|
||||
pub config: Option<ReplicationConfiguration>,
|
||||
/// Carried in the scanner's persisted data usage cache, so its serde form
|
||||
/// is the one the s3s DTO wrote there (see `config_cache_serde` in the replication crate).
|
||||
#[serde(with = "super::replication_config_boundary::config_cache_serde")]
|
||||
pub config: Option<PersistedReplicationConfiguration>,
|
||||
pub remotes: Option<BucketTargets>,
|
||||
}
|
||||
|
||||
impl ReplicationConfig {
|
||||
pub fn new(config: Option<ReplicationConfiguration>, remotes: Option<BucketTargets>) -> Self {
|
||||
pub fn new(config: Option<PersistedReplicationConfiguration>, remotes: Option<BucketTargets>) -> Self {
|
||||
Self { config, remotes }
|
||||
}
|
||||
|
||||
@@ -468,7 +473,7 @@ fn check_replicate_delete_with_config(
|
||||
oi: &ObjectInfo,
|
||||
del_opts: &ObjectOptions,
|
||||
source_error: bool,
|
||||
config: Option<&ReplicationConfiguration>,
|
||||
config: Option<&PersistedReplicationConfiguration>,
|
||||
existing_delete_marker: bool,
|
||||
trust_persisted_replica_status: bool,
|
||||
) -> ReplicateDecision {
|
||||
@@ -587,7 +592,7 @@ pub(crate) async fn must_replicate(bucket: &str, object: &str, mopts: MustReplic
|
||||
}
|
||||
|
||||
fn metadata_target_should_replicate(
|
||||
cfg: &ReplicationConfiguration,
|
||||
cfg: &PersistedReplicationConfiguration,
|
||||
opts: &ObjectOpts,
|
||||
mopts: &MustReplicateOptions,
|
||||
arn: &str,
|
||||
@@ -597,20 +602,21 @@ fn metadata_target_should_replicate(
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use s3s::dto::{
|
||||
DeleteMarkerReplication, DeleteMarkerReplicationStatus, DeleteReplication, DeleteReplicationStatus, Destination,
|
||||
ReplicaModifications, ReplicationRule, ReplicationRuleFilter, ReplicationRuleStatus, SourceSelectionCriteria, Tag,
|
||||
use rustfs_gateway_types::dto::Status;
|
||||
use rustfs_gateway_types::persistence::{
|
||||
PersistedOptionalReplicationStatus, PersistedReplicationDestination, PersistedReplicationFilter,
|
||||
PersistedReplicationRule, PersistedReplicationStatus, PersistedReplicationTag, PersistedSourceSelectionCriteria,
|
||||
};
|
||||
|
||||
use super::super::replication_filemeta_boundary::VersionPurgeStatusType;
|
||||
use super::super::replication_target_boundary::BucketTarget;
|
||||
use super::*;
|
||||
|
||||
fn replication_rule() -> ReplicationRule {
|
||||
ReplicationRule {
|
||||
fn replication_rule() -> PersistedReplicationRule {
|
||||
PersistedReplicationRule {
|
||||
delete_marker_replication: None,
|
||||
delete_replication: None,
|
||||
destination: Destination {
|
||||
destination: PersistedReplicationDestination {
|
||||
bucket: "arn:aws:s3:::target-bucket".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
@@ -620,7 +626,7 @@ mod tests {
|
||||
prefix: Some(String::new()),
|
||||
priority: Some(1),
|
||||
source_selection_criteria: None,
|
||||
status: ReplicationRuleStatus::from_static(ReplicationRuleStatus::ENABLED),
|
||||
status: Status::ENABLED.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -629,11 +635,11 @@ mod tests {
|
||||
let arn = "arn:rustfs:replication:us-east-1:target:bucket";
|
||||
let mut rule = replication_rule();
|
||||
rule.destination.bucket = arn.to_string();
|
||||
rule.filter = Some(ReplicationRuleFilter {
|
||||
rule.filter = Some(PersistedReplicationFilter {
|
||||
prefix: Some("admitted/".to_string()),
|
||||
..Default::default()
|
||||
});
|
||||
let cfg = ReplicationConfiguration {
|
||||
let cfg = PersistedReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![rule],
|
||||
};
|
||||
@@ -671,7 +677,7 @@ mod tests {
|
||||
assert!(!empty.replicate(&ObjectOpts::default()));
|
||||
|
||||
let config = ReplicationConfig::new(
|
||||
Some(ReplicationConfiguration {
|
||||
Some(PersistedReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![replication_rule()],
|
||||
}),
|
||||
@@ -725,7 +731,7 @@ mod tests {
|
||||
fn delete_snapshot_rejects_enabled_rules_without_a_destination() {
|
||||
let mut rule = replication_rule();
|
||||
rule.destination.bucket.clear();
|
||||
let config = ReplicationConfiguration {
|
||||
let config = PersistedReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![rule],
|
||||
};
|
||||
@@ -741,7 +747,7 @@ mod tests {
|
||||
let first = replication_rule();
|
||||
let mut second = replication_rule();
|
||||
second.destination.bucket = "arn:aws:s3:::other-target".to_string();
|
||||
let config = ReplicationConfiguration {
|
||||
let config = PersistedReplicationConfiguration {
|
||||
role: "arn:aws:s3:::role-target".to_string(),
|
||||
rules: vec![first, second],
|
||||
};
|
||||
@@ -752,47 +758,47 @@ mod tests {
|
||||
#[test]
|
||||
fn delete_snapshot_rejects_unknown_status_values() {
|
||||
let invalid_versioning = VersioningConfiguration {
|
||||
status: Some("Enabld".to_string().into()),
|
||||
status: Some("Enabld".to_string()),
|
||||
..Default::default()
|
||||
};
|
||||
assert!(validate_delete_replication_config(&invalid_versioning, None).is_err());
|
||||
|
||||
let mut invalid_rule = replication_rule();
|
||||
invalid_rule.status = "Enabld".to_string().into();
|
||||
let config = ReplicationConfiguration {
|
||||
invalid_rule.status = "Enabld".to_string();
|
||||
let config = PersistedReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![invalid_rule],
|
||||
};
|
||||
assert!(validate_delete_replication_config(&VersioningConfiguration::default(), Some(&config)).is_err());
|
||||
|
||||
let mut invalid_delete = replication_rule();
|
||||
invalid_delete.delete_replication = Some(DeleteReplication {
|
||||
status: "Enabld".to_string().into(),
|
||||
invalid_delete.delete_replication = Some(PersistedReplicationStatus {
|
||||
status: "Enabld".to_string(),
|
||||
});
|
||||
let config = ReplicationConfiguration {
|
||||
let config = PersistedReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![invalid_delete],
|
||||
};
|
||||
assert!(validate_delete_replication_config(&VersioningConfiguration::default(), Some(&config)).is_err());
|
||||
|
||||
let mut invalid_delete_marker = replication_rule();
|
||||
invalid_delete_marker.delete_marker_replication = Some(DeleteMarkerReplication {
|
||||
status: Some("Enabld".to_string().into()),
|
||||
invalid_delete_marker.delete_marker_replication = Some(PersistedOptionalReplicationStatus {
|
||||
status: Some("Enabld".to_string()),
|
||||
});
|
||||
let config = ReplicationConfiguration {
|
||||
let config = PersistedReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![invalid_delete_marker],
|
||||
};
|
||||
assert!(validate_delete_replication_config(&VersioningConfiguration::default(), Some(&config)).is_err());
|
||||
|
||||
let mut invalid_replica_modifications = replication_rule();
|
||||
invalid_replica_modifications.source_selection_criteria = Some(SourceSelectionCriteria {
|
||||
replica_modifications: Some(ReplicaModifications {
|
||||
status: "Enabld".to_string().into(),
|
||||
invalid_replica_modifications.source_selection_criteria = Some(PersistedSourceSelectionCriteria {
|
||||
replica_modifications: Some(PersistedReplicationStatus {
|
||||
status: "Enabld".to_string(),
|
||||
}),
|
||||
sse_kms_encrypted_objects: None,
|
||||
});
|
||||
let config = ReplicationConfiguration {
|
||||
let config = PersistedReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![invalid_replica_modifications],
|
||||
};
|
||||
@@ -804,11 +810,11 @@ mod tests {
|
||||
let mut metadata = BucketMetadata::new("bucket");
|
||||
metadata.versioning_config_xml = b"configured".to_vec();
|
||||
metadata.versioning_config = Some(VersioningConfiguration {
|
||||
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
|
||||
status: Some("Enabled".to_owned()),
|
||||
..Default::default()
|
||||
});
|
||||
metadata.replication_config_xml = b"configured".to_vec();
|
||||
metadata.replication_config = Some(ReplicationConfiguration {
|
||||
metadata.replication_config = Some(PersistedReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![replication_rule()],
|
||||
});
|
||||
@@ -848,7 +854,7 @@ mod tests {
|
||||
);
|
||||
|
||||
let mut inconsistent = BucketMetadata::new("bucket");
|
||||
inconsistent.replication_config = Some(ReplicationConfiguration {
|
||||
inconsistent.replication_config = Some(PersistedReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![replication_rule()],
|
||||
});
|
||||
@@ -863,11 +869,11 @@ mod tests {
|
||||
let arn = "arn:rustfs:replication:us-east-1:target:bucket";
|
||||
let mut rule = replication_rule();
|
||||
rule.destination.bucket = arn.to_string();
|
||||
rule.delete_marker_replication = Some(DeleteMarkerReplication {
|
||||
status: Some(DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::ENABLED)),
|
||||
rule.delete_marker_replication = Some(PersistedOptionalReplicationStatus {
|
||||
status: Some(Status::ENABLED.to_string()),
|
||||
});
|
||||
let mut metadata = BucketMetadata::new("bucket");
|
||||
metadata.replication_config = Some(ReplicationConfiguration {
|
||||
metadata.replication_config = Some(PersistedReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![rule],
|
||||
});
|
||||
@@ -898,27 +904,27 @@ mod tests {
|
||||
fn delete_marker_source_read_is_required_only_for_tag_filtered_rules() {
|
||||
let mut prefix_rule = replication_rule();
|
||||
prefix_rule.prefix = Some("logs/".to_string());
|
||||
prefix_rule.delete_marker_replication = Some(DeleteMarkerReplication {
|
||||
status: Some(DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::ENABLED)),
|
||||
prefix_rule.delete_marker_replication = Some(PersistedOptionalReplicationStatus {
|
||||
status: Some(Status::ENABLED.to_string()),
|
||||
});
|
||||
|
||||
let prefix_snapshot = DeleteReplicationConfigSnapshot::from_configs_for_test(
|
||||
VersioningConfiguration {
|
||||
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
|
||||
status: Some("Enabled".to_owned()),
|
||||
..Default::default()
|
||||
},
|
||||
Some(ReplicationConfiguration {
|
||||
Some(PersistedReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![prefix_rule],
|
||||
}),
|
||||
);
|
||||
|
||||
let mut tag_rule = replication_rule();
|
||||
tag_rule.delete_marker_replication = Some(DeleteMarkerReplication {
|
||||
status: Some(DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::ENABLED)),
|
||||
tag_rule.delete_marker_replication = Some(PersistedOptionalReplicationStatus {
|
||||
status: Some(Status::ENABLED.to_string()),
|
||||
});
|
||||
tag_rule.filter = Some(ReplicationRuleFilter {
|
||||
tag: Some(Tag {
|
||||
tag_rule.filter = Some(PersistedReplicationFilter {
|
||||
tag: Some(PersistedReplicationTag {
|
||||
key: Some("class".to_string()),
|
||||
value: Some("audit".to_string()),
|
||||
}),
|
||||
@@ -926,10 +932,10 @@ mod tests {
|
||||
});
|
||||
let tag_snapshot = DeleteReplicationConfigSnapshot::from_configs_for_test(
|
||||
VersioningConfiguration {
|
||||
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
|
||||
status: Some("Enabled".to_owned()),
|
||||
..Default::default()
|
||||
},
|
||||
Some(ReplicationConfiguration {
|
||||
Some(PersistedReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![tag_rule],
|
||||
}),
|
||||
@@ -944,14 +950,14 @@ mod tests {
|
||||
let arn = "arn:rustfs:replication:us-east-1:target:bucket";
|
||||
let mut rule = replication_rule();
|
||||
rule.destination.bucket = arn.to_string();
|
||||
rule.delete_replication = Some(DeleteReplication {
|
||||
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED),
|
||||
rule.delete_replication = Some(PersistedReplicationStatus {
|
||||
status: Status::ENABLED.to_string(),
|
||||
});
|
||||
rule.delete_marker_replication = Some(DeleteMarkerReplication {
|
||||
status: Some(DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::DISABLED)),
|
||||
rule.delete_marker_replication = Some(PersistedOptionalReplicationStatus {
|
||||
status: Some(Status::DISABLED.to_string()),
|
||||
});
|
||||
let config = ReplicationConfig::new(
|
||||
Some(ReplicationConfiguration {
|
||||
Some(PersistedReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![rule],
|
||||
}),
|
||||
@@ -985,22 +991,22 @@ mod tests {
|
||||
#[test]
|
||||
fn live_delete_does_not_trust_persisted_replica_status() {
|
||||
let mut rule = replication_rule();
|
||||
rule.delete_replication = Some(DeleteReplication {
|
||||
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED),
|
||||
rule.delete_replication = Some(PersistedReplicationStatus {
|
||||
status: Status::ENABLED.to_string(),
|
||||
});
|
||||
rule.source_selection_criteria = Some(SourceSelectionCriteria {
|
||||
replica_modifications: Some(ReplicaModifications {
|
||||
status: s3s::dto::ReplicaModificationsStatus::from_static(s3s::dto::ReplicaModificationsStatus::DISABLED),
|
||||
rule.source_selection_criteria = Some(PersistedSourceSelectionCriteria {
|
||||
replica_modifications: Some(PersistedReplicationStatus {
|
||||
status: Status::DISABLED.to_string(),
|
||||
}),
|
||||
sse_kms_encrypted_objects: None,
|
||||
});
|
||||
let replication = ReplicationConfiguration {
|
||||
let replication = PersistedReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![rule],
|
||||
};
|
||||
let snapshot = DeleteReplicationConfigSnapshot::from_configs_for_test(
|
||||
VersioningConfiguration {
|
||||
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
|
||||
status: Some("Enabled".to_owned()),
|
||||
..Default::default()
|
||||
},
|
||||
Some(replication.clone()),
|
||||
@@ -1036,14 +1042,14 @@ mod tests {
|
||||
let arn = "arn:rustfs:replication:us-east-1:target:bucket";
|
||||
let mut rule = replication_rule();
|
||||
rule.destination.bucket = arn.to_string();
|
||||
rule.delete_replication = Some(DeleteReplication {
|
||||
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED),
|
||||
rule.delete_replication = Some(PersistedReplicationStatus {
|
||||
status: Status::ENABLED.to_string(),
|
||||
});
|
||||
rule.delete_marker_replication = Some(DeleteMarkerReplication {
|
||||
status: Some(DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::DISABLED)),
|
||||
rule.delete_marker_replication = Some(PersistedOptionalReplicationStatus {
|
||||
status: Some(Status::DISABLED.to_string()),
|
||||
});
|
||||
let config = ReplicationConfig::new(
|
||||
Some(ReplicationConfiguration {
|
||||
Some(PersistedReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![rule],
|
||||
}),
|
||||
|
||||
@@ -86,10 +86,6 @@ const MRF_CORRUPT_FILE_PREFIX: &str = "config/replication/mrf.corrupt";
|
||||
const MRF_PENDING_CAP: usize = 200_000;
|
||||
const MRF_RETRY_INITIAL_DELAY: Duration = Duration::from_millis(100);
|
||||
const MRF_RETRY_MAX_DELAY: Duration = Duration::from_secs(5);
|
||||
/// Interval between runtime replays of the MRF journal, cluster-wide. A pass
|
||||
/// re-attempts every retained entry, so a target that stays down costs one
|
||||
/// attempt per entry per interval.
|
||||
const MRF_REPLAY_INTERVAL: Duration = Duration::from_secs(300);
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub struct DurableMrfBacklog {
|
||||
@@ -619,7 +615,7 @@ async fn read_mrf_entries<S: ReplicationObjectIO>(storage: Arc<S>) -> Result<Vec
|
||||
}
|
||||
|
||||
/// Acknowledge only the generation read by the recovery leader. Entries appended
|
||||
/// after that generation are retained as a suffix and replayed on the next pass.
|
||||
/// after that generation are retained as a suffix and replayed on the next startup.
|
||||
/// Lock order: recovery leader lock -> MRF journal object lock.
|
||||
async fn acknowledge_mrf_recovery<S: ReplicationStorage>(
|
||||
storage: Arc<S>,
|
||||
@@ -671,7 +667,7 @@ async fn acknowledge_mrf_recovery<S: ReplicationStorage>(
|
||||
Err(EcstoreError::PreconditionFailed)
|
||||
}
|
||||
|
||||
/// Acquires the MRF recovery leader lock for one journal replay pass.
|
||||
/// Acquires the MRF recovery leader lock for the startup replay.
|
||||
/// Returns `None` (after logging) when the lock cannot be created or another
|
||||
/// node is already processing the backlog.
|
||||
async fn acquire_mrf_recovery_guard<S: ReplicationStorage>(storage: &Arc<S>) -> Option<rustfs_lock::NamespaceLockGuard> {
|
||||
@@ -1022,7 +1018,7 @@ fn admitted_mrf_replicate_object(oi: ObjectInfo, entry: &MrfReplicateEntry, op_t
|
||||
}
|
||||
|
||||
/// Acknowledges the replayed MRF prefix and returns the retained backlog.
|
||||
/// On acknowledgement failure the backlog is preserved for the next replay pass and
|
||||
/// On acknowledgement failure the backlog is preserved for the next startup and
|
||||
/// re-read (falling back to the replayed snapshot) so the published summary stays accurate.
|
||||
async fn resolve_retained_mrf_entries<S: ReplicationStorage>(
|
||||
storage: &Arc<S>,
|
||||
@@ -1037,7 +1033,7 @@ async fn resolve_retained_mrf_entries<S: ReplicationStorage>(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||
error = %error,
|
||||
"Failed to acknowledge the MRF recovery prefix; preserving it for the next replay pass"
|
||||
"Failed to acknowledge the MRF recovery prefix; preserving it for the next startup"
|
||||
);
|
||||
match read_mrf_entries(storage.clone()).await {
|
||||
Ok(current) => current,
|
||||
@@ -1055,120 +1051,6 @@ async fn resolve_retained_mrf_entries<S: ReplicationStorage>(
|
||||
}
|
||||
}
|
||||
|
||||
fn unix_millis_now() -> i64 {
|
||||
i64::try_from(OffsetDateTime::now_utc().unix_timestamp_nanos() / 1_000_000).unwrap_or(i64::MAX)
|
||||
}
|
||||
|
||||
/// Whether a periodic pass may run, given the start time of the last pass on
|
||||
/// any node. A stamp more than one interval ahead of this node's clock is skew
|
||||
/// or a bad value, so it does not suppress replay.
|
||||
fn mrf_replay_due(last_started_ms: Option<i64>, now_ms: i64, interval: Duration) -> bool {
|
||||
let Some(last_started_ms) = last_started_ms else {
|
||||
return true;
|
||||
};
|
||||
let interval_ms = i64::try_from(interval.as_millis()).unwrap_or(i64::MAX);
|
||||
now_ms.saturating_sub(last_started_ms) >= interval_ms || last_started_ms.saturating_sub(now_ms) >= interval_ms
|
||||
}
|
||||
|
||||
async fn read_mrf_replay_stamp<S: ReplicationStorage>(storage: &Arc<S>) -> Option<i64> {
|
||||
let data = ReplicationConfigStore::read(storage.clone(), ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP)
|
||||
.await
|
||||
.ok()?;
|
||||
std::str::from_utf8(&data).ok()?.trim().parse().ok()
|
||||
}
|
||||
|
||||
async fn write_mrf_replay_stamp<S: ReplicationStorage>(storage: &Arc<S>, started_ms: i64) {
|
||||
if let Err(error) = ReplicationConfigStore::save(
|
||||
storage.clone(),
|
||||
ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP,
|
||||
started_ms.to_string().into_bytes(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
warn!(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||
error = %error,
|
||||
"Failed to record the MRF replay start time"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// One MRF journal replay pass. The recovery leader reads the journal,
|
||||
/// re-drives every entry, and acknowledges the replayed prefix; entries that
|
||||
/// could not be admitted, and entries appended meanwhile, stay for the next pass.
|
||||
///
|
||||
/// Every node runs the periodic loop, and the leader lock only serializes
|
||||
/// passes. With `min_interval`, a pass is skipped when a pass on any node
|
||||
/// started within that interval, read from the shared replay stamp, so a
|
||||
/// cluster replays once per interval rather than once per node. The startup
|
||||
/// pass passes `None` and always runs. A pass that finds entries records its
|
||||
/// start time. Returns whether the journal was read.
|
||||
/// Lock order: recovery leader lock -> replay stamp object lock.
|
||||
async fn replay_mrf_journal<S: ReplicationStorage>(storage: &Arc<S>, min_interval: Option<Duration>) -> bool {
|
||||
let Some(recovery_guard) = acquire_mrf_recovery_guard(storage).await else {
|
||||
return false;
|
||||
};
|
||||
let started_ms = unix_millis_now();
|
||||
if let Some(interval) = min_interval
|
||||
&& !mrf_replay_due(read_mrf_replay_stamp(storage).await, started_ms, interval)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
let Some(entries) = load_mrf_recovery_entries(storage).await else {
|
||||
return true;
|
||||
};
|
||||
if !entries.is_empty() {
|
||||
write_mrf_replay_stamp(storage, started_ms).await;
|
||||
}
|
||||
set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&entries));
|
||||
|
||||
let total = entries.len();
|
||||
let mut queued_count = 0usize;
|
||||
let mut retry_entries = Vec::new();
|
||||
|
||||
for entry in entries.iter() {
|
||||
let Some(admission) = replay_mrf_entry(entry, storage, &mut retry_entries).await else {
|
||||
continue;
|
||||
};
|
||||
|
||||
if admission == ReplicationQueueAdmission::Missed {
|
||||
retry_entries.push(entry.clone());
|
||||
} else if admission == ReplicationQueueAdmission::Queued {
|
||||
queued_count += 1;
|
||||
}
|
||||
}
|
||||
|
||||
let retained = resolve_retained_mrf_entries(storage, &recovery_guard, &entries, &retry_entries).await;
|
||||
let retained_count = retained.len();
|
||||
set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&retained));
|
||||
|
||||
if queued_count > 0 {
|
||||
info!(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||
recovered = queued_count,
|
||||
total,
|
||||
retained = retained_count,
|
||||
"Replayed MRF entries admitted for retry"
|
||||
);
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
/// Re-runs [`replay_mrf_journal`] every `period`. The recovery leader lock keeps
|
||||
/// one pass at a time across the cluster, including the startup pass, and the
|
||||
/// shared replay stamp spaces passes from different nodes by `period`.
|
||||
fn spawn_mrf_replay_loop<S: ReplicationStorage>(storage: Arc<S>, period: Duration) -> JoinHandle<()> {
|
||||
tokio::spawn(async move {
|
||||
loop {
|
||||
tokio::time::sleep(period).await;
|
||||
replay_mrf_journal(&storage, Some(period)).await;
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
#[error("replication resync {active_resync_id} is already active for {bucket}/{arn}")]
|
||||
struct ResyncActiveConflictError {
|
||||
@@ -1289,7 +1171,6 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
// Start background tasks
|
||||
pool.start_mrf_persister().await;
|
||||
pool.start_mrf_processor().await;
|
||||
pool.start_mrf_replayer().await;
|
||||
pool.start_force_delete_processor().await;
|
||||
|
||||
pool
|
||||
@@ -1758,23 +1639,52 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
|
||||
/// Starts the MRF processor — one-shot at startup.
|
||||
///
|
||||
/// Replays the on-disk MRF journal once; [`Self::start_mrf_replayer`] repeats
|
||||
/// the pass while the server runs.
|
||||
/// Reads the on-disk MRF file, re-injects admitted entries as Heal operations, and
|
||||
/// rewrites any entries that could not be admitted for a later startup retry.
|
||||
async fn start_mrf_processor(&self) {
|
||||
let storage = self.storage.clone();
|
||||
let handle = tokio::spawn(async move {
|
||||
replay_mrf_journal(&storage, None).await;
|
||||
});
|
||||
self.task_handles.lock().await.push(handle);
|
||||
}
|
||||
|
||||
/// Starts the runtime MRF replayer — ongoing background task.
|
||||
///
|
||||
/// Entries reach the journal while the server runs (worker-queue overflow,
|
||||
/// delete-marker purge intents whose watch window drained with targets still
|
||||
/// dirty). Replaying only at startup left them waiting for a restart.
|
||||
async fn start_mrf_replayer(&self) {
|
||||
let handle = spawn_mrf_replay_loop(self.storage.clone(), MRF_REPLAY_INTERVAL);
|
||||
let handle = tokio::spawn(async move {
|
||||
let Some(recovery_guard) = acquire_mrf_recovery_guard(&storage).await else {
|
||||
return;
|
||||
};
|
||||
|
||||
let Some(entries) = load_mrf_recovery_entries(&storage).await else {
|
||||
return;
|
||||
};
|
||||
set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&entries));
|
||||
|
||||
let total = entries.len();
|
||||
let mut queued_count = 0usize;
|
||||
let mut retry_entries = Vec::new();
|
||||
|
||||
for entry in entries.iter() {
|
||||
let Some(admission) = replay_mrf_entry(entry, &storage, &mut retry_entries).await else {
|
||||
continue;
|
||||
};
|
||||
|
||||
if admission == ReplicationQueueAdmission::Missed {
|
||||
retry_entries.push(entry.clone());
|
||||
} else if admission == ReplicationQueueAdmission::Queued {
|
||||
queued_count += 1;
|
||||
}
|
||||
}
|
||||
|
||||
let retained = resolve_retained_mrf_entries(&storage, &recovery_guard, &entries, &retry_entries).await;
|
||||
let retained_count = retained.len();
|
||||
set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&retained));
|
||||
|
||||
if queued_count > 0 {
|
||||
info!(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||
recovered = queued_count,
|
||||
total,
|
||||
retained = retained_count,
|
||||
"Replayed MRF entries admitted for retry"
|
||||
);
|
||||
}
|
||||
});
|
||||
self.task_handles.lock().await.push(handle);
|
||||
}
|
||||
|
||||
@@ -1845,7 +1755,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
/// writes them to the on-disk MRF file every flush interval (default 10s,
|
||||
/// overridable via `RUSTFS_REPL_MRF_FLUSH_INTERVAL_MS`) or when 1 000 new
|
||||
/// entries accumulate. Runtime writers append under the journal lock; only the
|
||||
/// recovery leader of a replay pass may remove an acknowledged prefix.
|
||||
/// startup recovery leader may remove an acknowledged prefix.
|
||||
async fn start_mrf_persister(&self) {
|
||||
let Some(mut rx) = self.mrf_save_rx.lock().await.take() else {
|
||||
return;
|
||||
@@ -3511,8 +3421,6 @@ mod tests {
|
||||
allow_first_read: Notify,
|
||||
read_count: AtomicUsize,
|
||||
delete_storage_calls: AtomicUsize,
|
||||
delete_error: StdMutex<EcstoreError>,
|
||||
replay_stamp: StdMutex<Option<Vec<u8>>>,
|
||||
reported_size: StdMutex<Option<i64>>,
|
||||
stream_read_bytes: Arc<AtomicUsize>,
|
||||
write_count: AtomicUsize,
|
||||
@@ -3580,24 +3488,6 @@ mod tests {
|
||||
_h: Self::HeaderMap,
|
||||
_opts: &Self::ObjectOptions,
|
||||
) -> Result<Self::GetObjectReader, Self::Error> {
|
||||
if object.ends_with(ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP) {
|
||||
let stamp = self
|
||||
.shared
|
||||
.replay_stamp
|
||||
.lock()
|
||||
.expect("test replay stamp lock should not be poisoned")
|
||||
.clone()
|
||||
.ok_or(EcstoreError::FileNotFound)?;
|
||||
return Ok(Self::GetObjectReader {
|
||||
object_info: ObjectInfo {
|
||||
size: i64::try_from(stamp.len()).expect("test stamp length should fit i64"),
|
||||
..Default::default()
|
||||
},
|
||||
stream: Box::new(Cursor::new(stamp)),
|
||||
buffered_body: None,
|
||||
body_source: Default::default(),
|
||||
});
|
||||
}
|
||||
if !object.ends_with("/.replication/resync.bin")
|
||||
&& !object.ends_with("config/replication/mrf.bin")
|
||||
&& !object.ends_with("config/replication/force-delete.bin")
|
||||
@@ -3661,16 +3551,6 @@ mod tests {
|
||||
let lock = self.new_ns_lock(bucket, object).await?;
|
||||
Some(lock.get_write_lock(Duration::from_secs(10)).await?)
|
||||
};
|
||||
if object.ends_with(ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP) {
|
||||
let mut stamp = Vec::new();
|
||||
data.stream.read_to_end(&mut stamp).await.map_err(EcstoreError::from)?;
|
||||
*self
|
||||
.shared
|
||||
.replay_stamp
|
||||
.lock()
|
||||
.expect("test replay stamp lock should not be poisoned") = Some(stamp);
|
||||
return Ok(ObjectInfo::default());
|
||||
}
|
||||
if opts.http_preconditions.is_some()
|
||||
&& let Some(replacement) = self
|
||||
.shared
|
||||
@@ -3792,12 +3672,7 @@ mod tests {
|
||||
_opts: Self::ObjectOptions,
|
||||
) -> Result<Self::ObjectInfo, Self::Error> {
|
||||
self.shared.delete_storage_calls.fetch_add(1, Ordering::SeqCst);
|
||||
Err(self
|
||||
.shared
|
||||
.delete_error
|
||||
.lock()
|
||||
.expect("delete error lock should not be poisoned")
|
||||
.clone())
|
||||
Err(EcstoreError::NotImplemented)
|
||||
}
|
||||
|
||||
async fn delete_objects(
|
||||
@@ -4296,8 +4171,6 @@ mod tests {
|
||||
allow_first_read: Notify::new(),
|
||||
read_count: AtomicUsize::new(0),
|
||||
delete_storage_calls: AtomicUsize::new(0),
|
||||
delete_error: StdMutex::new(EcstoreError::NotImplemented),
|
||||
replay_stamp: StdMutex::new(None),
|
||||
reported_size: StdMutex::new(None),
|
||||
stream_read_bytes: Arc::new(AtomicUsize::new(0)),
|
||||
write_count: AtomicUsize::new(0),
|
||||
@@ -4348,103 +4221,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_worker_already_purged_source_completes_without_retry() {
|
||||
let arn = "arn:test";
|
||||
for delete_marker_version in [false, true] {
|
||||
for version_id in [Uuid::new_v4(), Uuid::nil()] {
|
||||
for error in [
|
||||
EcstoreError::VersionNotFound("source".to_string(), "purged".to_string(), version_id.to_string()),
|
||||
EcstoreError::ObjectNotFound("source".to_string(), "purged".to_string()),
|
||||
EcstoreError::FileVersionNotFound,
|
||||
EcstoreError::FileNotFound,
|
||||
] {
|
||||
let shared = empty_resync_shared_state();
|
||||
*shared.delete_error.lock().expect("delete error lock should not be poisoned") = error;
|
||||
let storage = Arc::new(LoadResyncNodeStore::new("purged-source", shared.clone()));
|
||||
let delete = DeletedObjectReplicationInfo {
|
||||
bucket: "source".to_string(),
|
||||
delete_object: ReplicationDeletedObject {
|
||||
object_name: "purged".to_string(),
|
||||
version_id: (!delete_marker_version).then_some(version_id),
|
||||
delete_marker_version_id: delete_marker_version.then_some(version_id),
|
||||
replication_state: Some(ReplicationState {
|
||||
replicate_decision_str: test_replicate_decision(&[arn]).to_string(),
|
||||
purge_targets: HashMap::from([(arn.to_string(), VersionPurgeStatusType::Complete)]),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
},
|
||||
op_type: ReplicationType::Delete,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert!(
|
||||
replicate_delete_with_outcome(delete, storage).await,
|
||||
"a confirmed target purge must complete when the source is already absent (marker={delete_marker_version}, version={version_id})"
|
||||
);
|
||||
assert_eq!(
|
||||
shared.delete_storage_calls.load(Ordering::SeqCst),
|
||||
2 + usize::from(!delete_marker_version && !version_id.is_nil()),
|
||||
"the worker must lock and attempt writeback, reading a non-null data version first"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_worker_missing_source_does_not_hide_unfinished_replication() {
|
||||
let arn = "arn:test";
|
||||
for (version_purge, target_status, error) in [
|
||||
(
|
||||
true,
|
||||
VersionPurgeStatusType::Pending,
|
||||
EcstoreError::ObjectNotFound("source".to_string(), "purged".to_string()),
|
||||
),
|
||||
(true, VersionPurgeStatusType::Complete, EcstoreError::ErasureWriteQuorum),
|
||||
(
|
||||
false,
|
||||
VersionPurgeStatusType::Empty,
|
||||
EcstoreError::ObjectNotFound("source".to_string(), "purged".to_string()),
|
||||
),
|
||||
] {
|
||||
let shared = empty_resync_shared_state();
|
||||
*shared.delete_error.lock().expect("delete error lock should not be poisoned") = error;
|
||||
let storage = Arc::new(LoadResyncNodeStore::new("unfinished-delete", shared));
|
||||
let delete = DeletedObjectReplicationInfo {
|
||||
bucket: "source".to_string(),
|
||||
delete_object: ReplicationDeletedObject {
|
||||
object_name: "purged".to_string(),
|
||||
version_id: version_purge.then(Uuid::new_v4),
|
||||
delete_marker: !version_purge,
|
||||
replication_state: Some(ReplicationState {
|
||||
replicate_decision_str: test_replicate_decision(&[arn]).to_string(),
|
||||
targets: if version_purge {
|
||||
HashMap::new()
|
||||
} else {
|
||||
HashMap::from([(arn.to_string(), ReplicationStatusType::Completed)])
|
||||
},
|
||||
purge_targets: if version_purge {
|
||||
HashMap::from([(arn.to_string(), target_status)])
|
||||
} else {
|
||||
HashMap::new()
|
||||
},
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
},
|
||||
op_type: ReplicationType::Delete,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert!(
|
||||
!replicate_delete_with_outcome(delete, storage).await,
|
||||
"unfinished target work, source I/O failures, and marker creation must remain retryable"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_worker_preserves_persisted_admission_without_live_config() {
|
||||
let arn = "arn:test";
|
||||
@@ -5018,9 +4794,10 @@ mod tests {
|
||||
#[tokio::test]
|
||||
async fn heal_queue_marks_missing_versioning_state_as_missed() {
|
||||
use super::super::replication_target_boundary::BucketTargets;
|
||||
use s3s::dto::{
|
||||
DeleteReplication, DeleteReplicationStatus, Destination, ReplicationConfiguration, ReplicationRule,
|
||||
ReplicationRuleStatus,
|
||||
use rustfs_gateway_types::dto::Status;
|
||||
use rustfs_gateway_types::persistence::{
|
||||
PersistedReplicationConfiguration, PersistedReplicationDestination, PersistedReplicationRule,
|
||||
PersistedReplicationStatus,
|
||||
};
|
||||
|
||||
let arn = "arn:rustfs:replication:us-east-1:target:bucket";
|
||||
@@ -5035,14 +4812,14 @@ mod tests {
|
||||
..Default::default()
|
||||
},
|
||||
ReplicationConfig::new(
|
||||
Some(ReplicationConfiguration {
|
||||
Some(PersistedReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![ReplicationRule {
|
||||
rules: vec![PersistedReplicationRule {
|
||||
delete_marker_replication: None,
|
||||
delete_replication: Some(DeleteReplication {
|
||||
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED),
|
||||
delete_replication: Some(PersistedReplicationStatus {
|
||||
status: Status::ENABLED.to_string(),
|
||||
}),
|
||||
destination: Destination {
|
||||
destination: PersistedReplicationDestination {
|
||||
bucket: arn.to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
@@ -5052,7 +4829,7 @@ mod tests {
|
||||
prefix: Some(String::new()),
|
||||
priority: Some(1),
|
||||
source_selection_criteria: None,
|
||||
status: ReplicationRuleStatus::from_static(ReplicationRuleStatus::ENABLED),
|
||||
status: Status::ENABLED.to_string(),
|
||||
}],
|
||||
}),
|
||||
Some(BucketTargets::default()),
|
||||
@@ -5528,8 +5305,6 @@ mod tests {
|
||||
allow_first_read: Notify::new(),
|
||||
read_count: AtomicUsize::new(0),
|
||||
delete_storage_calls: AtomicUsize::new(0),
|
||||
delete_error: StdMutex::new(EcstoreError::NotImplemented),
|
||||
replay_stamp: StdMutex::new(None),
|
||||
reported_size: StdMutex::new(None),
|
||||
stream_read_bytes: Arc::new(AtomicUsize::new(0)),
|
||||
write_count: AtomicUsize::new(0),
|
||||
@@ -6494,108 +6269,6 @@ mod tests {
|
||||
assert_eq!(retained.target_arns, entry.target_arns);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mrf_replay_loop_drains_entries_persisted_after_startup() {
|
||||
let shared = empty_resync_shared_state();
|
||||
let storage = Arc::new(LoadResyncNodeStore::new("mrf-runtime-replay", shared.clone()));
|
||||
// A force-delete intent from a superseded generation replays as
|
||||
// Skipped, so a pass that reaches it acknowledges it off the journal.
|
||||
let stale_intent = |object: &str| MrfReplicateEntry {
|
||||
bucket: "mrf-runtime-replay".to_string(),
|
||||
object: object.to_string(),
|
||||
op: MrfOpKind::Delete,
|
||||
force_delete_id: Some(uuid::Uuid::new_v4()),
|
||||
force_delete_local_commit: false,
|
||||
target_arns: vec!["arn:rustfs:replication:target-a".to_string()],
|
||||
..Default::default()
|
||||
};
|
||||
let journal_drained = || async {
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
while !shared.data.lock().expect("test data lock should not be poisoned").is_empty() {
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.is_ok()
|
||||
};
|
||||
|
||||
let handle = spawn_mrf_replay_loop(storage, Duration::from_millis(20));
|
||||
for object in ["first", "second"] {
|
||||
*shared.data.lock().expect("test data lock should not be poisoned") =
|
||||
encode_mrf_file(&[stale_intent(object)]).expect("MRF entry should encode");
|
||||
assert!(
|
||||
journal_drained().await,
|
||||
"an MRF entry persisted while the server runs must be replayed without a restart ({object})"
|
||||
);
|
||||
}
|
||||
handle.abort();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mrf_periodic_replay_runs_once_per_interval_across_nodes() {
|
||||
let shared = empty_resync_shared_state();
|
||||
let node_a = Arc::new(LoadResyncNodeStore::new("mrf-node-a", shared.clone()));
|
||||
let node_b = Arc::new(LoadResyncNodeStore::new("mrf-node-b", shared.clone()));
|
||||
let interval = Duration::from_secs(1);
|
||||
let persist_backlog = |object: &str| {
|
||||
let entry = MrfReplicateEntry {
|
||||
bucket: "mrf-node-replay".to_string(),
|
||||
object: object.to_string(),
|
||||
op: MrfOpKind::Delete,
|
||||
force_delete_id: Some(uuid::Uuid::new_v4()),
|
||||
force_delete_local_commit: false,
|
||||
target_arns: vec!["arn:rustfs:replication:target-a".to_string()],
|
||||
..Default::default()
|
||||
};
|
||||
*shared.data.lock().expect("test data lock should not be poisoned") =
|
||||
encode_mrf_file(&[entry]).expect("MRF entry should encode");
|
||||
};
|
||||
let journal_is_empty = || shared.data.lock().expect("test data lock should not be poisoned").is_empty();
|
||||
|
||||
persist_backlog("first");
|
||||
assert!(replay_mrf_journal(&node_a, Some(interval)).await, "node A's periodic pass should replay");
|
||||
assert!(journal_is_empty(), "node A's pass should acknowledge the replayed entry");
|
||||
|
||||
// Node B's timer fires within the same interval with a backlog present.
|
||||
persist_backlog("second");
|
||||
assert!(
|
||||
!replay_mrf_journal(&node_b, Some(interval)).await,
|
||||
"a node must not replay within one interval of another node's pass"
|
||||
);
|
||||
assert!(!journal_is_empty(), "the backlog should wait for the next interval");
|
||||
|
||||
assert!(
|
||||
replay_mrf_journal(&node_b, None).await,
|
||||
"the startup pass should replay regardless of the shared stamp"
|
||||
);
|
||||
assert!(journal_is_empty(), "the startup pass should acknowledge the replayed entry");
|
||||
|
||||
persist_backlog("third");
|
||||
tokio::time::sleep(interval).await;
|
||||
assert!(
|
||||
replay_mrf_journal(&node_a, Some(interval)).await,
|
||||
"a periodic pass should replay once the interval has elapsed"
|
||||
);
|
||||
assert!(journal_is_empty(), "the due pass should acknowledge the replayed entry");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mrf_replay_due_tolerates_missing_and_skewed_stamps() {
|
||||
let interval = Duration::from_secs(300);
|
||||
let now = 1_000_000_000;
|
||||
assert!(mrf_replay_due(None, now, interval));
|
||||
assert!(!mrf_replay_due(Some(now - 299_999), now, interval));
|
||||
assert!(mrf_replay_due(Some(now - 300_000), now, interval));
|
||||
assert!(
|
||||
!mrf_replay_due(Some(now + 1_000), now, interval),
|
||||
"small clock skew should still space passes"
|
||||
);
|
||||
assert!(
|
||||
mrf_replay_due(Some(now + 300_000), now, interval),
|
||||
"a stamp an interval ahead must not suppress replay"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mrf_entry_delete_marker_roundtrip() {
|
||||
let dm_vid = Uuid::new_v4();
|
||||
|
||||
@@ -85,6 +85,8 @@ use http_body_util::StreamBody;
|
||||
use metrics::counter;
|
||||
#[cfg(test)]
|
||||
use rmp_serde;
|
||||
#[cfg(test)]
|
||||
use rustfs_gateway_types::persistence::PersistedReplicationConfiguration;
|
||||
use rustfs_s3_types::EventName;
|
||||
use rustfs_utils::http::{
|
||||
AMZ_OBJECT_LOCK_LEGAL_HOLD, AMZ_OBJECT_LOCK_MODE, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, AMZ_TAGGING_DIRECTIVE,
|
||||
@@ -92,8 +94,6 @@ use rustfs_utils::http::{
|
||||
insert_str, replication_target_versions,
|
||||
};
|
||||
use rustfs_utils::{DEFAULT_SIP_HASH_KEY, get_env_usize, sip_hash};
|
||||
#[cfg(test)]
|
||||
use s3s::dto::ReplicationConfiguration;
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::fmt::Display;
|
||||
use std::pin::Pin;
|
||||
@@ -2809,22 +2809,17 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
|
||||
true
|
||||
}
|
||||
Err(e) => {
|
||||
// A batch purge or another worker may have removed this version
|
||||
// already. Remote failures still control the final outcome below.
|
||||
let source_already_purged = is_version_purge && (is_err_object_not_found(&e) || is_err_version_not_found(&e));
|
||||
if !source_already_purged {
|
||||
error!(
|
||||
event = EVENT_RESYNC_TARGET_OPERATION_FAILED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||
bucket = %bucket,
|
||||
arn = %dobj.target_arn,
|
||||
object = %dobj.delete_object.object_name,
|
||||
operation = "apply_replication_delete_state",
|
||||
error = %e,
|
||||
"Replication target operation failed"
|
||||
);
|
||||
}
|
||||
error!(
|
||||
event = EVENT_RESYNC_TARGET_OPERATION_FAILED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||
bucket = %bucket,
|
||||
arn = %dobj.target_arn,
|
||||
object = %dobj.delete_object.object_name,
|
||||
operation = "apply_replication_delete_state",
|
||||
error = %e,
|
||||
"Replication target operation failed"
|
||||
);
|
||||
send_event(EventArgs {
|
||||
event_name,
|
||||
bucket_name: bucket.clone(),
|
||||
@@ -2837,7 +2832,7 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
|
||||
},
|
||||
..Default::default()
|
||||
});
|
||||
source_already_purged
|
||||
false
|
||||
}
|
||||
};
|
||||
|
||||
@@ -3015,7 +3010,7 @@ const DELETE_MARKER_PURGE_WATCH_INTERVAL: TokioDuration = TokioDuration::from_se
|
||||
/// If the marker disappears (deleted before or while the replica landed),
|
||||
/// purge the replicated marker from the targets, retrying failed targets on
|
||||
/// later rounds. When the window drains with targets still dirty, persist the
|
||||
/// purge intent as a durable MRF entry so the next MRF replay pass re-drives it through
|
||||
/// purge intent as a durable MRF entry so the next startup replays it through
|
||||
/// `purge_stale_delete_marker_targets`.
|
||||
async fn watch_and_purge_source_delete_marker<S: ReplicationStorage>(
|
||||
bucket: String,
|
||||
@@ -3106,7 +3101,7 @@ async fn enqueue_delete_marker_purge_mrf(dobj: &DeletedObjectReplicationInfo, fa
|
||||
/// Heal callers retry for the full watch window because the startup MRF
|
||||
/// processor runs before bucket metadata (and thus target clients) finishes
|
||||
/// initializing — the first pass can see `target_client_missing` and a later
|
||||
/// round resolves the client; replay passes are serial background work, so
|
||||
/// round resolves the client; the replay loop is serial and startup-only, so
|
||||
/// blocking it for up to the window per dirty entry is acceptable. Live
|
||||
/// callers run on replication workers where a down target would pin a worker
|
||||
/// for the whole window, so they attempt once and lean on the durable intent
|
||||
@@ -3287,18 +3282,10 @@ async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &Deleted
|
||||
|
||||
let mut join_set = JoinSet::new();
|
||||
let mut all_succeeded = true;
|
||||
let mut target_infos = Vec::with_capacity(tgt_arns.len());
|
||||
|
||||
for arn in tgt_arns {
|
||||
let Some(tgt_client) = ReplicationTargetStore::remote_target_client(bucket, &arn).await else {
|
||||
all_succeeded = false;
|
||||
target_infos.push(force_delete_target_info(
|
||||
dobj.op_type,
|
||||
&arn,
|
||||
"",
|
||||
false,
|
||||
Err(TARGET_CLIENT_UNAVAILABLE_ERROR.to_string()),
|
||||
));
|
||||
debug!(
|
||||
event = EVENT_REPLICATION_FORCE_DELETE_SKIPPED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
@@ -3324,7 +3311,6 @@ async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &Deleted
|
||||
|
||||
let bucket = bucket.clone();
|
||||
let object_name = object_name.clone();
|
||||
let op_type = dobj.op_type;
|
||||
|
||||
join_set.spawn(async move {
|
||||
if ReplicationTargetStore::target_is_offline(&tgt_client).await {
|
||||
@@ -3349,16 +3335,9 @@ async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &Deleted
|
||||
user_agent: "Internal: [Replication]".to_string(),
|
||||
..Default::default()
|
||||
});
|
||||
return force_delete_target_info(
|
||||
op_type,
|
||||
&tgt_client.arn,
|
||||
&tgt_client.endpoint,
|
||||
tgt_client.secure,
|
||||
Err(FORCE_DELETE_TARGET_OFFLINE_ERROR.to_string()),
|
||||
);
|
||||
return false;
|
||||
}
|
||||
|
||||
let started = Instant::now();
|
||||
if let Err(e) = tgt_client
|
||||
.remove_object(&tgt_client.bucket, &object_name, None, replication_force_delete_remove_options())
|
||||
.await
|
||||
@@ -3385,25 +3364,16 @@ async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &Deleted
|
||||
user_agent: "Internal: [Replication]".to_string(),
|
||||
..Default::default()
|
||||
});
|
||||
return force_delete_target_info(
|
||||
op_type,
|
||||
&tgt_client.arn,
|
||||
&tgt_client.endpoint,
|
||||
tgt_client.secure,
|
||||
Err(e.to_string()),
|
||||
);
|
||||
return false;
|
||||
}
|
||||
|
||||
force_delete_target_info(op_type, &tgt_client.arn, &tgt_client.endpoint, tgt_client.secure, Ok(started.elapsed()))
|
||||
true
|
||||
});
|
||||
}
|
||||
|
||||
while let Some(result) = join_set.join_next().await {
|
||||
match result {
|
||||
Ok(info) => {
|
||||
all_succeeded &= info.replication_status == ReplicationStatusType::Completed;
|
||||
target_infos.push(info);
|
||||
}
|
||||
Ok(success) => all_succeeded &= success,
|
||||
Err(error) => {
|
||||
all_succeeded = false;
|
||||
error!(
|
||||
@@ -3420,8 +3390,6 @@ async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &Deleted
|
||||
}
|
||||
}
|
||||
|
||||
record_force_delete_target_stats(bucket, &target_infos).await;
|
||||
|
||||
if all_succeeded
|
||||
&& let Some(operation_id) = dobj.delete_object.force_delete_id
|
||||
&& let Err(error) = super::replication_pool::complete_force_delete_intent(storage, operation_id).await
|
||||
@@ -3441,53 +3409,6 @@ async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &Deleted
|
||||
all_succeeded
|
||||
}
|
||||
|
||||
const FORCE_DELETE_TARGET_OFFLINE_ERROR: &str = "replication target is offline";
|
||||
|
||||
/// One target's outcome of a replicated force-delete in the shape the
|
||||
/// replication statistics consume. A force-delete carries no per-version
|
||||
/// replication state, so the previous status is always `Pending`: that is
|
||||
/// the transition `ReplicationStats::update` counts as a failure, whereas an
|
||||
/// `Empty` previous status would drop the outcome and leave `replication
|
||||
/// status` reporting zero failures while the peer still holds the objects.
|
||||
fn force_delete_target_info(
|
||||
op_type: ReplicationType,
|
||||
arn: &str,
|
||||
endpoint: &str,
|
||||
secure: bool,
|
||||
outcome: std::result::Result<std::time::Duration, String>,
|
||||
) -> ReplicatedTargetInfo {
|
||||
let mut rinfo = ReplicatedTargetInfo {
|
||||
arn: arn.to_string(),
|
||||
op_type,
|
||||
endpoint: endpoint.to_string(),
|
||||
secure,
|
||||
prev_replication_status: ReplicationStatusType::Pending,
|
||||
..Default::default()
|
||||
};
|
||||
match outcome {
|
||||
Ok(duration) => {
|
||||
rinfo.replication_status = ReplicationStatusType::Completed;
|
||||
rinfo.duration = duration;
|
||||
}
|
||||
Err(error) => {
|
||||
rinfo.replication_status = ReplicationStatusType::Failed;
|
||||
rinfo.error = Some(error);
|
||||
}
|
||||
}
|
||||
rinfo
|
||||
}
|
||||
|
||||
async fn record_force_delete_target_stats(bucket: &str, targets: &[ReplicatedTargetInfo]) {
|
||||
let Some(stats) = runtime_sources::replication_stats() else {
|
||||
return;
|
||||
};
|
||||
for target in targets {
|
||||
stats
|
||||
.update(bucket, target, target.replication_status.clone(), target.prev_replication_status.clone())
|
||||
.await;
|
||||
}
|
||||
}
|
||||
|
||||
fn unavailable_delete_target_info(dobj: &DeletedObjectReplicationInfo, arn: &str) -> ReplicatedTargetInfo {
|
||||
let mut rinfo = dobj
|
||||
.delete_object
|
||||
@@ -6579,9 +6500,10 @@ mod tests {
|
||||
|
||||
use super::super::replication_target_boundary::{BucketTarget, BucketTargets};
|
||||
use super::*;
|
||||
use s3s::dto::{
|
||||
BucketVersioningStatus, DeleteReplication, DeleteReplicationStatus, Destination, ExcludedPrefix, ReplicationRule,
|
||||
ReplicationRuleStatus, VersioningConfiguration,
|
||||
use rustfs_gateway_types::dto::Status;
|
||||
use rustfs_gateway_types::persistence::PersistedVersioningConfiguration as VersioningConfiguration;
|
||||
use rustfs_gateway_types::persistence::{
|
||||
PersistedReplicationDestination, PersistedReplicationRule, PersistedReplicationStatus,
|
||||
};
|
||||
use std::collections::HashMap;
|
||||
use time::OffsetDateTime;
|
||||
@@ -7523,51 +7445,6 @@ mod tests {
|
||||
server.join().expect("test HTTP server should finish");
|
||||
}
|
||||
|
||||
/// A force-delete the peer rejects must reach the replication failure
|
||||
/// counters. A replicated force-delete used to only log the rejection, so
|
||||
/// `replication status` reported zero failures while the sites diverged.
|
||||
#[tokio::test]
|
||||
async fn force_delete_target_outcomes_reach_replication_stats() {
|
||||
let stats = super::super::replication_state::ReplicationStats::new();
|
||||
let denied = force_delete_target_info(
|
||||
ReplicationType::Delete,
|
||||
"arn:rustfs:replication::denied:bucket",
|
||||
"http://peer-a:9000",
|
||||
false,
|
||||
Err("remove_object failed: AccessDenied".to_string()),
|
||||
);
|
||||
assert_eq!(denied.replication_status, ReplicationStatusType::Failed);
|
||||
assert_eq!(denied.prev_replication_status, ReplicationStatusType::Pending);
|
||||
assert_eq!(denied.error.as_deref(), Some("remove_object failed: AccessDenied"));
|
||||
|
||||
let accepted = force_delete_target_info(
|
||||
ReplicationType::Heal,
|
||||
"arn:rustfs:replication::accepted:bucket",
|
||||
"https://peer-b:9000",
|
||||
true,
|
||||
Ok(std::time::Duration::from_millis(3)),
|
||||
);
|
||||
assert_eq!(accepted.replication_status, ReplicationStatusType::Completed);
|
||||
assert!(accepted.error.is_none());
|
||||
|
||||
for target in [&denied, &accepted] {
|
||||
stats
|
||||
.update(
|
||||
"bucket",
|
||||
target,
|
||||
target.replication_status.clone(),
|
||||
target.prev_replication_status.clone(),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
let bucket_stats = stats.get("bucket").await;
|
||||
let denied_stat = bucket_stats.stats.get(&denied.arn).expect("denied target: stats entry");
|
||||
assert_eq!((denied_stat.failed.count, denied_stat.replicated_count), (1, 0));
|
||||
let accepted_stat = bucket_stats.stats.get(&accepted.arn).expect("accepted target: stats entry");
|
||||
assert_eq!((accepted_stat.failed.count, accepted_stat.replicated_count), (0, 1));
|
||||
}
|
||||
|
||||
/// AWS answers `HEAD ?versionId=<source marker id>` with a bodiless 400.
|
||||
/// The marker is verified through the latest version instead: a
|
||||
/// versionless 404 with `x-amz-delete-marker: true` proves it propagated.
|
||||
@@ -8692,7 +8569,7 @@ mod tests {
|
||||
..Default::default()
|
||||
};
|
||||
let rcfg = ReplicationConfig::new(
|
||||
Some(ReplicationConfiguration {
|
||||
Some(PersistedReplicationConfiguration {
|
||||
role: role.to_string(),
|
||||
rules: vec![],
|
||||
}),
|
||||
@@ -8718,7 +8595,7 @@ mod tests {
|
||||
ReplicationVersioningStore::install_prefix_state_test_config(
|
||||
&bucket,
|
||||
VersioningConfiguration {
|
||||
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
|
||||
status: Some("Enabled".to_owned()),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
@@ -8751,22 +8628,20 @@ mod tests {
|
||||
ReplicationVersioningStore::install_prefix_state_test_config(
|
||||
&bucket,
|
||||
VersioningConfiguration {
|
||||
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
|
||||
excluded_prefixes: Some(vec![ExcludedPrefix {
|
||||
prefix: Some("archive/".to_string()),
|
||||
}]),
|
||||
status: Some("Enabled".to_owned()),
|
||||
excluded_prefixes: Some(vec![Some("archive/".to_string())]),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
let rcfg = ReplicationConfig::new(
|
||||
Some(ReplicationConfiguration {
|
||||
Some(PersistedReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![ReplicationRule {
|
||||
rules: vec![PersistedReplicationRule {
|
||||
delete_marker_replication: None,
|
||||
delete_replication: Some(DeleteReplication {
|
||||
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED),
|
||||
delete_replication: Some(PersistedReplicationStatus {
|
||||
status: Status::ENABLED.to_string(),
|
||||
}),
|
||||
destination: Destination {
|
||||
destination: PersistedReplicationDestination {
|
||||
bucket: arn.to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
@@ -8776,7 +8651,7 @@ mod tests {
|
||||
prefix: Some(String::new()),
|
||||
priority: Some(1),
|
||||
source_selection_criteria: None,
|
||||
status: ReplicationRuleStatus::from_static(ReplicationRuleStatus::ENABLED),
|
||||
status: Status::ENABLED.to_string(),
|
||||
}],
|
||||
}),
|
||||
Some(BucketTargets {
|
||||
@@ -8812,14 +8687,14 @@ mod tests {
|
||||
let admitted_arn = "arn:rustfs:replication:us-east-1:target:admitted";
|
||||
let current_role = "arn:rustfs:replication:us-east-1:target:current";
|
||||
let rcfg = ReplicationConfig::new(
|
||||
Some(ReplicationConfiguration {
|
||||
Some(PersistedReplicationConfiguration {
|
||||
role: current_role.to_string(),
|
||||
rules: vec![ReplicationRule {
|
||||
rules: vec![PersistedReplicationRule {
|
||||
delete_marker_replication: None,
|
||||
delete_replication: Some(DeleteReplication {
|
||||
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::DISABLED),
|
||||
delete_replication: Some(PersistedReplicationStatus {
|
||||
status: Status::DISABLED.to_string(),
|
||||
}),
|
||||
destination: Destination {
|
||||
destination: PersistedReplicationDestination {
|
||||
bucket: current_role.to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
@@ -8829,7 +8704,7 @@ mod tests {
|
||||
prefix: Some(String::new()),
|
||||
priority: Some(1),
|
||||
source_selection_criteria: None,
|
||||
status: ReplicationRuleStatus::from_static(ReplicationRuleStatus::ENABLED),
|
||||
status: Status::ENABLED.to_string(),
|
||||
}],
|
||||
}),
|
||||
Some(BucketTargets {
|
||||
@@ -8877,12 +8752,12 @@ mod tests {
|
||||
#[tokio::test]
|
||||
async fn heal_rejects_semantically_invalid_replication_config() {
|
||||
let rcfg = ReplicationConfig::new(
|
||||
Some(ReplicationConfiguration {
|
||||
Some(PersistedReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![ReplicationRule {
|
||||
rules: vec![PersistedReplicationRule {
|
||||
delete_marker_replication: None,
|
||||
delete_replication: None,
|
||||
destination: Destination {
|
||||
destination: PersistedReplicationDestination {
|
||||
bucket: "arn:rustfs:replication:us-east-1:target:bucket".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
@@ -8892,7 +8767,7 @@ mod tests {
|
||||
prefix: Some(String::new()),
|
||||
priority: Some(1),
|
||||
source_selection_criteria: None,
|
||||
status: ReplicationRuleStatus::from_static("Enabld"),
|
||||
status: "Enabld".to_owned(),
|
||||
}],
|
||||
}),
|
||||
Some(BucketTargets::default()),
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use s3s::dto::ReplicationConfiguration;
|
||||
use rustfs_gateway_types::persistence::PersistedReplicationConfiguration;
|
||||
|
||||
use super::replication_config_boundary::{ObjectOpts, ReplicationConfigurationExt};
|
||||
use super::replication_filemeta_boundary::ReplicationType;
|
||||
@@ -20,7 +20,7 @@ use super::replication_filemeta_boundary::ReplicationType;
|
||||
pub(crate) struct ReplicationTargetConfigBridge;
|
||||
|
||||
impl ReplicationTargetConfigBridge {
|
||||
pub(crate) fn target_is_used_by_rules(config: &ReplicationConfiguration, arn: &str) -> bool {
|
||||
pub(crate) fn target_is_used_by_rules(config: &PersistedReplicationConfiguration, arn: &str) -> bool {
|
||||
config
|
||||
.filter_target_arns(&ObjectOpts {
|
||||
op_type: ReplicationType::All,
|
||||
@@ -37,7 +37,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn target_config_bridge_matches_role_target() {
|
||||
let config = ReplicationConfiguration {
|
||||
let config = PersistedReplicationConfiguration {
|
||||
role: " arn:target ".to_string(),
|
||||
rules: Vec::new(),
|
||||
};
|
||||
|
||||
@@ -15,7 +15,7 @@
|
||||
use super::replication_error_boundary::Result;
|
||||
use crate::bucket::{versioning::VersioningApi as _, versioning_sys::BucketVersioningSys};
|
||||
#[cfg(test)]
|
||||
use s3s::dto::VersioningConfiguration;
|
||||
use rustfs_gateway_types::persistence::PersistedVersioningConfiguration as VersioningConfiguration;
|
||||
#[cfg(test)]
|
||||
use std::{collections::HashMap, sync::LazyLock, sync::Mutex};
|
||||
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use s3s::dto::Tag;
|
||||
use rustfs_gateway_types::cors_tagging::PersistedTag as Tag;
|
||||
use std::cmp::Ordering;
|
||||
use std::collections::HashMap;
|
||||
use url::form_urlencoded;
|
||||
|
||||
@@ -15,9 +15,23 @@
|
||||
use crate::disk::{MIGRATING_META_BUCKET, RUSTFS_META_BUCKET};
|
||||
use crate::error::{Error, Result, StorageError};
|
||||
use regex::Regex;
|
||||
use rustfs_gateway_types::cors_tagging::{
|
||||
CorsTaggingCodecError, PersistedCorsConfiguration, PersistedTagging, parse_cors, parse_tagging, serialize_cors,
|
||||
serialize_tagging,
|
||||
};
|
||||
use rustfs_gateway_types::persistence::{
|
||||
PersistedAccelerateConfiguration, PersistedBucketEncryptionConfiguration, PersistedBucketLoggingStatus,
|
||||
PersistedLifecycleConfiguration, PersistedNotificationConfiguration, PersistedObjectLockConfiguration,
|
||||
PersistedPublicAccessBlockConfiguration, PersistedReplicationConfiguration, PersistedRequestPaymentConfiguration,
|
||||
PersistedVersioningConfiguration, PersistedWebsiteConfiguration, PersistenceCodecError, parse_accelerate,
|
||||
parse_bucket_encryption, parse_bucket_logging, parse_lifecycle_legacy_dates, parse_notification, parse_object_lock,
|
||||
parse_public_access_block, parse_replication, parse_request_payment, parse_versioning, parse_website, serialize_accelerate,
|
||||
serialize_bucket_encryption, serialize_bucket_logging, serialize_lifecycle_legacy_dates, serialize_notification,
|
||||
serialize_object_lock, serialize_public_access_block, serialize_replication, serialize_request_payment, serialize_versioning,
|
||||
serialize_website,
|
||||
};
|
||||
use rustfs_utils::WindowsFileNameError;
|
||||
use rustfs_utils::path::SLASH_SEPARATOR;
|
||||
use s3s::xml;
|
||||
use tracing::instrument;
|
||||
|
||||
pub fn is_meta_bucketname(name: &str) -> bool {
|
||||
@@ -74,37 +88,173 @@ pub fn check_valid_bucket_name_strict(bucket_name: &str) -> Result<()> {
|
||||
check_bucket_name_common(bucket_name, true)
|
||||
}
|
||||
|
||||
// RUSTFS_COMPAT_TODO(s3gate-metadata-xml): the s3s codec reads persisted XML during migration. Remove after every supported writer uses the gateway codec and every retained metadata object and backup archive is verified or rewritten.
|
||||
pub fn deserialize<T>(input: &[u8]) -> xml::DeResult<T>
|
||||
where
|
||||
T: for<'xml> xml::Deserialize<'xml>,
|
||||
{
|
||||
let mut d = xml::Deserializer::new(input);
|
||||
let ans = T::deserialize(&mut d)?;
|
||||
d.expect_eof()?;
|
||||
Ok(ans)
|
||||
/// A bucket configuration persisted as XML in the bucket metadata, together
|
||||
/// with the codec that reads and writes its stored bytes.
|
||||
///
|
||||
/// Every family reads and writes its stored bytes through the gateway
|
||||
/// persistence codecs (`rustfs_gateway_types::persistence` and
|
||||
/// `rustfs_gateway_types::cors_tagging`, rustfs/backlog#2745). The codecs keep
|
||||
/// the persisted bytes the `bucket_config_goldens` pin, so a document the old
|
||||
/// s3s codec wrote reads back and re-encodes unchanged.
|
||||
pub trait BucketConfigXml: Sized {
|
||||
/// Decodes the stored bytes of one configuration document.
|
||||
fn decode_config_xml(input: &[u8]) -> std::result::Result<Self, BucketConfigXmlError>;
|
||||
|
||||
/// Encodes the value as RustFS stores it.
|
||||
fn encode_config_xml(&self) -> std::result::Result<Vec<u8>, BucketConfigXmlError>;
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "xml serialize helper with no caller in this port; the live sibling is deserialize (backlog#1823)"
|
||||
)]
|
||||
pub fn serialize_content<T: xml::SerializeContent>(val: &T) -> xml::SerResult<String> {
|
||||
let mut buf = Vec::with_capacity(256);
|
||||
{
|
||||
let mut ser = xml::Serializer::new(&mut buf);
|
||||
val.serialize_content(&mut ser)?;
|
||||
}
|
||||
Ok(String::from_utf8(buf).unwrap())
|
||||
/// Why a persisted bucket configuration could not be decoded or encoded.
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum BucketConfigXmlError {
|
||||
/// A gateway persistence codec refused the bytes or the value.
|
||||
#[error(transparent)]
|
||||
Persisted(#[from] PersistenceCodecError),
|
||||
/// The gateway CORS or Tagging codec refused the bytes or the value.
|
||||
#[error(transparent)]
|
||||
CorsTagging(#[from] CorsTaggingCodecError),
|
||||
}
|
||||
|
||||
pub fn serialize<T: xml::Serialize>(val: &T) -> xml::SerResult<Vec<u8>> {
|
||||
let mut buf = Vec::with_capacity(256);
|
||||
{
|
||||
let mut ser = xml::Serializer::new(&mut buf);
|
||||
val.serialize(&mut ser)?;
|
||||
/// Decodes one persisted bucket configuration document.
|
||||
pub fn deserialize<T: BucketConfigXml>(input: &[u8]) -> std::result::Result<T, BucketConfigXmlError> {
|
||||
T::decode_config_xml(input)
|
||||
}
|
||||
|
||||
/// Encodes one bucket configuration as RustFS persists it.
|
||||
pub fn serialize<T: BucketConfigXml>(val: &T) -> std::result::Result<Vec<u8>, BucketConfigXmlError> {
|
||||
val.encode_config_xml()
|
||||
}
|
||||
|
||||
impl BucketConfigXml for PersistedReplicationConfiguration {
|
||||
fn decode_config_xml(input: &[u8]) -> std::result::Result<Self, BucketConfigXmlError> {
|
||||
Ok(parse_replication(input)?)
|
||||
}
|
||||
|
||||
fn encode_config_xml(&self) -> std::result::Result<Vec<u8>, BucketConfigXmlError> {
|
||||
Ok(serialize_replication(self)?)
|
||||
}
|
||||
}
|
||||
|
||||
impl BucketConfigXml for PersistedLifecycleConfiguration {
|
||||
fn decode_config_xml(input: &[u8]) -> std::result::Result<Self, BucketConfigXmlError> {
|
||||
Ok(parse_lifecycle_legacy_dates(input)?)
|
||||
}
|
||||
|
||||
fn encode_config_xml(&self) -> std::result::Result<Vec<u8>, BucketConfigXmlError> {
|
||||
Ok(serialize_lifecycle_legacy_dates(self)?)
|
||||
}
|
||||
}
|
||||
|
||||
// The gateway writers of these families are infallible: a value that exists is
|
||||
// always encodable, so only the readers can refuse bytes.
|
||||
impl BucketConfigXml for PersistedAccelerateConfiguration {
|
||||
fn decode_config_xml(input: &[u8]) -> std::result::Result<Self, BucketConfigXmlError> {
|
||||
Ok(parse_accelerate(input)?)
|
||||
}
|
||||
|
||||
fn encode_config_xml(&self) -> std::result::Result<Vec<u8>, BucketConfigXmlError> {
|
||||
Ok(serialize_accelerate(self))
|
||||
}
|
||||
}
|
||||
|
||||
impl BucketConfigXml for PersistedBucketLoggingStatus {
|
||||
fn decode_config_xml(input: &[u8]) -> std::result::Result<Self, BucketConfigXmlError> {
|
||||
Ok(parse_bucket_logging(input)?)
|
||||
}
|
||||
|
||||
fn encode_config_xml(&self) -> std::result::Result<Vec<u8>, BucketConfigXmlError> {
|
||||
Ok(serialize_bucket_logging(self))
|
||||
}
|
||||
}
|
||||
|
||||
impl BucketConfigXml for PersistedCorsConfiguration {
|
||||
fn decode_config_xml(input: &[u8]) -> std::result::Result<Self, BucketConfigXmlError> {
|
||||
Ok(parse_cors(input)?)
|
||||
}
|
||||
|
||||
fn encode_config_xml(&self) -> std::result::Result<Vec<u8>, BucketConfigXmlError> {
|
||||
Ok(serialize_cors(self))
|
||||
}
|
||||
}
|
||||
|
||||
impl BucketConfigXml for PersistedNotificationConfiguration {
|
||||
fn decode_config_xml(input: &[u8]) -> std::result::Result<Self, BucketConfigXmlError> {
|
||||
Ok(parse_notification(input)?)
|
||||
}
|
||||
|
||||
fn encode_config_xml(&self) -> std::result::Result<Vec<u8>, BucketConfigXmlError> {
|
||||
Ok(serialize_notification(self))
|
||||
}
|
||||
}
|
||||
|
||||
impl BucketConfigXml for PersistedObjectLockConfiguration {
|
||||
fn decode_config_xml(input: &[u8]) -> std::result::Result<Self, BucketConfigXmlError> {
|
||||
Ok(parse_object_lock(input)?)
|
||||
}
|
||||
|
||||
fn encode_config_xml(&self) -> std::result::Result<Vec<u8>, BucketConfigXmlError> {
|
||||
Ok(serialize_object_lock(self))
|
||||
}
|
||||
}
|
||||
|
||||
impl BucketConfigXml for PersistedPublicAccessBlockConfiguration {
|
||||
fn decode_config_xml(input: &[u8]) -> std::result::Result<Self, BucketConfigXmlError> {
|
||||
Ok(parse_public_access_block(input)?)
|
||||
}
|
||||
|
||||
fn encode_config_xml(&self) -> std::result::Result<Vec<u8>, BucketConfigXmlError> {
|
||||
Ok(serialize_public_access_block(self))
|
||||
}
|
||||
}
|
||||
|
||||
impl BucketConfigXml for PersistedRequestPaymentConfiguration {
|
||||
fn decode_config_xml(input: &[u8]) -> std::result::Result<Self, BucketConfigXmlError> {
|
||||
Ok(parse_request_payment(input)?)
|
||||
}
|
||||
|
||||
fn encode_config_xml(&self) -> std::result::Result<Vec<u8>, BucketConfigXmlError> {
|
||||
Ok(serialize_request_payment(self))
|
||||
}
|
||||
}
|
||||
|
||||
impl BucketConfigXml for PersistedBucketEncryptionConfiguration {
|
||||
fn decode_config_xml(input: &[u8]) -> std::result::Result<Self, BucketConfigXmlError> {
|
||||
Ok(parse_bucket_encryption(input)?)
|
||||
}
|
||||
|
||||
fn encode_config_xml(&self) -> std::result::Result<Vec<u8>, BucketConfigXmlError> {
|
||||
Ok(serialize_bucket_encryption(self))
|
||||
}
|
||||
}
|
||||
|
||||
impl BucketConfigXml for PersistedTagging {
|
||||
fn decode_config_xml(input: &[u8]) -> std::result::Result<Self, BucketConfigXmlError> {
|
||||
Ok(parse_tagging(input)?)
|
||||
}
|
||||
|
||||
fn encode_config_xml(&self) -> std::result::Result<Vec<u8>, BucketConfigXmlError> {
|
||||
Ok(serialize_tagging(self))
|
||||
}
|
||||
}
|
||||
|
||||
impl BucketConfigXml for PersistedVersioningConfiguration {
|
||||
fn decode_config_xml(input: &[u8]) -> std::result::Result<Self, BucketConfigXmlError> {
|
||||
Ok(parse_versioning(input)?)
|
||||
}
|
||||
|
||||
fn encode_config_xml(&self) -> std::result::Result<Vec<u8>, BucketConfigXmlError> {
|
||||
Ok(serialize_versioning(self))
|
||||
}
|
||||
}
|
||||
|
||||
impl BucketConfigXml for PersistedWebsiteConfiguration {
|
||||
fn decode_config_xml(input: &[u8]) -> std::result::Result<Self, BucketConfigXmlError> {
|
||||
Ok(parse_website(input)?)
|
||||
}
|
||||
|
||||
fn encode_config_xml(&self) -> std::result::Result<Vec<u8>, BucketConfigXmlError> {
|
||||
Ok(serialize_website(self))
|
||||
}
|
||||
Ok(buf)
|
||||
}
|
||||
|
||||
pub fn has_bad_path_component(path: &str) -> bool {
|
||||
@@ -434,6 +584,40 @@ mod tests {
|
||||
use super::*;
|
||||
use proptest::prelude::*;
|
||||
|
||||
/// Every bucket-configuration family reads and writes through its own
|
||||
/// gateway codec (rustfs/backlog#2745): the family's golden document
|
||||
/// decodes and re-encodes to the same bytes, and a document of another
|
||||
/// family is refused instead of decoding into an empty configuration.
|
||||
#[test]
|
||||
fn every_bucket_config_family_uses_its_own_gateway_codec() {
|
||||
fn assert_family<T: BucketConfigXml>(family: &str, own: &[u8], foreign: &[u8]) {
|
||||
let value = deserialize::<T>(own).unwrap_or_else(|err| panic!("{family}: own document must decode: {err}"));
|
||||
let encoded = serialize(&value).unwrap_or_else(|err| panic!("{family}: value must encode: {err}"));
|
||||
assert_eq!(encoded, own, "{family}: re-encoding must keep the persisted bytes");
|
||||
assert!(deserialize::<T>(foreign).is_err(), "{family}: a foreign root element must be refused");
|
||||
}
|
||||
|
||||
macro_rules! golden {
|
||||
($family:literal) => {
|
||||
include_bytes!(concat!("../../tests/fixtures/bucket-config-goldens/", $family, "/minio.out.xml"))
|
||||
};
|
||||
}
|
||||
let tagging = golden!("tagging");
|
||||
let versioning = golden!("versioning");
|
||||
|
||||
assert_family::<PersistedAccelerateConfiguration>("accelerate", golden!("accelerate"), tagging);
|
||||
assert_family::<PersistedBucketLoggingStatus>("logging", golden!("logging"), tagging);
|
||||
assert_family::<PersistedCorsConfiguration>("cors", golden!("cors"), tagging);
|
||||
assert_family::<PersistedNotificationConfiguration>("notification", golden!("notification"), tagging);
|
||||
assert_family::<PersistedObjectLockConfiguration>("object_lock", golden!("object_lock"), tagging);
|
||||
assert_family::<PersistedPublicAccessBlockConfiguration>("public_access_block", golden!("public_access_block"), tagging);
|
||||
assert_family::<PersistedRequestPaymentConfiguration>("request_payment", golden!("request_payment"), tagging);
|
||||
assert_family::<PersistedBucketEncryptionConfiguration>("encryption", golden!("encryption"), tagging);
|
||||
assert_family::<PersistedTagging>("tagging", tagging, versioning);
|
||||
assert_family::<PersistedVersioningConfiguration>("versioning", versioning, tagging);
|
||||
assert_family::<PersistedWebsiteConfiguration>("website", golden!("website"), tagging);
|
||||
}
|
||||
|
||||
/// rustfs#5785: keys whose path segments exceed the on-disk NAME_MAX
|
||||
/// budget must be rejected up front as ObjectNameInvalid (4xx), not leak
|
||||
/// ENAMETOOLONG as InternalError 500 from the disk layer.
|
||||
|
||||
@@ -12,8 +12,8 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use rustfs_gateway_types::persistence::PersistedVersioningConfiguration as VersioningConfiguration;
|
||||
use rustfs_utils::string::match_simple;
|
||||
use s3s::dto::{BucketVersioningStatus, VersioningConfiguration};
|
||||
|
||||
pub trait VersioningApi {
|
||||
fn enabled(&self) -> bool;
|
||||
@@ -28,10 +28,10 @@ pub trait VersioningApi {
|
||||
|
||||
impl VersioningApi for VersioningConfiguration {
|
||||
fn enabled(&self) -> bool {
|
||||
self.status == Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED))
|
||||
self.status.as_deref() == Some("Enabled")
|
||||
}
|
||||
fn prefix_enabled(&self, prefix: &str) -> bool {
|
||||
if self.status != Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)) {
|
||||
if self.status.as_deref() != Some("Enabled") {
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -44,12 +44,10 @@ impl VersioningApi for VersioningConfiguration {
|
||||
}
|
||||
|
||||
if let Some(ref excluded_prefixes) = self.excluded_prefixes {
|
||||
for p in excluded_prefixes.iter() {
|
||||
if let Some(ref sprefix) = p.prefix {
|
||||
let pattern = format!("{sprefix}*");
|
||||
if match_simple(&pattern, prefix) {
|
||||
return false;
|
||||
}
|
||||
for sprefix in excluded_prefixes.iter().flatten() {
|
||||
let pattern = format!("{sprefix}*");
|
||||
if match_simple(&pattern, prefix) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -58,11 +56,11 @@ impl VersioningApi for VersioningConfiguration {
|
||||
}
|
||||
|
||||
fn prefix_suspended(&self, prefix: &str) -> bool {
|
||||
if self.status == Some(BucketVersioningStatus::from_static(BucketVersioningStatus::SUSPENDED)) {
|
||||
if self.status.as_deref() == Some("Suspended") {
|
||||
return true;
|
||||
}
|
||||
|
||||
if self.status == Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)) {
|
||||
if self.status.as_deref() == Some("Enabled") {
|
||||
if prefix.is_empty() {
|
||||
return false;
|
||||
}
|
||||
@@ -75,12 +73,10 @@ impl VersioningApi for VersioningConfiguration {
|
||||
}
|
||||
|
||||
if let Some(ref excluded_prefixes) = self.excluded_prefixes {
|
||||
for p in excluded_prefixes.iter() {
|
||||
if let Some(ref sprefix) = p.prefix {
|
||||
let pattern = format!("{sprefix}*");
|
||||
if match_simple(&pattern, prefix) {
|
||||
return true;
|
||||
}
|
||||
for sprefix in excluded_prefixes.iter().flatten() {
|
||||
let pattern = format!("{sprefix}*");
|
||||
if match_simple(&pattern, prefix) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -92,14 +88,12 @@ impl VersioningApi for VersioningConfiguration {
|
||||
self.prefix_enabled(prefix) || self.prefix_suspended(prefix)
|
||||
}
|
||||
fn suspended(&self) -> bool {
|
||||
self.status == Some(BucketVersioningStatus::from_static(BucketVersioningStatus::SUSPENDED))
|
||||
self.status.as_deref() == Some("Suspended")
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use s3s::dto::{BucketVersioningStatus, ExcludedPrefix};
|
||||
|
||||
use super::*;
|
||||
|
||||
struct LegacyVersioning;
|
||||
@@ -134,10 +128,8 @@ mod tests {
|
||||
#[test]
|
||||
fn delete_state_treats_excluded_prefixes_as_unversioned() {
|
||||
let config = VersioningConfiguration {
|
||||
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
|
||||
excluded_prefixes: Some(vec![ExcludedPrefix {
|
||||
prefix: Some("archive/".to_string()),
|
||||
}]),
|
||||
status: Some("Enabled".to_owned()),
|
||||
excluded_prefixes: Some(vec![Some("archive/".to_string())]),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
@@ -146,7 +138,7 @@ mod tests {
|
||||
assert_eq!(config.delete_state("live/object"), (true, false));
|
||||
|
||||
let suspended = VersioningConfiguration {
|
||||
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::SUSPENDED)),
|
||||
status: Some("Suspended".to_owned()),
|
||||
..Default::default()
|
||||
};
|
||||
assert_eq!(suspended.delete_state("archive/object"), (false, true));
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user